notebooks/End-to-End-Matching-Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "14315d5a",
   "metadata": {},
   "source": [
    "# End to End Matching Example"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3080847d",
   "metadata": {},
   "source": [
    "Please run [Record-Linkage-Example.ipynb](Record-Linkage-Example.ipynb) before this one in order to get the trained model and data."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2eae51fb",
   "metadata": {},
   "source": [
    "## Boilerplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e6be39ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "41394338",
   "metadata": {},
   "outputs": [],
   "source": [
    "from importlib import reload\n",
    "import logging\n",
    "reload(logging)\n",
    "logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.INFO, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "96e239b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.insert(0, '..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23bdf412",
   "metadata": {},
   "outputs": [],
   "source": [
    "import entity_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d581925f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "random_seed = 42\n",
    "torch.manual_seed(random_seed)\n",
    "np.random.seed(random_seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a81e6680",
   "metadata": {},
   "source": [
    "## Loading Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "105a7df5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from ordered_set import OrderedSet\n",
    "\n",
    "def load_pair_set(filepath):\n",
    "    with open(filepath, 'r') as f:\n",
    "        test_pos_pair_set = json.load(f)\n",
    "        return OrderedSet(tuple(pair) for pair in test_pos_pair_set)\n",
    "\n",
    "train_pos_pair_set = load_pair_set('../trained-models/notebooks/rl/rl-train-pos-pairs.json')\n",
    "valid_pos_pair_set = load_pair_set('../trained-models/notebooks/rl/rl-valid-pos-pairs.json')\n",
    "test_pos_pair_set = load_pair_set('../trained-models/notebooks/rl/rl-test-pos-pairs.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0af3fb30",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def load_record_dict(filepath):\n",
    "    with open(filepath, 'r') as f:\n",
    "        record_dict = json.load(f)\n",
    "        return {int(id_): record for id_, record in record_dict.items()}\n",
    "\n",
    "train_record_dict = load_record_dict('../trained-models/notebooks/rl/rl-train-records.json')\n",
    "valid_record_dict = load_record_dict('../trained-models/notebooks/rl/rl-valid-records.json')\n",
    "test_record_dict = load_record_dict('../trained-models/notebooks/rl/rl-test-records.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e691355",
   "metadata": {},
   "source": [
    "## Loading Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9a29f154",
   "metadata": {},
   "outputs": [],
   "source": [
    "from entity_embed import LinkageEmbed\n",
    "\n",
    "model = LinkageEmbed.load_from_checkpoint('../trained-models/notebooks/rl/rl-model.ckpt')\n",
    "model = model.to(torch.device('cuda'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43e1861a",
   "metadata": {},
   "source": [
    "## Blocking"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9287762",
   "metadata": {},
   "source": [
    "Use `sim_threshold = 0.375` to have ~6k pairs in `train_found_pair_set` and have a fair comparison with [baseline-models/End-to-End-Matching-Baseline.ipynb](./baseline-models/End-to-End-Matching-Baseline.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "00a07f05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f1e6f70df41491ab6ae84fdf95952fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "# batch embedding:   0%|          | 0/37 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.45 s, sys: 525 ms, total: 2.98 s\n",
      "Wall time: 1.62 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "5776"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "eval_batch_size = 64\n",
    "ann_k = 100\n",
    "sim_threshold = 0.375\n",
    "\n",
    "train_found_pair_set, train_left_field_vector_dict, train_right_field_vector_dict = model.predict_pairs(\n",
    "    record_dict=train_record_dict,\n",
    "    batch_size=eval_batch_size,\n",
    "    ann_k=ann_k,\n",
    "    sim_threshold=sim_threshold,\n",
    "    show_progress=True,\n",
    "    return_field_embeddings=True\n",
    ")\n",
    "len(train_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2739357b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c36b6ba812c64e5b8b77a0d88729a884",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "# batch embedding:   0%|          | 0/15 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.06 s, sys: 447 ms, total: 1.51 s\n",
      "Wall time: 916 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1026"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "valid_found_pair_set, valid_left_field_vector_dict, valid_right_field_vector_dict = model.predict_pairs(\n",
    "    record_dict=valid_record_dict,\n",
    "    batch_size=eval_batch_size,\n",
    "    ann_k=ann_k,\n",
    "    sim_threshold=sim_threshold,\n",
    "    show_progress=True,\n",
    "    return_field_embeddings=True\n",
    ")\n",
    "len(valid_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5d16e8ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0f503c140004d91b84133a2cb3be997",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "# batch embedding:   0%|          | 0/22 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.5 s, sys: 502 ms, total: 2 s\n",
      "Wall time: 1.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2027"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "test_found_pair_set, test_left_field_vector_dict, test_right_field_vector_dict = model.predict_pairs(\n",
    "    record_dict=test_record_dict,\n",
    "    batch_size=eval_batch_size,\n",
    "    ann_k=ann_k,\n",
    "    sim_threshold=sim_threshold,\n",
    "    show_progress=True,\n",
    "    return_field_embeddings=True\n",
    ")\n",
    "len(test_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "69257b71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cc08df180b24886bcb9fdbd857f1c8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "# batch embedding:   0%|          | 0/22 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_attn_scores_dict = model.interpret_attention(\n",
    "    record_dict=test_record_dict,\n",
    "    batch_size=eval_batch_size,\n",
    "    field='title',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "75b09386",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.4806428049671292"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from entity_embed.evaluation import pair_entity_ratio\n",
    "\n",
    "pair_entity_ratio(len(test_found_pair_set), len(test_record_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3ad9d6e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.18549580661075482, 0.9868766404199475)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from entity_embed.evaluation import precision_and_recall\n",
    "\n",
    "precision_and_recall(test_found_pair_set, test_pos_pair_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bb67907",
   "metadata": {},
   "source": [
    "Complement the train/valid `found_pair_set` with `pos_pair_set` for training.  \n",
    "Leave test untoched, to reproduce production behavior:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ad6cd20d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_found_pair_set |= train_pos_pair_set\n",
    "valid_found_pair_set |= valid_pos_pair_set"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e6925ac",
   "metadata": {},
   "source": [
    "## Matching: Compare"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93d29e62",
   "metadata": {},
   "source": [
    "Make a dataframe `df` with all records (train, valid, test) to add additional features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "14577317",
   "metadata": {},
   "outputs": [],
   "source": [
    "record_dict = {**train_record_dict, **valid_record_dict, **test_record_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cc03d21f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame.from_dict(record_dict, orient='index')\n",
    "df = df.drop(columns='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8e3e72de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>manufacturer</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>__source</th>\n",
       "      <th>all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1728</th>\n",
       "      <td></td>\n",
       "      <td>1149.00</td>\n",
       "      <td>filemaker pro 8.5 5-user pack</td>\n",
       "      <td>google</td>\n",
       "      <td>filemaker pro 8.5 5-user pack -  - 1149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>diskeeper corporation</td>\n",
       "      <td>736.66</td>\n",
       "      <td>diskeeper 2007 pro premier 5-lic pack</td>\n",
       "      <td>amazon</td>\n",
       "      <td>diskeeper 2007 pro premier 5-lic pack - diskee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2889</th>\n",
       "      <td></td>\n",
       "      <td>16.00</td>\n",
       "      <td>emc securid appl mnt - ent/ std 17mo per u 2k5...</td>\n",
       "      <td>google</td>\n",
       "      <td>emc securid appl mnt - ent/ std 17mo per u 2k5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               manufacturer    price  \\\n",
       "1728                         1149.00   \n",
       "255   diskeeper corporation   736.66   \n",
       "2889                           16.00   \n",
       "\n",
       "                                                  title __source  \\\n",
       "1728                      filemaker pro 8.5 5-user pack   google   \n",
       "255               diskeeper 2007 pro premier 5-lic pack   amazon   \n",
       "2889  emc securid appl mnt - ent/ std 17mo per u 2k5...   google   \n",
       "\n",
       "                                                    all  \n",
       "1728            filemaker pro 8.5 5-user pack -  - 1149  \n",
       "255   diskeeper 2007 pro premier 5-lic pack - diskee...  \n",
       "2889  emc securid appl mnt - ent/ std 17mo per u 2k5...  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['all'] = df.agg('{0[title]} - {0[manufacturer]} - {0[price]}'.format, axis=1)\n",
    "df['price'] = pd.to_numeric(df['price'].str.replace(' ', ''), errors='coerce')\n",
    "del df['cluster']\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1789693",
   "metadata": {},
   "source": [
    "Replace all `record_dict`s (train, valid, test) to add additional features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f1732a76",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_record_dict = df.loc[train_record_dict.keys()].to_dict(orient='index')\n",
    "valid_record_dict = df.loc[valid_record_dict.keys()].to_dict(orient='index')\n",
    "test_record_dict = df.loc[test_record_dict.keys()].to_dict(orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "986bef99",
   "metadata": {},
   "outputs": [],
   "source": [
    "import textdistance as td\n",
    "import math\n",
    "\n",
    "def exact_eq(x, y):\n",
    "    return float(x == y)\n",
    "\n",
    "def token_ops(func):\n",
    "    def new_func(x, y):\n",
    "        return func(x.split(), y.split())    \n",
    "    return new_func\n",
    "\n",
    "def abs_diff(x, y):\n",
    "    return abs(x - y)\n",
    "\n",
    "def abs_diff_log10(x, y):\n",
    "    diff = abs_diff(x, y)\n",
    "    if diff > 1:\n",
    "        return math.log10(diff)\n",
    "    else:\n",
    "        return 0.0\n",
    "\n",
    "SIM_FUNC_DICT = {\n",
    "    (\"all\", \"jaccard\"): token_ops(td.jaccard.normalized_similarity),\n",
    "    (\"all\", \"overlap\"): token_ops(td.overlap.normalized_similarity),\n",
    "    (\"all\", \"damerau_levenshtein\"): td.damerau_levenshtein.normalized_similarity,\n",
    "    (\"all\", \"jaro_winkler\"): td.jaro_winkler.normalized_similarity,\n",
    "    (\"manufacturer\", \"jaccard\"): token_ops(td.jaccard.normalized_similarity),\n",
    "    (\"manufacturer\", \"overlap\"): token_ops(td.overlap.normalized_similarity),\n",
    "    (\"manufacturer\", \"damerau_levenshtein\"): td.damerau_levenshtein.normalized_similarity,\n",
    "    (\"manufacturer\", \"jaro_winkler\"): td.jaro_winkler.normalized_similarity,\n",
    "    (\"title\", \"jaccard\"): token_ops(td.jaccard.normalized_similarity),\n",
    "    (\"title\", \"overlap\"): token_ops(td.overlap.normalized_similarity),\n",
    "    (\"title\", \"damerau_levenshtein\"): td.damerau_levenshtein.normalized_similarity,\n",
    "    (\"title\", \"jaro_winkler\"): td.jaro_winkler.normalized_similarity,\n",
    "    (\"price\", \"abs_diff\"): abs_diff,\n",
    "    (\"price\", \"abs_diff_log10\"): abs_diff_log10,\n",
    "}\n",
    "\n",
    "def record_sim_func(record_pair):\n",
    "    record_left, record_right = record_pair\n",
    "    feature_dict = {}\n",
    "    \n",
    "    for (field, sim_func_name), sim_func in SIM_FUNC_DICT.items():\n",
    "        x = record_left[field]\n",
    "        y = record_right[field]\n",
    "        if x and y:\n",
    "            if sim_func_name.startswith('abs_diff') and (math.isnan(x) or math.isnan(y)):\n",
    "                sim = -1.0\n",
    "            else:\n",
    "                sim = sim_func(x, y)\n",
    "        else:\n",
    "            sim = -1.0\n",
    "        feature_dict[f\"{field}_{sim_func_name}\"] = sim\n",
    "    \n",
    "    return feature_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9789cd55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.58 ms, sys: 759 µs, total: 6.34 ms\n",
      "Wall time: 6.06 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'all_jaccard': 0.41666666666666674,\n",
       " 'all_overlap': 0.625,\n",
       " 'all_damerau_levenshtein': 0.7222222222222222,\n",
       " 'all_jaro_winkler': 0.9258397932816538,\n",
       " 'manufacturer_jaccard': -1.0,\n",
       " 'manufacturer_overlap': -1.0,\n",
       " 'manufacturer_damerau_levenshtein': -1.0,\n",
       " 'manufacturer_jaro_winkler': -1.0,\n",
       " 'title_jaccard': 0.4285714285714286,\n",
       " 'title_overlap': 0.6,\n",
       " 'title_damerau_levenshtein': 0.9411764705882353,\n",
       " 'title_jaro_winkler': 0.9882352941176471,\n",
       " 'price_abs_diff': 14.709999999999994,\n",
       " 'price_abs_diff_log10': 1.16761267272753}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "pair = next(iter(test_pos_pair_set))\n",
    "id_left, id_right = pair\n",
    "feature_dict = record_sim_func((test_record_dict[id_left], test_record_dict[id_right]))\n",
    "\n",
    "# display(test_record_dict[id_left], test_record_dict[id_right])\n",
    "feature_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "29ad8dfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import multiprocessing\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "def compare_pairs(record_dict, found_pair_set):\n",
    "    all_feature_dict = defaultdict(list)\n",
    "    chunksize = 100\n",
    "    tasks = (\n",
    "        (record_dict[id_left], record_dict[id_right])\n",
    "        for (id_left, id_right)\n",
    "        in found_pair_set\n",
    "    )\n",
    "\n",
    "    with multiprocessing.Pool() as pool:\n",
    "        for feature_dict in tqdm(\n",
    "            pool.imap(record_sim_func, tasks, chunksize=chunksize),\n",
    "            total=len(found_pair_set)\n",
    "        ):\n",
    "            for feature, val in feature_dict.items():\n",
    "                all_feature_dict[feature].append(val)\n",
    "\n",
    "        pool.close()\n",
    "        pool.join()\n",
    "    \n",
    "    return pd.DataFrame(all_feature_dict, index=pd.MultiIndex.from_tuples(found_pair_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c73ff3a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "558ac21359074ecfa046710eea5e089d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5776 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 437 ms, sys: 400 ms, total: 836 ms\n",
      "Wall time: 10.4 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "5776"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train_feature_df = compare_pairs(train_record_dict, train_found_pair_set)\n",
    "assert len(train_feature_df) == len(train_found_pair_set)\n",
    "len(train_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1bf8bb26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "550dc744426a44b48337c7136477cb35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1028 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 45.6 ms, sys: 372 ms, total: 417 ms\n",
      "Wall time: 2.76 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1028"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "valid_feature_df = compare_pairs(valid_record_dict, valid_found_pair_set)\n",
    "assert len(valid_feature_df) == len(valid_found_pair_set)\n",
    "len(valid_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d7e8c66d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7205b723794f44e8937e5a77a6e09669",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2027 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 123 ms, sys: 392 ms, total: 514 ms\n",
      "Wall time: 4.63 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2027"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "test_feature_df = compare_pairs(test_record_dict, test_found_pair_set)\n",
    "assert len(test_feature_df) == len(test_found_pair_set)\n",
    "len(test_found_pair_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbe58af0",
   "metadata": {},
   "source": [
    "## Matching: Compare - TFIDF Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "871df4b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(analyzer='char', min_df=2, ngram_range=(2, 4))"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "def get_tfidf_vectorizer(train_record_dict, valid_record_dict, field='all'):\n",
    "    tfidf_vectorizer = TfidfVectorizer(\n",
    "        analyzer='char',\n",
    "        ngram_range=(2,4),\n",
    "        min_df=2\n",
    "    )\n",
    "    train_valid_record_dict = {**train_record_dict, **valid_record_dict}\n",
    "    tfidf_vectorizer.fit(record[field] for record in train_valid_record_dict.values())\n",
    "    return tfidf_vectorizer\n",
    "\n",
    "tfidf_vectorizer = get_tfidf_vectorizer(train_record_dict, valid_record_dict)\n",
    "tfidf_vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "bef71ffb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def add_tfidf_feature(tfidf_vectorizer, feature_df, record_dict, found_pair_set, field='all'):\n",
    "    tfidf_matrix = tfidf_vectorizer.transform(record[field] for record in record_dict.values())\n",
    "\n",
    "    id_to_idx = {id_: idx for idx, id_ in enumerate(record_dict.keys())}\n",
    "    left_idx = [id_to_idx[left_id] for left_id, __ in found_pair_set]\n",
    "    right_idx = [id_to_idx[right_id] for __, right_id in found_pair_set]\n",
    "    tfidf_sim = tfidf_matrix[left_idx].multiply(tfidf_matrix[right_idx]).sum(axis=1)  # cos per row\n",
    "\n",
    "    feature_df[f'{field}_tfidf'] = tfidf_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3c6f5af1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 194 ms, sys: 0 ns, total: 194 ms\n",
      "Wall time: 193 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "add_tfidf_feature(tfidf_vectorizer, train_feature_df, train_record_dict, train_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4bfc48a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 79.8 ms, sys: 0 ns, total: 79.8 ms\n",
      "Wall time: 79.5 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "add_tfidf_feature(tfidf_vectorizer, valid_feature_df, valid_record_dict, valid_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "bdb7247c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 117 ms, sys: 18 µs, total: 117 ms\n",
      "Wall time: 116 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "add_tfidf_feature(tfidf_vectorizer, test_feature_df, test_record_dict, test_found_pair_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3a02e01",
   "metadata": {},
   "source": [
    "## Matching: Compare - Embedding Cosine Feature "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "772f99fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "def add_embed_cos_features(feature_df, left_field_vector_dict, right_field_vector_dict, found_pair_set):\n",
    "    fields = next(iter(left_field_vector_dict.values())).keys()\n",
    "    \n",
    "    for field in fields:\n",
    "        left_embed = np.stack([left_field_vector_dict[left_id][field] for left_id, __ in found_pair_set])\n",
    "        right_embed = np.stack([right_field_vector_dict[right_id][field] for __, right_id in found_pair_set])\n",
    "        normalize(left_embed, copy=False)\n",
    "        normalize(right_embed, copy=False)\n",
    "        sim = np.multiply(left_embed, right_embed).sum(axis=1)  # cos per row\n",
    "        feature_df[f'embed_{field}_cos'] = sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "967964d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 77.4 ms, sys: 477 µs, total: 77.9 ms\n",
      "Wall time: 77.5 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "add_embed_cos_features(\n",
    "    train_feature_df,\n",
    "    train_left_field_vector_dict,\n",
    "    train_right_field_vector_dict,\n",
    "    train_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9549bc72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.5 ms, sys: 222 µs, total: 16.7 ms\n",
      "Wall time: 16.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "add_embed_cos_features(\n",
    "    valid_feature_df,\n",
    "    valid_left_field_vector_dict,\n",
    "    valid_right_field_vector_dict,\n",
    "    valid_found_pair_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b1cccd31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 29.7 ms, sys: 0 ns, total: 29.7 ms\n",
      "Wall time: 29.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "add_embed_cos_features(\n",
    "    test_feature_df,\n",
    "    test_left_field_vector_dict,\n",
    "    test_right_field_vector_dict,\n",
    "    test_found_pair_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dd1185e",
   "metadata": {},
   "source": [
    "## Matching: Classify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f6f60ac7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>all_jaccard</th>\n",
       "      <th>all_overlap</th>\n",
       "      <th>all_damerau_levenshtein</th>\n",
       "      <th>all_jaro_winkler</th>\n",
       "      <th>manufacturer_jaccard</th>\n",
       "      <th>manufacturer_overlap</th>\n",
       "      <th>manufacturer_damerau_levenshtein</th>\n",
       "      <th>manufacturer_jaro_winkler</th>\n",
       "      <th>title_jaccard</th>\n",
       "      <th>title_overlap</th>\n",
       "      <th>title_damerau_levenshtein</th>\n",
       "      <th>title_jaro_winkler</th>\n",
       "      <th>price_abs_diff</th>\n",
       "      <th>price_abs_diff_log10</th>\n",
       "      <th>all_tfidf</th>\n",
       "      <th>embed_title_cos</th>\n",
       "      <th>embed_title_semantic_cos</th>\n",
       "      <th>embed_manufacturer_cos</th>\n",
       "      <th>embed_price_cos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>272</th>\n",
       "      <th>3735</th>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.365854</td>\n",
       "      <td>0.671396</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.257143</td>\n",
       "      <td>0.851429</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.504547</td>\n",
       "      <td>0.994367</td>\n",
       "      <td>0.989950</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.755560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>938</th>\n",
       "      <th>2238</th>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.714776</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.346154</td>\n",
       "      <td>0.690654</td>\n",
       "      <td>3.15</td>\n",
       "      <td>0.498311</td>\n",
       "      <td>0.636458</td>\n",
       "      <td>0.985939</td>\n",
       "      <td>0.962725</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.691161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <th>4234</th>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.410714</td>\n",
       "      <td>0.657091</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.719444</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.467429</td>\n",
       "      <td>0.907812</td>\n",
       "      <td>0.976248</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          all_jaccard  all_overlap  all_damerau_levenshtein  all_jaro_winkler  \\\n",
       "272 3735     0.357143     0.714286                 0.365854          0.671396   \n",
       "938 2238     0.538462     0.700000                 0.181818          0.714776   \n",
       "352 4234     0.545455     0.857143                 0.410714          0.657091   \n",
       "\n",
       "          manufacturer_jaccard  manufacturer_overlap  \\\n",
       "272 3735                  -1.0                  -1.0   \n",
       "938 2238                  -1.0                  -1.0   \n",
       "352 4234                  -1.0                  -1.0   \n",
       "\n",
       "          manufacturer_damerau_levenshtein  manufacturer_jaro_winkler  \\\n",
       "272 3735                              -1.0                       -1.0   \n",
       "938 2238                              -1.0                       -1.0   \n",
       "352 4234                              -1.0                       -1.0   \n",
       "\n",
       "          title_jaccard  title_overlap  title_damerau_levenshtein  \\\n",
       "272 3735       0.333333       1.000000                   0.257143   \n",
       "938 2238       0.333333       0.600000                   0.346154   \n",
       "352 4234       0.400000       0.666667                   0.375000   \n",
       "\n",
       "          title_jaro_winkler  price_abs_diff  price_abs_diff_log10  all_tfidf  \\\n",
       "272 3735            0.851429            0.05              0.000000   0.504547   \n",
       "938 2238            0.690654            3.15              0.498311   0.636458   \n",
       "352 4234            0.719444            0.00              0.000000   0.467429   \n",
       "\n",
       "          embed_title_cos  embed_title_semantic_cos  embed_manufacturer_cos  \\\n",
       "272 3735         0.994367                  0.989950                     0.0   \n",
       "938 2238         0.985939                  0.962725                     0.0   \n",
       "352 4234         0.907812                  0.976248                     0.0   \n",
       "\n",
       "          embed_price_cos  \n",
       "272 3735         0.755560  \n",
       "938 2238         0.691161  \n",
       "352 4234         1.000000  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_valid_feature_df = pd.concat([train_feature_df, valid_feature_df])\n",
    "train_valid_feature_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "f4e3eda5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_true_y = np.array([pair in train_pos_pair_set for pair in train_found_pair_set], dtype='i4')\n",
    "valid_true_y = np.array([pair in valid_pos_pair_set for pair in valid_found_pair_set], dtype='i4')\n",
    "test_true_y = np.array([pair in test_pos_pair_set for pair in test_found_pair_set], dtype='i4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "471adca2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_valid_true_y = np.concatenate([train_true_y, valid_true_y])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d98c3077",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 45 candidates, totalling 45 fits\n",
      "CPU times: user 262 ms, sys: 548 ms, total: 810 ms\n",
      "Wall time: 7.1 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fjsj/workspace/entity-embed/venv/lib/python3.8/site-packages/sklearn/ensemble/_forest.py:541: UserWarning: Some inputs do not have OOB scores. This probably means too few trees were used to compute any reliable oob estimates.\n",
      "  warn(\"Some inputs do not have OOB scores. \"\n",
      "/home/fjsj/workspace/entity-embed/venv/lib/python3.8/site-packages/sklearn/ensemble/_forest.py:545: RuntimeWarning: invalid value encountered in true_divide\n",
      "  decision = (predictions[k] /\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=PredefinedSplit(test_fold=array([-1, -1, ...,  0,  0])),\n",
       "             estimator=RandomForestClassifier(oob_score=True, random_state=42),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'max_depth': [5, 10, 25, 50, None],\n",
       "                         'min_samples_leaf': [1, 3, 5],\n",
       "                         'n_estimators': [10, 100, 200]},\n",
       "             scoring='f1', verbose=10)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import PredefinedSplit, GridSearchCV\n",
    "\n",
    "cv = PredefinedSplit(\n",
    "    np.concatenate([\n",
    "        np.full(train_true_y.shape[0], -1 ,dtype='i4'),\n",
    "        np.zeros(valid_true_y.shape[0], dtype='i4')\n",
    "    ])\n",
    ")\n",
    "param_grid = {\n",
    "    'n_estimators': [10, 100, 200],\n",
    "    'max_depth': [5, 10, 25, 50, None],\n",
    "    'min_samples_leaf': [1, 3, 5],\n",
    "}\n",
    "clf = RandomForestClassifier(oob_score=True, random_state=random_seed)\n",
    "clf = GridSearchCV(clf, param_grid, scoring='f1', cv=cv, verbose=10, n_jobs=-1)\n",
    "clf.fit(train_valid_feature_df, train_valid_true_y);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "76241b3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 10, 'min_samples_leaf': 5, 'n_estimators': 10}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "ce5861cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7378190255220417"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "8f8f2c87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9275426219870664"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_estimator_.oob_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "8b20dd9c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('embed_title_cos', 0.2782738604073386),\n",
       " ('all_tfidf', 0.16454214766561395),\n",
       " ('embed_title_semantic_cos', 0.11720263869652318),\n",
       " ('all_overlap', 0.06782599085266797),\n",
       " ('all_jaccard', 0.057667329705184386),\n",
       " ('title_overlap', 0.05379988685400776),\n",
       " ('title_jaro_winkler', 0.04123526258525678),\n",
       " ('all_damerau_levenshtein', 0.039007243354376504),\n",
       " ('price_abs_diff_log10', 0.03873013011468886),\n",
       " ('all_jaro_winkler', 0.035023973210820875),\n",
       " ('embed_price_cos', 0.03385684177185366),\n",
       " ('title_damerau_levenshtein', 0.02478815120555989),\n",
       " ('price_abs_diff', 0.02276508203858747),\n",
       " ('title_jaccard', 0.01736975375286672),\n",
       " ('manufacturer_damerau_levenshtein', 0.004450853255918805),\n",
       " ('manufacturer_jaro_winkler', 0.0017193768839414644),\n",
       " ('manufacturer_jaccard', 0.0015489302867269122),\n",
       " ('embed_manufacturer_cos', 0.0001731633148695912),\n",
       " ('manufacturer_overlap', 1.9384043196644225e-05)]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_importances = dict(zip(train_valid_feature_df.columns, clf.best_estimator_.feature_importances_))\n",
    "sorted(feature_importances.items(), key=lambda kv: kv[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "dde175ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9512051734273956"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "cls_threshold = 0.3\n",
    "train_valid_pred_y = clf.predict_proba(train_valid_feature_df)\n",
    "train_valid_pred_y[train_valid_pred_y >= cls_threshold] = 1\n",
    "train_valid_pred_y[train_valid_pred_y < cls_threshold] = 0\n",
    "train_valid_pred_y = train_valid_pred_y[:, 1]\n",
    "accuracy_score(train_valid_pred_y, train_valid_true_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "e6a1a542",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.7890411]), array([0.76595745]), array([0.77732794]), array([376]))"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "\n",
    "prob_y = clf.predict_proba(test_feature_df)\n",
    "pred_y = np.copy(prob_y)\n",
    "pred_y[pred_y >= cls_threshold] = 1\n",
    "pred_y[pred_y < cls_threshold] = 0\n",
    "pred_y = pred_y[:, 1]\n",
    "precision_recall_fscore_support(test_true_y, pred_y, labels=[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e378f9fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7890410958904109, 0.7559055118110236)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_found_pair_set = OrderedSet(test_feature_df[pred_y.astype(bool)].index)\n",
    "\n",
    "precision_and_recall(cls_found_pair_set, test_pos_pair_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b137b59d",
   "metadata": {},
   "source": [
    "False negatives:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "0df0cb98",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.89283103 0.10716897]]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>manufacturer</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>__source</th>\n",
       "      <th>all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>840</th>\n",
       "      <td>sony pictures digital entertainment</td>\n",
       "      <td>49.35</td>\n",
       "      <td>digital video and audio production book for ve...</td>\n",
       "      <td>amazon</td>\n",
       "      <td>digital video and audio production book for ve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4304</th>\n",
       "      <td>sony creative software</td>\n",
       "      <td>27.99</td>\n",
       "      <td>digital video&amp; audio prod book</td>\n",
       "      <td>google</td>\n",
       "      <td>digital video&amp; audio prod book - sony creative...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             manufacturer  price  \\\n",
       "840   sony pictures digital entertainment  49.35   \n",
       "4304               sony creative software  27.99   \n",
       "\n",
       "                                                  title __source  \\\n",
       "840   digital video and audio production book for ve...   amazon   \n",
       "4304                     digital video& audio prod book   google   \n",
       "\n",
       "                                                    all  \n",
       "840   digital video and audio production book for ve...  \n",
       "4304  digital video& audio prod book - sony creative...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.8442389 0.1557611]]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>manufacturer</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>__source</th>\n",
       "      <th>all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>855</th>\n",
       "      <td>micromat</td>\n",
       "      <td>69.00</td>\n",
       "      <td>micromat podlock (mac)</td>\n",
       "      <td>amazon</td>\n",
       "      <td>micromat podlock (mac) - micromat - 69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1552</th>\n",
       "      <td></td>\n",
       "      <td>39.99</td>\n",
       "      <td>micromat podlock ipod utility software</td>\n",
       "      <td>google</td>\n",
       "      <td>micromat podlock ipod utility software -  - 39.99</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     manufacturer  price                                   title __source  \\\n",
       "855      micromat  69.00                  micromat podlock (mac)   amazon   \n",
       "1552               39.99  micromat podlock ipod utility software   google   \n",
       "\n",
       "                                                    all  \n",
       "855              micromat podlock (mac) - micromat - 69  \n",
       "1552  micromat podlock ipod utility software -  - 39.99  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.79649727 0.20350273]]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>manufacturer</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>__source</th>\n",
       "      <th>all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>netmanage</td>\n",
       "      <td>0.00</td>\n",
       "      <td>netmanage upg rumba unix-hp edition level ( 90...</td>\n",
       "      <td>amazon</td>\n",
       "      <td>netmanage upg rumba unix-hp edition level ( 90...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3080</th>\n",
       "      <td></td>\n",
       "      <td>106.48</td>\n",
       "      <td>902453-007-e rumba unix-hp edition - upgrade l...</td>\n",
       "      <td>google</td>\n",
       "      <td>902453-007-e rumba unix-hp edition - upgrade l...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     manufacturer   price                                              title  \\\n",
       "199     netmanage    0.00  netmanage upg rumba unix-hp edition level ( 90...   \n",
       "3080               106.48  902453-007-e rumba unix-hp edition - upgrade l...   \n",
       "\n",
       "     __source                                                all  \n",
       "199    amazon  netmanage upg rumba unix-hp edition level ( 90...  \n",
       "3080   google  902453-007-e rumba unix-hp edition - upgrade l...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.93718231 0.06281769]]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>manufacturer</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>__source</th>\n",
       "      <th>all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1340</th>\n",
       "      <td>makemusic!</td>\n",
       "      <td>375.00</td>\n",
       "      <td>make music allegro 2007 labpack - 5 user</td>\n",
       "      <td>amazon</td>\n",
       "      <td>make music allegro 2007 labpack - 5 user - mak...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011</th>\n",
       "      <td></td>\n",
       "      <td>272.94</td>\n",
       "      <td>hal leonard allegro 2007 5-user labpack</td>\n",
       "      <td>google</td>\n",
       "      <td>hal leonard allegro 2007 5-user labpack -  - 2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     manufacturer   price                                     title __source  \\\n",
       "1340   makemusic!  375.00  make music allegro 2007 labpack - 5 user   amazon   \n",
       "2011               272.94   hal leonard allegro 2007 5-user labpack   google   \n",
       "\n",
       "                                                    all  \n",
       "1340  make music allegro 2007 labpack - 5 user - mak...  \n",
       "2011  hal leonard allegro 2007 5-user labpack -  - 2...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[9.99122389e-01 8.77610840e-04]]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>manufacturer</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>__source</th>\n",
       "      <th>all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>microsoft licenses</td>\n",
       "      <td>205.73</td>\n",
       "      <td>microsoft licenses word olp c (05903871)</td>\n",
       "      <td>amazon</td>\n",
       "      <td>microsoft licenses word olp c (05903871) - mic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3727</th>\n",
       "      <td></td>\n",
       "      <td>111.78</td>\n",
       "      <td>microsoft 059-03871 molpc word sa</td>\n",
       "      <td>google</td>\n",
       "      <td>microsoft 059-03871 molpc word sa -  - 111.78</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            manufacturer   price                                     title  \\\n",
       "423   microsoft licenses  205.73  microsoft licenses word olp c (05903871)   \n",
       "3727                      111.78         microsoft 059-03871 molpc word sa   \n",
       "\n",
       "     __source                                                all  \n",
       "423    amazon  microsoft licenses word olp c (05903871) - mic...  \n",
       "3727   google      microsoft 059-03871 molpc word sa -  - 111.78  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for x, y in list(test_pos_pair_set - cls_found_pair_set)[:5]:\n",
    "    print(clf.predict_proba(test_feature_df.loc[[(x, y)]]))\n",
    "    display(df.loc[[x, y]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "edcef2d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>all_jaccard</th>\n",
       "      <th>all_overlap</th>\n",
       "      <th>all_damerau_levenshtein</th>\n",
       "      <th>all_jaro_winkler</th>\n",
       "      <th>manufacturer_jaccard</th>\n",
       "      <th>manufacturer_overlap</th>\n",
       "      <th>manufacturer_damerau_levenshtein</th>\n",
       "      <th>manufacturer_jaro_winkler</th>\n",
       "      <th>title_jaccard</th>\n",
       "      <th>title_overlap</th>\n",
       "      <th>title_damerau_levenshtein</th>\n",
       "      <th>title_jaro_winkler</th>\n",
       "      <th>price_abs_diff</th>\n",
       "      <th>price_abs_diff_log10</th>\n",
       "      <th>all_tfidf</th>\n",
       "      <th>embed_title_cos</th>\n",
       "      <th>embed_title_semantic_cos</th>\n",
       "      <th>embed_manufacturer_cos</th>\n",
       "      <th>embed_price_cos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>725</th>\n",
       "      <th>1803</th>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.874259</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.94697</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.778151</td>\n",
       "      <td>0.601628</td>\n",
       "      <td>0.602881</td>\n",
       "      <td>0.587976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.893622</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          all_jaccard  all_overlap  all_damerau_levenshtein  all_jaro_winkler  \\\n",
       "725 1803     0.444444     0.666667                    0.625          0.874259   \n",
       "\n",
       "          manufacturer_jaccard  manufacturer_overlap  \\\n",
       "725 1803                  -1.0                  -1.0   \n",
       "\n",
       "          manufacturer_damerau_levenshtein  manufacturer_jaro_winkler  \\\n",
       "725 1803                              -1.0                       -1.0   \n",
       "\n",
       "          title_jaccard  title_overlap  title_damerau_levenshtein  \\\n",
       "725 1803           0.25            0.5                   0.916667   \n",
       "\n",
       "          title_jaro_winkler  price_abs_diff  price_abs_diff_log10  all_tfidf  \\\n",
       "725 1803             0.94697             6.0              0.778151   0.601628   \n",
       "\n",
       "          embed_title_cos  embed_title_semantic_cos  embed_manufacturer_cos  \\\n",
       "725 1803         0.602881                  0.587976                     0.0   \n",
       "\n",
       "          embed_price_cos  \n",
       "725 1803         0.893622  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[0.8047619, 0.1952381]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pair = (725, 1803)\n",
    "display(test_feature_df.loc[[pair]])\n",
    "clf.predict_proba(test_feature_df.loc[[pair]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34dde2b4",
   "metadata": {},
   "source": [
    "Hard cases attention:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "39fa4c3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>prob</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <th>3715</th>\n",
       "      <td>0.304131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <th>3193</th>\n",
       "      <td>0.303955</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               prob\n",
       "1042 3715  0.304131\n",
       "143  3193  0.303955"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob_df = pd.DataFrame(prob_y[:,1], columns=['prob'], index=test_feature_df.index)\n",
    "hard_prob_df = prob_df.loc[(prob_df['prob'] >= cls_threshold) & (prob_df['prob'] <= cls_threshold + 0.01)]\n",
    "hard_prob_df = hard_prob_df.loc[hard_prob_df.index.intersection(test_pos_pair_set)]\n",
    "hard_prob_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "207f5d89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_e5699_row0_col0{\n",
       "            background-color:  #f3f0f0;\n",
       "            color:  #000000;\n",
       "        }#T_e5699_row0_col1{\n",
       "            background-color:  #f4cfcf;\n",
       "            color:  #000000;\n",
       "        }#T_e5699_row0_col2{\n",
       "            background-color:  #f7a6a6;\n",
       "            color:  #000000;\n",
       "        }#T_e5699_row0_col3{\n",
       "            background-color:  #fc4343;\n",
       "            color:  #000000;\n",
       "        }#T_e5699_row0_col4{\n",
       "            background-color:  #fe0a0a;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_e5699_row0_col5{\n",
       "            background-color:  #f6aeae;\n",
       "            color:  #000000;\n",
       "        }#T_e5699_row0_col6{\n",
       "            background-color:  #fb4a4a;\n",
       "            color:  #000000;\n",
       "        }#T_e5699_row0_col7{\n",
       "            background-color:  #ff0000;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_e5699_row0_col8{\n",
       "            background-color:  #fb5555;\n",
       "            color:  #000000;\n",
       "        }#T_e5699_row0_col9{\n",
       "            background-color:  #f5c8c8;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_e5699_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >genuine</th>        <th class=\"col_heading level0 col1\" >fractals</th>        <th class=\"col_heading level0 col2\" >print</th>        <th class=\"col_heading level0 col3\" >pro</th>        <th class=\"col_heading level0 col4\" >4</th>        <th class=\"col_heading level0 col5\" >.</th>        <th class=\"col_heading level0 col6\" >1</th>        <th class=\"col_heading level0 col7\" >sharpener</th>        <th class=\"col_heading level0 col8\" >2</th>        <th class=\"col_heading level0 col9\" >0</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_e5699_level0_row0\" class=\"row_heading level0 row0\" >1042</th>\n",
       "                        <td id=\"T_e5699_row0_col0\" class=\"data row0 col0\" >0.027111</td>\n",
       "                        <td id=\"T_e5699_row0_col1\" class=\"data row0 col1\" >0.042644</td>\n",
       "                        <td id=\"T_e5699_row0_col2\" class=\"data row0 col2\" >0.062215</td>\n",
       "                        <td id=\"T_e5699_row0_col3\" class=\"data row0 col3\" >0.108261</td>\n",
       "                        <td id=\"T_e5699_row0_col4\" class=\"data row0 col4\" >0.134674</td>\n",
       "                        <td id=\"T_e5699_row0_col5\" class=\"data row0 col5\" >0.058149</td>\n",
       "                        <td id=\"T_e5699_row0_col6\" class=\"data row0 col6\" >0.104626</td>\n",
       "                        <td id=\"T_e5699_row0_col7\" class=\"data row0 col7\" >0.139683</td>\n",
       "                        <td id=\"T_e5699_row0_col8\" class=\"data row0 col8\" >0.100042</td>\n",
       "                        <td id=\"T_e5699_row0_col9\" class=\"data row0 col9\" >0.045997</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7efeb6be8490>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_55dbd_row0_col0{\n",
       "            background-color:  #f4d4d4;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col1{\n",
       "            background-color:  #f3f0f0;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col2{\n",
       "            background-color:  #f4d6d6;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col3{\n",
       "            background-color:  #f6afaf;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col4{\n",
       "            background-color:  #f88888;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col5{\n",
       "            background-color:  #f4d8d7;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col6{\n",
       "            background-color:  #fa6c6c;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col7{\n",
       "            background-color:  #f6acac;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col8{\n",
       "            background-color:  #f88a8a;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col9{\n",
       "            background-color:  #fa6a6a;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col10{\n",
       "            background-color:  #f97272;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col11{\n",
       "            background-color:  #fd3131;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55dbd_row0_col12{\n",
       "            background-color:  #ff0000;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55dbd_row0_col13{\n",
       "            background-color:  #fa5e5e;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col14{\n",
       "            background-color:  #f98080;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col15{\n",
       "            background-color:  #f88787;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col16{\n",
       "            background-color:  #fb4f4f;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col17{\n",
       "            background-color:  #f97878;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col18{\n",
       "            background-color:  #f5c2c2;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col19{\n",
       "            background-color:  #f4dbdb;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col20{\n",
       "            background-color:  #f6bbbb;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col21{\n",
       "            background-color:  #f4dcdc;\n",
       "            color:  #000000;\n",
       "        }#T_55dbd_row0_col22{\n",
       "            background-color:  #f4e0e0;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_55dbd_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >onone</th>        <th class=\"col_heading level0 col1\" >software</th>        <th class=\"col_heading level0 col2\" >inc</th>        <th class=\"col_heading level0 col3\" >.</th>        <th class=\"col_heading level0 col4\" >gfn</th>        <th class=\"col_heading level0 col5\" >-</th>        <th class=\"col_heading level0 col6\" >42001</th>        <th class=\"col_heading level0 col7\" >genuine</th>        <th class=\"col_heading level0 col8\" >fractals</th>        <th class=\"col_heading level0 col9\" >print</th>        <th class=\"col_heading level0 col10\" >pro</th>        <th class=\"col_heading level0 col11\" >v</th>        <th class=\"col_heading level0 col12\" >4</th>        <th class=\"col_heading level0 col13\" >1</th>        <th class=\"col_heading level0 col14\" >with</th>        <th class=\"col_heading level0 col15\" >nik</th>        <th class=\"col_heading level0 col16\" >sharpener</th>        <th class=\"col_heading level0 col17\" >2</th>        <th class=\"col_heading level0 col18\" >0</th>        <th class=\"col_heading level0 col19\" >license</th>        <th class=\"col_heading level0 col20\" >standard</th>        <th class=\"col_heading level0 col21\" >pc</th>        <th class=\"col_heading level0 col22\" >mac</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_55dbd_level0_row0\" class=\"row_heading level0 row0\" >3715</th>\n",
       "                        <td id=\"T_55dbd_row0_col0\" class=\"data row0 col0\" >0.011743</td>\n",
       "                        <td id=\"T_55dbd_row0_col1\" class=\"data row0 col1\" >0.003193</td>\n",
       "                        <td id=\"T_55dbd_row0_col2\" class=\"data row0 col2\" >0.011038</td>\n",
       "                        <td id=\"T_55dbd_row0_col3\" class=\"data row0 col3\" >0.022462</td>\n",
       "                        <td id=\"T_55dbd_row0_col4\" class=\"data row0 col4\" >0.033694</td>\n",
       "                        <td id=\"T_55dbd_row0_col5\" class=\"data row0 col5\" >0.010436</td>\n",
       "                        <td id=\"T_55dbd_row0_col6\" class=\"data row0 col6\" >0.041960</td>\n",
       "                        <td id=\"T_55dbd_row0_col7\" class=\"data row0 col7\" >0.023367</td>\n",
       "                        <td id=\"T_55dbd_row0_col8\" class=\"data row0 col8\" >0.033081</td>\n",
       "                        <td id=\"T_55dbd_row0_col9\" class=\"data row0 col9\" >0.042529</td>\n",
       "                        <td id=\"T_55dbd_row0_col10\" class=\"data row0 col10\" >0.040480</td>\n",
       "                        <td id=\"T_55dbd_row0_col11\" class=\"data row0 col11\" >0.059632</td>\n",
       "                        <td id=\"T_55dbd_row0_col12\" class=\"data row0 col12\" >0.074030</td>\n",
       "                        <td id=\"T_55dbd_row0_col13\" class=\"data row0 col13\" >0.046308</td>\n",
       "                        <td id=\"T_55dbd_row0_col14\" class=\"data row0 col14\" >0.036183</td>\n",
       "                        <td id=\"T_55dbd_row0_col15\" class=\"data row0 col15\" >0.034366</td>\n",
       "                        <td id=\"T_55dbd_row0_col16\" class=\"data row0 col16\" >0.050648</td>\n",
       "                        <td id=\"T_55dbd_row0_col17\" class=\"data row0 col17\" >0.038531</td>\n",
       "                        <td id=\"T_55dbd_row0_col18\" class=\"data row0 col18\" >0.017022</td>\n",
       "                        <td id=\"T_55dbd_row0_col19\" class=\"data row0 col19\" >0.009353</td>\n",
       "                        <td id=\"T_55dbd_row0_col20\" class=\"data row0 col20\" >0.018803</td>\n",
       "                        <td id=\"T_55dbd_row0_col21\" class=\"data row0 col21\" >0.009058</td>\n",
       "                        <td id=\"T_55dbd_row0_col22\" class=\"data row0 col22\" >0.008021</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7efeb20c98b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_e4301_row0_col0,#T_e4301_row0_col3{\n",
       "            background-color:  #fa6262;\n",
       "            color:  #000000;\n",
       "        }#T_e4301_row0_col1{\n",
       "            background-color:  #ff0404;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_e4301_row0_col2{\n",
       "            background-color:  #ff0000;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_e4301_row0_col4{\n",
       "            background-color:  #fc3f3f;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_e4301_row0_col5{\n",
       "            background-color:  #fb5252;\n",
       "            color:  #000000;\n",
       "        }#T_e4301_row0_col6{\n",
       "            background-color:  #f97070;\n",
       "            color:  #000000;\n",
       "        }#T_e4301_row0_col7{\n",
       "            background-color:  #f6aeae;\n",
       "            color:  #000000;\n",
       "        }#T_e4301_row0_col8{\n",
       "            background-color:  #f4dada;\n",
       "            color:  #000000;\n",
       "        }#T_e4301_row0_col9{\n",
       "            background-color:  #f4cfcf;\n",
       "            color:  #000000;\n",
       "        }#T_e4301_row0_col10{\n",
       "            background-color:  #f3f0f0;\n",
       "            color:  #000000;\n",
       "        }#T_e4301_row0_col11{\n",
       "            background-color:  #f3ecec;\n",
       "            color:  #000000;\n",
       "        }#T_e4301_row0_col12{\n",
       "            background-color:  #f4d9d9;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_e4301_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >adobe</th>        <th class=\"col_heading level0 col1\" >acrobat</th>        <th class=\"col_heading level0 col2\" >distiller</th>        <th class=\"col_heading level0 col3\" >svr</th>        <th class=\"col_heading level0 col4\" >v6</th>        <th class=\"col_heading level0 col5\" >-</th>        <th class=\"col_heading level0 col6\" >cd</th>        <th class=\"col_heading level0 col7\" >linux</th>        <th class=\"col_heading level0 col8\" >u</th>        <th class=\"col_heading level0 col9\" >/</th>        <th class=\"col_heading level0 col10\" >(</th>        <th class=\"col_heading level0 col11\" >42050142</th>        <th class=\"col_heading level0 col12\" >)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_e4301_level0_row0\" class=\"row_heading level0 row0\" >143</th>\n",
       "                        <td id=\"T_e4301_row0_col0\" class=\"data row0 col0\" >0.094359</td>\n",
       "                        <td id=\"T_e4301_row0_col1\" class=\"data row0 col1\" >0.146489</td>\n",
       "                        <td id=\"T_e4301_row0_col2\" class=\"data row0 col2\" >0.148935</td>\n",
       "                        <td id=\"T_e4301_row0_col3\" class=\"data row0 col3\" >0.094240</td>\n",
       "                        <td id=\"T_e4301_row0_col4\" class=\"data row0 col4\" >0.113421</td>\n",
       "                        <td id=\"T_e4301_row0_col5\" class=\"data row0 col5\" >0.103050</td>\n",
       "                        <td id=\"T_e4301_row0_col6\" class=\"data row0 col6\" >0.086589</td>\n",
       "                        <td id=\"T_e4301_row0_col7\" class=\"data row0 col7\" >0.051958</td>\n",
       "                        <td id=\"T_e4301_row0_col8\" class=\"data row0 col8\" >0.027447</td>\n",
       "                        <td id=\"T_e4301_row0_col9\" class=\"data row0 col9\" >0.033503</td>\n",
       "                        <td id=\"T_e4301_row0_col10\" class=\"data row0 col10\" >0.015153</td>\n",
       "                        <td id=\"T_e4301_row0_col11\" class=\"data row0 col11\" >0.017486</td>\n",
       "                        <td id=\"T_e4301_row0_col12\" class=\"data row0 col12\" >0.027704</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7efee094d1c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_de0b5_row0_col0{\n",
       "            background-color:  #f6adad;\n",
       "            color:  #000000;\n",
       "        }#T_de0b5_row0_col1{\n",
       "            background-color:  #f79b9b;\n",
       "            color:  #000000;\n",
       "        }#T_de0b5_row0_col2{\n",
       "            background-color:  #fe1414;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_de0b5_row0_col3{\n",
       "            background-color:  #ff0101;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_de0b5_row0_col4{\n",
       "            background-color:  #ff0000;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_de0b5_row0_col5{\n",
       "            background-color:  #f3f0f0;\n",
       "            color:  #000000;\n",
       "        }#T_de0b5_row0_col6{\n",
       "            background-color:  #f88585;\n",
       "            color:  #000000;\n",
       "        }#T_de0b5_row0_col7{\n",
       "            background-color:  #f6b3b3;\n",
       "            color:  #000000;\n",
       "        }#T_de0b5_row0_col8{\n",
       "            background-color:  #f4d8d7;\n",
       "            color:  #000000;\n",
       "        }#T_de0b5_row0_col9{\n",
       "            background-color:  #f3e5e5;\n",
       "            color:  #000000;\n",
       "        }#T_de0b5_row0_col10{\n",
       "            background-color:  #f3e4e4;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_de0b5_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >adobe</th>        <th class=\"col_heading level0 col1\" >42050142</th>        <th class=\"col_heading level0 col2\" >acrobat</th>        <th class=\"col_heading level0 col3\" >distiller</th>        <th class=\"col_heading level0 col4\" >6</th>        <th class=\"col_heading level0 col5\" >lnx</th>        <th class=\"col_heading level0 col6\" >ret</th>        <th class=\"col_heading level0 col7\" >cd</th>        <th class=\"col_heading level0 col8\" >u</th>        <th class=\"col_heading level0 col9\" >/</th>        <th class=\"col_heading level0 col10\" >l</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_de0b5_level0_row0\" class=\"row_heading level0 row0\" >3193</th>\n",
       "                        <td id=\"T_de0b5_row0_col0\" class=\"data row0 col0\" >0.063330</td>\n",
       "                        <td id=\"T_de0b5_row0_col1\" class=\"data row0 col1\" >0.074675</td>\n",
       "                        <td id=\"T_de0b5_row0_col2\" class=\"data row0 col2\" >0.160524</td>\n",
       "                        <td id=\"T_de0b5_row0_col3\" class=\"data row0 col3\" >0.172431</td>\n",
       "                        <td id=\"T_de0b5_row0_col4\" class=\"data row0 col4\" >0.173607</td>\n",
       "                        <td id=\"T_de0b5_row0_col5\" class=\"data row0 col5\" >0.020399</td>\n",
       "                        <td id=\"T_de0b5_row0_col6\" class=\"data row0 col6\" >0.089129</td>\n",
       "                        <td id=\"T_de0b5_row0_col7\" class=\"data row0 col7\" >0.059495</td>\n",
       "                        <td id=\"T_de0b5_row0_col8\" class=\"data row0 col8\" >0.036238</td>\n",
       "                        <td id=\"T_de0b5_row0_col9\" class=\"data row0 col9\" >0.027625</td>\n",
       "                        <td id=\"T_de0b5_row0_col10\" class=\"data row0 col10\" >0.028520</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7efeb6d55040>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "from entity_embed import default_tokenizer\n",
    "\n",
    "def display_attention(id_, field):\n",
    "    tokens = default_tokenizer(test_record_dict[id_][field])\n",
    "    attn_scores = test_attn_scores_dict[id_][:len(tokens)]\n",
    "    attn_df = pd.DataFrame(dict(zip(tokens, attn_scores)), index=[id_])\n",
    "    cm = sns.light_palette(\"red\", as_cmap=True)\n",
    "    display(attn_df.style.background_gradient(cmap=cm, axis=1))\n",
    "\n",
    "def display_pair_attention(pair, field):\n",
    "    left_id, right_id = pair\n",
    "    display_attention(left_id, field)\n",
    "    display_attention(right_id, field)\n",
    "\n",
    "for pair in hard_prob_df.index:\n",
    "    display_pair_attention(pair, 'title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "01227cba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1152x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1UElEQVR4nO3deXhU9fX48fchJCGBhLDvkLBo2AJCQFFBFqkIVm0RAbUqteAK/WmtVVsVsba02sUVxYpLvwqoWAFFERFQRGQNSAIIxAgBZN9JIAnn98e9GYeQZAaSmUky5/U8eTL33s/ce24IOXM/q6gqxhhjwle1UAdgjDEmtCwRGGNMmLNEYIwxYc4SgTHGhDlLBMYYE+aqhzqAs1W/fn1NTEwMdRjGGFOprFy5cq+qNijuWKVLBImJiaxYsSLUYRhjTKUiIj+UdMyqhowxJsxZIjDGmDBnicAYY8KcJQJjjAlzlgiMMSbMBSwRiMgUEdktIutKOC4i8qyIbBaRtSLSLVCxGGOMKVkgnwheBwaVcvxKoJ37NQaYFMBYjDHGlCBg4whU9QsRSSylyDXAm+rMg71URBJEpImq7gxEPMuz9vPld3sCceoyiYmqzqhLEqkRGRHqUIwxYSqUA8qaAdu8trPdfWckAhEZg/PUQMuWLc/pYqt+OMBzCzaf03sDpXApiM7NanNpu/qhDcYYE7YqxchiVZ0MTAZITU09p5V0br+sDbdf1qZc4yqrlT8cYOikJRTY4kDGmBAKZa+h7UALr+3m7j5jjDFBFMpEMAu42e09dBFwKFDtA8YYY0oWsKohEZkK9AXqi0g28BgQCaCqLwFzgMHAZuA4MCpQsVRcTpXQ03M38tHaHRw4nseGHw8TXT2C9k3ieW7kBSGOzxgTDgLZa2ikj+MK3B2o61cGB47lAfDt9kN8u/0QifViiRBhx8EcfjyUG+LojDHholI0FldVl3doxMODkxnQvhHNEmI8XUif+DCD6cu3+Xi3McaUD0sEITamT8XqyWSMCT8211AFtGn3UY6eyOeWKcv4esu+UIdjjKniLBFUQFl7jwGw6Ls9zE3/McTRGGOqOqsaqoAW/b4vqtB1wqehDsUYEwYsEVRAIoJIqKMwxoQLqxoyxpgwZ4nAGGPCnFUNmaDLPnCcddsPsXrrQfYcOcGm3Uc5pUrOyQKeuLYTl7Q9fSbW3LwCNu8+SsbOw/x4KJesfccoOKX0bteA67o3D9FdGFN1WCKo4I6eyGf2mh20qBtL1xYJpZZVVb7fe4wjufl0blabatVC19CQm1dAxs7DfLVpLxt+PEL6jkNk7TtebNm46OpEVa/GvmMnufE/39Dv/AZs3X+c6tWqsXHXkRKvkbnnmCUCY8qBJYIKLDKiGu+tzOa9ldkAZE0cckaZFVn7WbX1ANOWbSPT7XYK8Oave9LnvAbnfO2dh3L4cM1OlmbuI65GdW69JKnYRKSqbNlzjCVb9jJ//W42/HiYXYdPlHjeVvVi6ZFYl45N4+ncrDYdm9YmJiqC3LwCkh/5BIAFG50FhBrERTMguSGt6tWkce1oOjWrTVL9mtSvFc3t/13JniPOdY7k5pG55xg7D+WSk5dP/+RG1I6JPOd7NybcWCKowJ6/oRubdh/hqbkbOZKb79m/63AukxZu4Z0V2zh+suC09zSIi2bPkRMcO5Ff9HQlyis4xdLMfby/ajvLs/aTfSDnjDJ1akbRpXlt0ncc5otNe5ibvos12w4We77YqAg6NIknuUkcl7atT4/EutStGYWU0hWqRmQET12XQoO4aHok1qVmdOm/mp9v2A1A4oMfnXHs+tTm9D2/IRk7DnMiv4C8AuW3A9pRp2ZUqec0JlyJVrJFUVJTU3XFihWhDiOo/vnpRp79fDPjBrTj2fmbTjuWEBvJU9d1oVvLBOrViiZjx2EGP/slAPf0a8v9V5x/xvkKTilfbtrDtGXbWLx5L0eLJI3o6tW4oGUCV3ZqwpWdG9Pzyfklxla/VjR9z2/ABS0T6J/ckCa1Y8rhjn3r8/cFbN1/nJZ1Y2ndoCaDOjYmITaSO/5vVbHlYyIjuLpLU47nFXDrxYl0b1UnKHEaU1GIyEpVTS3umD0RVAIfr3NGFxcmgejq1fjDoGRG9GxBbNTp/4Q5eT89ITy/YLMnEXy/9xhvLMniw7U72Xv09KqbejWj6N2uPjdc2IoLWiYQGVF8Z7KYyAiu6NiIfskN6Xt+w5BWv3zxQL9i9/dMqkvd2CgGpzShS/Pa7Dt2kl++uIScvAKmr3Am8mtQK9oSgTFeLBFUAv2SG7Jp91HGDWjHmD6tqVVKtUn3VnV4bVQPFm3cw+tLshj12jJPnXuhqIhqDOzQiN/0TiKleQIRPhqVi2ubqKjeub3Xadut6tXkxRu7cUHLBBrG1aDr4zZa25iiLBFUAg8Pbs/Dg9v7Xb7f+Q0Z9dpy4KeG18R6sYzt344hKU08012Hi8Gdm5yxb+/RE8RERvhsizAmHAT0f4GIDAKeASKA/6jqxCLHWwFTgAbAfuAmVc0OZEzh4h/DuvC7d9fw9m8upFebeqU21IaTIyfymfLV90z56ntqRVdn+u0XsSLrAEsz9xFdvRp9z2/ItRc0C3WYxgRVwBqLRSQC+A4YCGQDy4GRqprhVeZd4ENVfUNE+gOjVPVXpZ03HBuLTfkprpdRUb+8oBl39WtL24a1yCs4xaGcPOrXig5CdMYETmmNxYFMBL2A8ap6hbv9EICq/tWrTDowSFW3ifOR9ZCqxpd2XksEpjzM+XYnd721ilGXJNKhSTyXtK3PxRM/L7H89anNSaxfE4DebRvQuXltAE6dUrbsOcqJ/FN0aBIf0kF8xpQmVIngOpw/8r9xt38FXKiq93iVeRv4RlWfEZFfAjOA+qq6r8i5xgBjAFq2bNn9hx9+CEjMJrypKnkFynl/+hiA6tWE1MQ6LM3cf0bZ2jGRHMrJO21fk9o1SG4cxw/7jtO4dg0evDKZpgkx7DiYQ7uGccREhVfbjKlYKnIiaAo8DyQBXwBDgU6qerCk89oTgQm2Ua8to5oIv+jWjHveXn3asS7Na9O8TiwffbvT53km/rIz117QjBqRERScUnYczGFp5j5y8wrokVSX5MbOw3BuXgHpOw5TcErp2iKBqOpOd15VRRV76jDnpMJWDRUpXwvYoKqlTh5jicCE0pHcPE7mn6JekTaDSQu3cH7jWvRMqkdsZAStH54DwOXtG/HZ+l1+nbuawKli/js2rV2DHYdyPdvPjryAq7s0PfebMGEpVImgOk5j8QBgO05j8Q2qmu5Vpj6wX1VPiciTQIGqPlraeS0RmMpm1dYDfJaxixcXbgGcAYHJTeJJqudMJDh+tqf/BHViI+nUrDbtm8Qz+YtMz/5a0dVPGwF+VUoTRvRoyUWt66JQ4iBAYwqFJBG4Fx4M/Bun++gUVX1SRCYAK1R1llt99FdAcaqG7lbVkmcswxKBCR/bD+ZQOybytAGEJfV6GtK5CYdz82gcX4NxA9rRom5ssMI0lUTIEkEgWCIw4UxV2bY/h1tfW3babLNFLXt4AA3jawQxMlPRWSIwpooqOKW8sGAzV3RsTFL9mp4eTw3jnMkAr72gGamt6noanE34skRgTJiYvnwrf5jx7Rn7kxvHeWaUTaxXkxqR1eyJIczY7KPGhInhPVpSOyaSi1rX41evLuPb7YcA2PDjETb8eISpy7Z5yj75i04M7dac4ycLqGtrNYQ1v54IRKQhcAnQFMgB1uE0+J4KbHhnsicCY87OzLTt9GpTj9+9s4YvN+0ttsyVnRrzwg3dbIxCFXbOVUMi0g94EKgLrAZ2AzWA84A2wHvAP1T1cHkHXRJLBMaUjary9KcbeWHBFto2rMXm3UcBaNewFvPuuyzE0ZlAKUsieAp4TlW3FnOsOnAVEKGqM8orWF8sERhTvhZs2M2o151pyxPrxbJ1/3HOaxTHf2+7kAZxNtleVXHObQSq+vtSjuUDH5QtNGNMqPVLbkjHpvGk7zhM1r7jgNOm8MO+Yxw8fpLjJwvo2DSe6jZorco6515DIjJKVV8r53h8sicCY8pfwSklJ6+AmlERTFu+jYfeP7PnUUxkBAM7NOLZkReQV3CKrfuPU79mNLVjQ7dkqfFfoHoNPQ4EPREYY8pfRDXxjGDukVgXcKqJerWpz9RlTs1wTl4Bs9bsYNaaHZ73NY6vwdKHBwQ/YFOufLURrC3pEHCeqga9AtGeCIwJjYH/XMSm3UeJq1GdxHo1PV1Th6e2oEFcNLddmkQd64ZaYZWlsXgXcAVwoOghYImqBn0KREsExlQMrR/66IzZUlvXr0n+KaVtw1pcn9qCn3VoZF1SK4iyVA19CNRS1bRiTrqw7KEZYyqrRb/vx4drd9KxaTw3T1kG4Jn/aOv+43y+YTcAc8b1pkPTUhceNCFmU0wYY8rs0/Qf6doygQa1ovkgbTtpWw/yxtc/rSS4dvzPiK9hjcqhZHMNGWOC7tiJfDo+NheAi1rXZeroi3CWJjehUFoisI7BxpiAqBldnVn3XALA0sz9vLr4+xBHZEpiicAYEzApzRO4/2fnAfDnj9azzu1pZCqWgCYCERkkIhtFZLOIPFjM8ZYiskBEVovIWndFM2NMFXJP/3ae11c9t5jfv7uG/3yZyTvLt5GbVxDCyEwhvweUichkVR1T0nYx5SOAF4CBQDawXERmqWqGV7E/Ae+o6iQR6QDMARLP8h6MMRXc+gmDaP/oJwC8uzLbs/9EwSkGJDekYVy0TWERQmfzk3/Zx3ZRPYHNqpqpqieBacA1RcooUNivrDawA2NMlRMTFUHWxCG8NqoHF7Wuy9BuzQF45IN1XDzxc4ZPXkpl67hSlfidCFR1ZWnbxWgGbPPaznb3eRsP3CQi2ThPA2OLO5GIjBGRFSKyYs+ePf6GbIypYPqd35BpY3rx8OBkAEb2bAnAyh8OMGnRllCGFtZKrRoSkdk4n9qLpapXl/H6I4HXVfUfItIL+K+IdCq64I2qTgYmg9N9tIzXNMaEWL1a0WRNHAJA+yZxPDoznX1HT4Y4qvDlq43g6TKcezvQwmu7ubvP223AIABV/VpEagD1cRbAMcaEgZt7JfLozHReXfw9s9bsYEzv1vRMqkuXFgmhDi1s+FqPYFHhaxGJAVqq6kY/z70caCciSTgJYARwQ5EyW4EBwOsi0h5n9TOr+zEmTO05coIn56wHYEyf1jw8uH2IIwoPfvUaEpGf4zwdRAFJItIVmFBa1ZCq5ovIPcBcIAKYoqrpIjIBZ73jWcDvgFdE5F6cKqhb1VqMjAk7G54YBMCEDzPYffgEn63fxeQvMkmIjeT2Pm1YvfUAS7bsI3PPUTo0jWdMnzYhjrhq8Xfx+pVAf2Chql7g7vtWVTsHOL4z2BQTxlR9t/93BXPTd5V4fPUjA23K67NUHlNM5Klq0SGB9sndGBMQL/8qlW4tEwBngZzRvZOYdGM37hvojFLef9walsuTvwPK0kXkBiBCRNoB44AlgQvLGBPu3r/rkjP2vfWfbwAYOXkpy/54ebBDqrL8fSIYC3QETgBTgcPA/wtQTMYYU6ynhqUAsPvICTbtOhLiaKoOv54IVPU48EcR+ZuzqfYvYIwJuia1YzyvB/7rCwC+eXgAjeJrhCqkKsGvJwIR6SEi3wJrgW9FZI2IdA9saMYYc6ZvHh5Aq3qxnu3VWw+GLpgqwt9eQ2uBu1X1S3f7UuBFVU0JcHxnsF5DxhiAddsPcdVziwE4v1Ec1/dowW2XJoU4qoqrLGsWFyooTAIAqrpYRPLLJTpjjCmjjbuO8MSHGcxM207XFgm0axTHry5qFeqwKg1fcw11c18uEpGXcRqKFRgOLAxsaMYYU7JOzWqz8k+XU69WNIkPfgTA2uxDrM12erpfmFSX8xrFhTLESqPUqiERWVDKe1VV+5d/SKWzqiFjTHGmL99KTFR1/jpnPTsP5dIjsQ7v3nFxqMOqMGzxemNM2Dh0PI8uEz4F4IKWCbx6Sw/q2ijkcmkjQESG4Iwl8PTTUtUJZQ/PGGPKT+3YSJrXiSH7QA6rtx7kvZXbuLlXIjUiI0IdWoXlb/fRl3DaBcYCAgwDrCXGGFMhLf5Dfz4adykAf5mzgW5PzKPgVOWq/Qgmf0cWX6yqNwMHVPVxoBdwXuDCMsaYskluHO95ffxkAUdy80IYTcXmbyLIcb8fF5GmQB7QJDAhGWNM2UVUE7ImDqFdw1oA3PLa8hBHVHH5mwg+FJEE4ClgFZCF05XUGGMqtGdHXgDAkRx7IiiJv3MNPeG+nCEiHwI1ipmW2hhjKpz2TeIZktKEDTsPhzqUCsvXgLJflnIMVX3fx/sHAc/grFD2H1WdWOT4v4B+7mYs0FBVE/yI2xhj/Pbld3s4nJtP1wmfMrxHC8b1b0fNaL87TVZ5vn4SPy/lmAIlJgIRiQBeAAYC2cByEZmlqhmeE6je61V+LHCBP0EbY8zZyCtwegwdPJ7Hy4syeXlRpudYvZpR1IiM4JkRXUlNrBuqEEPK1+L1o8pw7p7AZlXNBBCRacA1QEYJ5UcCj5XhesYYU6z1Twxiy56jLM3cxx//t+60Y/uOOaudXffS1559X/y+Hy29Zjit6gL5bNQM2Oa1nQ1cWFxBEWkFJAGfl3B8DDAGoGXLluUbpTEmLLRpUIs2DWpx44U/DYHatOsICbFR9PrrfPK9xhn0eWoBGROuIDYqPKqPKspdjgDeU9WC4g6q6mRgMjhTTAQzMGNM1dXOnZRu818GA3Dw+Em6TpgHwLb9OZzfODwmrfO3++i52A608Npu7u4rzgisO6oxJsQSYqN48UZn0uX0HYfCZhCav1NMxIrIIyLyirvdTkSu8vG25UA7EUkSkSicP/azijl3MlAH+LroMWOMCbY12QcBuO+dNdz11qrQBhMk/j4RvIazcH0vd3s78OfS3qCq+cA9wFxgPfCOqqaLyAQRudqr6Ahgmla2aVCNMVWS94I2h8JkEJq/S1WuUNVUEVmtqhe4+9aoapeAR1iETUNtjAmGwsVuANo1rMXboy+iQVx0CCMqm9Kmofb3ieCkiMTgjB1ARNrgPCEYY0yV1D+5oef1pt1H2XbgeAijCSx/ew2NBz4BWojIW8AlwK0BiskYY0Juyq09AFi4cTe3VvEJ6/x6IlDVT4Ff4vzxnwqkqurCwIVljDEVw8HjTjvB3z7eEOJIAsevJwIRmQ28DcxS1WOBDckYYyqOwmUuv/l+PyfyC4iuXvVWOvO3jeBpoDeQISLvich1IlLD15uMMaay63NeAy5q7cxBdNf/reJIbh5ffLeHmWnb+SxjF3uPVv7mUn+noV4ELHInkusPjAamAPGlvtEYY6qAn3dpytLM/czfsJvO4z8949hzIyv3fJl+jyx2ew0NBe4AegBvBCooY4ypSG68sBV/vraTZ/vy9o2Iru78+Zy9Zgebdx8NVWjlwt82gndwZhP9BHgeWKSqpwIZmDHGVCQ3XdSKm7wGm8FPYw1mr9nBvQMr7zLu/j4RvAq0UdU7VHWBJQFjjIGsiUMAeGb+JhIf/IilmftCHNG58bVCWX9V/RyoCVwjIqcd97VCmTHGhJMRk5dyx2VtGJbanDYNaoU6HL/5qhq6DGeNgOJWKit1hTJjjAkHWROHcDL/FFc+8wVb9hzjpUVbOKXKw4Pbhzo0v/laoaxwxbAJqvq99zERSQpYVMYYU4lEVa/GZ/ddxvaDOfT/xyJOnapcc2j620Ywo5h975VnIMYYU5mJCM3rxBJZTXwXrmB8tREkAx2B2iLyS69D8YANKDPGmCKOnSwgc+8xVJWi7aoVla82gvOBq4AETm8nOIIzqMwYY0wRn2/YzYKNu+mf3CjUofjFVxvBTGCmiPRSVVtBzBhjfBjYoRHzMnZVqkVtSm0jEJEH3Jc3iMizRb98nVxEBonIRhHZLCIPllDmehHJEJF0EXn7HO7BGGMqjD9Wot5ChXxVDa13v5/1kmDuvEQvAAOBbGC5iMxS1QyvMu2Ah4BLVPWAiDQs/mzGGGMCxVfV0Gz3u2deIRGpBtRS1cM+zt0T2Kyqme77pgHXABleZUYDL6jqAfc6u8/6DowxxpSJX91HReRtEYkXkZrAOpzpqH/v423NgG1e29nuPm/nAeeJyFcislREBpVw/TEiskJEVuzZs8efkI0xJqTSth4MdQh+83ccQQf3CeBa4GMgCfhVOVy/OtAO6AuMBF4RkYSihVR1sqqmqmpqgwYNyuGyxhgTGLn5BQC88fUPbNtfOdY59jcRRIpIJE4imKWqebgL2ZdiO9DCa7u5u89bduH53JHL3+EkBmOMqZSSG8cT4Q4qe3x2eoij8Y+/ieBlIAtn8rkvRKQV4KuNYDnQTkSSRCQKGAHMKlLmA5ynAUSkPk5VUaafMRljTIX0zIiuAHy2fjeXTPycfRV8FTN/F69/VlWbqepgdfwA9PPxnnzgHmAuTu+jd1Q1XUQmiMjVbrG5wD4RyQAWAL9X1co5j6sxxriuSmnKg1cmA7D9YA7Pfb65Qs8/JKq+gxOR2sBjQB931yKciegOBTC2YqWmpuqKFWfdm9UYY4Ju3fZDXPXcYs/2zzo04uVfdQ/J1BMislJVU4s75m/V0BScaSWud78OA6+VT3jGGFM1dWpW+7TtTzN2cSK/4q3r5W8iaKOqj6lqpvv1ONA6kIEZY0xVkDVxCFkTh9ChSTwAo9+seDUa/iaCHBG5tHBDRC4BcgITkjHGVD2FbQY7Dla8P51+LV4P3AG86bYVABwAbglMSMYYU/X0Oa8BQzo3YeOuI6EO5Qw+E4GIdAXa4nT/3A7gx/QSxhhjivhi0x6O5Obz1ea9XNK2fqjD8fA1++ijwDvAUOAjYLglAWOMOTc5J51Rx++s2OajZHD5aiMYDnRV1ZFAD2BM4EMyxpiqafNfBtOybiwz03awtwINMvOVCE6o6nEAd6CXv43LxhhjitG8TgwAH6wuOuNO6Pj6w95aRGa5X7OBNl7bRaeLMMYY48MrNztjugoq0EhjX43F1xTZfjpQgRhjTDioiOvZ+1qYZlGwAjHGGBMavnoNzRaRn7tTUBc91tqdQO7XgQvPGGOqpu92HcWfud6CwVfV0GjgPuDfIrIf2APUABKBLcDzqjozoBEaY0wVUvi3f8aqbBrGR/OHQcmhDQjfVUM/Ag8AD4hIItAEZ2qJ7wp7ExljjPFfzejqNIyLZveRE0xauIU7+7YhvsYZlS5B5Xd3UFXNUtWvVTXNkoAxxpy7ZX+8nAHJDQEoKAh99ZCNCzDGmBDo3a6STDFRViIySEQ2ishmEXmwmOO3isgeEUlzv34TyHiMMcacyd/ZR8+aiEQALwADcRapXy4is1Q1o0jR6ap6T6DiMMYYUzq/nghE5BIRmSci34lIpoh8LyK+FpnvCWx2F7I5CUzjzAFqxhhjQszfqqFXgX8Cl+JMPpfqfi9NM8B7ir1sd19RQ0VkrYi8JyItijuRiIwRkRUismLPnj1+hmyMMRXfsqz9HMnNC2kM/iaCQ6r6saruVtV9hV/lcP3ZQKKqpgDzgDeKK6Sqk1U1VVVTGzRoUA6XNcaY0Jq/YTcAt/93JY/OTA9pLP4mggUi8pSI9BKRboVfPt6zHfD+hN/c3efhJpTCuVj/A3T3Mx5jjKnU7v/Z+Z7XR3LzQxiJ/43FF7rfU732KdC/lPcsB9qJSBJOAhgB3OBdQESaqOpOd/NqYL2f8RhjTKXWpUUCWROHMPiZL0Mdin+JQFX7ne2JVTVfRO4B5gIRwBRVTReRCcAKVZ0FjBORq4F8YD9w69lexxhjTNn4lQjcResfA/q4uxYBE1T1UGnvU9U5wJwi+x71ev0Q8NDZBGyMMaZ8+dtGMAU4Alzvfh0GXgtUUMYYY4LH3zaCNqo61Gv7cRFJC0A8xhhjgszfJ4IcEbm0cENELsGZhdQYY0wl5+8TwZ3AG25bgWANu8YYU2X422soDegiIvHu9uFABmWMMSZ4Sk0EInKTqv6fiNxXZD8AqvrPAMZmjDEmCHw9EdR0v8cFOhBjjDGh4Wupypfd748HJxxjjDHB5u801H8XkXgRiRSR+e5iMjcFOjhjjDGB52/30Z+5DcRXAVlAW+D3gQrKGGPCRcbOw3y2fhfzMnaFLAZ/E0FhFdIQ4F1fU0sYY4zxT5sGTlPst9tD92fV30TwoYhswJkmer6INAByAxeWMcaEh/m/6xvqEPxLBKr6IHAxkKqqecAxbNlJY4ypEnyNI+ivqp+LyC+99nkXeT9QgRljjAkOX+MILgM+B35ezDHFEoExxpSLZ+dvYlj35rSoGxv0a/saR/CY+31UcMIxxpjw9caSLH59aRJNE2KCel1/xxH8RUQSvLbriMif/XjfIBHZKCKbReTBUsoNFREVkdSSyhhjTFX1+qgeAPxn8fdcPPFz9h87GdTr+9tr6EpVPVi4oaoHgMGlvUFEIoAXgCuBDsBIEelQTLk44LfAN37GYowxVcpl5zWge6s6nu0H3lsb1Ov7mwgiRCS6cENEYoDoUsoD9AQ2q2qmqp4EplF8T6MngL9h3VGNMWFKRJhx58V88v96A3DsRH5Qr+9vIngLZ/zAbSJyGzAPeMPHe5oB27y2s919HiLSDWihqh/5GYcxxlRZyY3j6ZlYN+jX9Xc9gr+JyBrgcnfXE6o6tywXFpFqwD/xY4EbERkDjAFo2bJlWS5rjDGmCH9XKANYD+Sr6mciEisicap6pJTy24EWXtvN3X2F4oBOwEJ3bEJjYJaIXK2qK7xPpKqTgckAqampehYxG2OM8cHfXkOjgfeAl91dzYAPfLxtOdBORJJEJAoYAcwqPKiqh1S1vqomqmoisBQ4IwkYY4wJLH/bCO4GLgEOA6jqJqBhaW9Q1XzgHmAuztPEO6qaLiITROTqcw/ZGGNMefK3auiEqp4snF5CRKrjjCwularOAeYU2fdoCWX7+hmLMcaYcuTvE8EiEXkYiBGRgcC7wOzAhWWMMSZY/E0EfwD2AN8Ct+N8yv9ToIIyxhgTPD6rhtwRwumqmgy8EviQjDHGBJPPJwJVLQA2ioh14DfGmCrI38biOkC6iCzDWZQGAFW13j/GGFPJ+ZsIHgloFMYYY0LG1wplNYA7gLY4DcWvuuMDjDHGVBG+2gjeAFJxksCVwD8CHpExxpig8lU11EFVOwOIyKvAssCHZIwx4WtZ1n4Almzey8Vt6wflmr6eCPIKX1iVkDHGBF5khDODw+cbdgftmr4SQRcROex+HQFSCl+LyOFgBGiMMeFk05PO4o//Wfw9h3LyfJQuH6UmAlWNUNV49ytOVat7vY4PSoTGGBOmFm4MzlOBv1NMGGOMCZIPx14KwCkNzvIrlgiMMaaCqRV9NmuGlZ0lAmOMCXOWCIwxJswFNBGIyCAR2Sgim0XkwWKO3yEi34pImogsFpEOgYzHGGPMmQKWCNzpq1/AGZHcARhZzB/6t1W1s6p2Bf4O/DNQ8RhjjCleIJ8IegKbVTVTVU8C04BrvAuoqvdYhJr4sfylMcaY8hXIpulmwDav7WzgwqKFRORu4D4gCugfwHiMMcYUI+SNxar6gqq2wVkOs9jlL0VkjIisEJEVe/bsCW6AxhgTIu+v2o4GYSxBIBPBdqCF13Zzd19JpgHXFndAVSeraqqqpjZo0KD8IjTGmAooITYSgC837eX7vcd8lC67QCaC5UA7EUkSkShgBDDLu4CItPPaHAJsCmA8xhhTKSTERnH/z84D4M8frQ/49QLWRqCq+SJyDzAXiACmqGq6iEwAVqjqLOAeEbkcZ5bTA8AtgYrHGGMqk/MbO9O5BWMW0oCOY1bVOcCcIvse9Xr920Be3xhjKquBHRpxV982vPJlZsCvFfLGYmOMMaFlicAYY8KcJQJjjAlzlgiMMSbMBXfS6wDJy8sjOzub3NzcUIdiDAA1atSgefPmREZGhjoUY3yqEokgOzubuLg4EhMTEZFQh2PCnKqyb98+srOzSUpKCnU4xvhUJaqGcnNzqVevniUBUyGICPXq1bMnVFMu8gqUuek/BvQaVSIRAJYETIViv4+mPHRrWQeAFxduCeh1qkwiMMaYqubyDo3oe34DCPDEc5YIysmuXbu44YYbaN26Nd27d6dXr17873//K7bsjh07uO6664o91rdvX1asWAHAlClT6Ny5MykpKXTq1ImZM2cGLP6srCw6depU4vGnn36a5ORkunbtSo8ePXjzzTd5/PHHeeihh04rl5aWRvv27Ys9x3XXXUdmZuZpZUWETz755LRyERERdO3alU6dOjFs2DCOHz9ehjtz6uzHjRtH27ZtSUlJYdWqVcWWmzp1qufnPWjQIPbu3QvA+PHjadasGV27dqVr167MmeMMlv/222+59dZbyxSbMRWBJYJyoKpce+219OnTh8zMTFauXMm0adPIzs4+o2x+fj5NmzblvffeK/Wc2dnZPPnkkyxevJi1a9eydOlSUlJSyhxrfn7+Wb/npZdeYt68eSxbtoy0tDTmz5+PqjJy5EimT59+Wtlp06YxcuTIM86Rnp5OQUEBrVu39uybOnUql156KVOnTj2tbExMDGlpaaxbt46oqCheeumls47Z28cff8ymTZvYtGkTkydP5s477zyjTH5+Pr/97W9ZsGABa9euJSUlheeff95z/N577yUtLY20tDQGDx4MQOfOncnOzmbr1q1lis+YUKsSvYa8PT47nYwdh30XPAsdmsbz2M87lnj8888/JyoqijvuuMOzr1WrVowdOxaA119/nffff5+jR49SUFDAG2+8wVVXXcW6devIyclh1KhRrFmzhuTkZHJycgDYvXs3cXFx1KpVC4BatWp5Xm/ZsoW7776bPXv2EBsbyyuvvEJycjKzZ8/mz3/+MydPnqRevXq89dZbNGrUiPHjx7NlyxYyMzNp2bIl//73v7njjjs8n84nTZpE06ZNKSgoYPTo0SxZsoRmzZoxc+ZMYmJi+Mtf/sLChQuJj3cmwYqPj+eWW5z5AevUqcM333zDhRc6aw698847zJ0794yf0VtvvcU11/y0QJ2q8u677zJv3jx69+5Nbm4uNWrUOON9vXv3Zu3atX7+SxVv5syZ3HzzzYgIF110EQcPHmTnzp00adLktHhUlWPHjlGvXj0OHz5M27ZtfZ775z//OdOmTeOBBx4oU4zGhJI9EZSD9PR0unXrVmqZVatW8d5777Fo0aLT9k+aNInY2FjWr1/P448/zsqVKwHo0qULjRo1IikpiVGjRjF79mzPe8aMGcNzzz3HypUrefrpp7nrrrsAuPTSS1m6dCmrV69mxIgR/P3vf/e8JyMjg88++4ypU6cybtw4LrvsMtasWcOqVavo2NFJcps2beLuu+8mPT2dhIQEZsyYweHDhzly5Mhpn+S9jRw5kmnTpgGwdOlS6tatS7t27c4o99VXX9G9e3fP9pIlS0hKSqJNmzb07duXjz766Iz35Ofn8/HHH9O5c+czjg0fPtxTVeP99eabb55Rdvv27bRo8dPSGM2bN2f79tOXxoiMjGTSpEl07tyZpk2bkpGRwW233eY5/vzzz5OSksKvf/1rDhw44NmfmprKl19+WezPxpjKoso9EZT2yT1Y7r77bhYvXkxUVBTLly8HYODAgdStW/eMsl988QXjxo0DICUlxVP9ExERwSeffMLy5cuZP38+9957LytXruT+++9nyZIlDBs2zHOOEydOAE510vDhw9m5cycnT548rQ/71VdfTUxMDOA8wRT+wYyIiKB27docOHCApKQkunbtCkD37t3Jysryea/Dhw/n4osv5h//+EeJ1UIAO3fuxHtRoalTpzJixAgARowYwZtvvsnQoUMByMnJ8cTRu3fv0/4gFypaJVVWeXl5TJo0idWrV9O6dWvGjh3LX//6V/70pz9x55138sgjjyAiPPLII/zud79jypQpADRs2JAdO3aUayzGBFuVSwSh0LFjR2bMmOHZfuGFF9i7dy+pqamefTVr1jzr84oIPXv2pGfPngwcOJBRo0Zx3333kZCQQFpa2hnlx44dy3333cfVV1/NwoULGT9+/FldPzo62vM6IiKCnJwc4uPjqVWrFpmZmcU+FbRo0YKkpCQWLVrEjBkz+Prrr4s9d0xMjKdffUFBATNmzGDmzJk8+eSTngFYR44cIS4uztNGUJrhw4ezcePGM/bfd9993Hzzzafta9asGdu2/bR8dnZ2Ns2aNTutTOH12rRpA8D111/PxIkTAWjUqJGn3OjRo7nqqqs827m5uZ4Ea0xlZVVD5aB///7k5uYyadIkzz5/e7r06dOHt99+G4B169Z56sN37NhxWu+WtLQ0WrVqRXx8PElJSbz77ruAU7e9Zs0aAA4dOuT5A/fGG2+UeM0BAwZ4Yi0oKODQoUOlxvjQQw9x9913c/iw0/Zy9OjR06pgRo4cyb333kvr1q1p3rx5sedo3749mzdvBmD+/PmkpKSwbds2srKy+OGHHxg6dGiJvayKM336dE/jrfdX0SQAztPQm2++iaqydOlSateufVr7ADjJIiMjg8I1sefNm+fp/bRz505Puf/973+n9a767rvvSu1tZUxlENBEICKDRGSjiGwWkQeLOX6fiGSIyFoRmS8irQIZT6CICB988AGLFi0iKSmJnj17csstt/C3v/3N53vvvPNOjh49Svv27Xn00Uc99eh5eXncf//9ni6b06dP55lnngGchtdXX32VLl260LFjR0+30vHjxzNs2DC6d+9O/fr1S7zmM888w4IFC+jcuTPdu3cnIyPDZ4z9+vWjR48edOrUid69e1Ot2k+/OsOGDSM9Pb3EaiGAIUOGsHDhQsCpFvrFL35x2vGhQ4ee0XuovAwePJjWrVvTtm1bRo8ezYsvvug5VlgF1bRpUx577DH69OlDSkoKaWlpPPzwwwA88MADnm6lCxYs4F//+pfn/QsWLGDIkCEBiduYQht+PMLAfy5i9prAVEOKBmiggohEAN8BA4FsnDWMR6pqhleZfsA3qnpcRO4E+qrq8NLOm5qaqoX97AutX7++xL7rpmLIycmhX79+fPXVV0RERIQ6nHJx4sQJLrvsMhYvXkz16mfWstrvpSkPc9N/ZGaa07lhRI+W9DmvgY93FE9EVqpqanHHAtlG0BPYrKqZbhDTgGsATyJQ1QVe5ZcCNwUwHhNCMTExPP7442zfvp2WLVuGOpxysXXrViZOnFhsEjCmvFzRsTFXdGwc0GsE8je4GbDNazsbuLCU8rcBHwcwHhNiV1xxRahDKFft2rUrtqusMZVNhfgoIyI3AanAZSUcHwOMAUr8NKmqNtGXqTACVeVqTCAEsrF4O9DCa7u5u+80InI58EfgalU9UdyJVHWyqqaqaqp3X/RCNWrUYN++ffafz1QIhd1hixspbUxFFMgnguVAOxFJwkkAI4AbvAuIyAXAy8AgVd19rhdq3rw52dnZnq5/xoRa4QplxlQGAUsEqpovIvcAc4EIYIqqpovIBGCFqs4CngJqAe+61TpbVfXqs71WZGSkrQRljDHnKKBtBKo6B5hTZN+jXq8vD+T1jTHG+GYji40xJsxZIjDGmDAXsJHFgSIie4AfzvHt9YG95RhOZWD3HB7snsNDWe65laoWOyy50iWCshCRFSUNsa6q7J7Dg91zeAjUPVvVkDHGhDlLBMYYE+bCLRFMDnUAIWD3HB7snsNDQO45rNoIjDHGnCncngiMMcYUYYnAGGPCXJVMBH4skRktItPd49+ISGIIwixX4bIsqDdf9+xVbqiIqIhU+q6G/tyziFzv/luni8jbwY6xvPnxu91SRBaIyGr393twKOIsLyIyRUR2i8i6Eo6LiDzr/jzWiki3Ml9UVavUF84Ed1uA1kAUsAboUKTMXcBL7usRwPRQxx2Ee+4HxLqv7wyHe3bLxQFf4KyAlxrquIPw79wOWA3UcbcbhjruINzzZOBO93UHICvUcZfxnvsA3YB1JRwfjLOIlwAX4Sz3W6ZrVsUnAs8Smap6EihcItPbNcAb7uv3gAFSuVe18XnPqrpAVY+7m0tx1oeozPz5dwZ4AvgbkBvM4ALEn3seDbygqgcAtAzTu1cQ/tyzAvHu69pAYFZ4DxJV/QLYX0qRa4A31bEUSBCRJmW5ZlVMBMUtkdmspDKqmg8cAuoFJbrA8OeevVWFZUF93rP7yNxCVT8KZmAB5M+/83nAeSLylYgsFZFBQYsuMPy55/HATSKSjTPb8djghBYyZ/v/3acKsVSlCR5fy4JWFSJSDfgncGuIQwm26jjVQ31xnvq+EJHOqnowlEEF2EjgdVX9h4j0Av4rIp1U9VSoA6ssquITgT9LZHrKiEh1nMfJfUGJLjDKbVnQSsTXPccBnYCFIpKFU5c6q5I3GPvz75wNzFLVPFX9HvgOJzFUVv7c823AOwCq+jVQA2dytqrKr//vZ6MqJgLPEpkiEoXTGDyrSJlZwC3u6+uAz9VthamkfN6z17KgV1eBemPwcc+qekhV66tqoqom4rSLXK2qK0ITbrnw53f7A5ynAUSkPk5VUWYQYyxv/tzzVmAAgIi0x0kEVXnd2lnAzW7voYuAQ6q6sywnrHJVQ+rfEpmv4jw+bsZplBkRuojLzs97LpdlQSsKP++5SvHznucCPxORDKAA+L2qVtqnXT/v+XfAKyJyL07D8a2V+YOdiEzFSeb13XaPx4BIAFV9CacdZDCwGTgOjCrzNSvxz8sYY0w5qIpVQ8YYY86CJQJjjAlzlgiMMSbMWSIwxpgwZ4nAGGPCnCUCE3AiUiAiaSKyTkRmi0hCOZ8/y+0zj4gcLaFMjIgsEpEIEUkUkRw3pgwReckdiXw210wVkWfd131F5GKvY3eIyM1luSf3PONF5H4fZV4XkevO4pyJJc1qWaTckyKyrejPU0TuEZFf+3s9UzlYIjDBkKOqXVW1E864jbtDEMOvgfdVtcDd3qKqXYEUnBkrrz2bk6nqClUd5272BS72OvaSqr5Z1oBDbDbOhG9FTaHqz+UTdiwRmGD7GneCLBFpIyKfiMhKEflSRJLd/Y1E5H8issb9utjd/4FbNl1ExpzldW8EZhbd6U46uARo635a/lx+WrOhpXvdYe7TzBoR+cLd11dEPhRnLYs7gHvdJ4zehZ/kRSRZRJYVXss9/7fu6+7uE8pKEZkrPmaPFJHRIrLcjWGGiMR6Hb5cRFaIyHcicpVbPkJEnnLfs1ZEbj+bH5aqLi1utKo7g22WiBSXJEwlZYnABI2IROBMBVA46ncyMFZVuwP3Ay+6+58FFqlqF5x52dPd/b92y6YC40TErxlj3akJWqtqVjHHYt2YvgWeA95Q1RTgLTcOgEeBK9x4ThuN7Z7zJeBf7lPPl17HNgBRIpLk7hoOTBeRSPda17n3MwV40sdtvK+qPdwY1uPMr1MoEefT+xDgJRGp4R4/pKo9gB7AaK84Cu+9qYjM8XHd4qwAep/D+0wFVeWmmDAVUoyIpOE8CawH5olILZzqlMIpLwCi3e/9gZsB3KqcQ+7+cSLyC/d1C5zJ1PyZPqE+cLDIvjZuTArMVNWPReS/wC/d4/8F/u6+/gp4XUTeAd7343re3sFJABPd78OB83EmxJvn3nsE4GuumE4i8mcgAWeqkLne13Bn2twkIplAMvAzIMWr/aA2zs/ru8I3qeoOnKkKztZu9xqmirBEYIIhR1W7up++5+K0EbwOHHTr6X0Skb7A5UAvVT0uIgtxJhfz6/rFlN3i77VV9Q4RuRDnE/dKEenu53UBpuMku/edU+kmEekMpKtqr7M4z+vAtaq6RkRuxZ1YrjDEoiHjrF41VlW9EwZSPsuy1sD5mZoqwqqGTNC49cvjcCYJOw58LyLDwLMOaxe36Hyc5TQL67pr43yiPeAmgWScaaX9ve4BIMKtMinNEn6agPBG4Es3hjaq+o2qPoozq2WLIu87gjPtdXHX3oIz+dsjOEkBYCPQQJy58xGRSBHp6CO2OGCnW610Y5Fjw0Skmoi0wVnScSNOwr3TLY+InCciNX1cw1/nAT57HpnKwxKBCSpVXQ2sxVlM5EbgNhFZg9MOULgE4W+Bfm7D6kqcXj2fANVFZD1ONcvSs7z0p8ClPsqMBUaJyFrgV24cAE+JyLdut8slOOvmepsN/KKwsbiY804HbuKnOfNP4kx//jf33tPw6nVUgkeAb3CqqTYUObYVWIaz6twdqpoL/AfIAFa5cb9MkRqA0toIROTv4sx8GSsi2SIy3uvwJcA8H/GaSsRmHzVhQZxlK+9V1V+FOpbKTJx1Le6zn2PVYk8EJiyo6ipggdtzyZy7+jhPJ6YKsScCY4wJc/ZEYIwxYc4SgTHGhDlLBMYYE+YsERhjTJizRGCMMWHu/wNjT6/CTKCiIwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import plot_precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "\n",
    "plt.figure(figsize=(16,10))\n",
    "\n",
    "disp = plot_precision_recall_curve(clf, test_feature_df, test_true_y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}