bert_amazon_fine_food_score_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Amazon Fine Food Score Prediction using BERT",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1JTUG_zuTwDo0wpmQW4ILtKM2TSar_4t5",
      "authorship_tag": "ABX9TyMj6qp7y9Pby4EiqUaz1vXm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "49fd906fd5be40f3a070ac0fbdb8cb7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e47520e035eb477fbcbb740034abb96b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_353f5b1d49df4d5dbafb5b6af1bc8acb",
              "IPY_MODEL_aac730db09f64696a61bdcf2cc8b303a"
            ]
          }
        },
        "e47520e035eb477fbcbb740034abb96b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "353f5b1d49df4d5dbafb5b6af1bc8acb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bb0017909bfd40f3aa2cb6b93231f5ee",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 442,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 442,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_223f5d7ce6e4479a97f4c5d863a3295b"
          }
        },
        "aac730db09f64696a61bdcf2cc8b303a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_423acf71a625430abf154269b2e611f3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 442/442 [00:00&lt;00:00, 516B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0a6b333b7533484ea6a8a9d5a1b4f3c2"
          }
        },
        "bb0017909bfd40f3aa2cb6b93231f5ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "223f5d7ce6e4479a97f4c5d863a3295b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "423acf71a625430abf154269b2e611f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0a6b333b7533484ea6a8a9d5a1b4f3c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4366394a2c434d618955e90c69b7f26a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f3ac9ade05fa4a7bb32f3dcec44e0122",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2669d019cf3547169e0af2f15aa5a32e",
              "IPY_MODEL_1148bfb3edc84bd99b5639c41218580d"
            ]
          }
        },
        "f3ac9ade05fa4a7bb32f3dcec44e0122": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2669d019cf3547169e0af2f15aa5a32e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6cc14cf5e5034f62ab44a6ecae0de519",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c1f70d1964a744f6836720b1fd94a489"
          }
        },
        "1148bfb3edc84bd99b5639c41218580d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4bdf22a594fd4826b9568ad90ad4c661",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 1.01MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_31c2fbad30dc4a3fb41f5a120cb07638"
          }
        },
        "6cc14cf5e5034f62ab44a6ecae0de519": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c1f70d1964a744f6836720b1fd94a489": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4bdf22a594fd4826b9568ad90ad4c661": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "31c2fbad30dc4a3fb41f5a120cb07638": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2f5a9d126b9a492bbae66ec02fc45f57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_abb0ecbd4b854ecba42e82141daa89e3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cb9484d6b97041c4ac344c5bcf6d931c",
              "IPY_MODEL_60549ba240af4e578298bfc7b401f9af"
            ]
          }
        },
        "abb0ecbd4b854ecba42e82141daa89e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cb9484d6b97041c4ac344c5bcf6d931c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f55467092ecc4a78ad7dfdc2dd807109",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 442,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 442,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_98122e237e6e42b08ea4d95be5a354a3"
          }
        },
        "60549ba240af4e578298bfc7b401f9af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2a108e2173944e67bc9a08e133e6e74d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 442/442 [00:00&lt;00:00, 1.96kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_168a9f4eb24349deb369a99fd0bde35b"
          }
        },
        "f55467092ecc4a78ad7dfdc2dd807109": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "98122e237e6e42b08ea4d95be5a354a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2a108e2173944e67bc9a08e133e6e74d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "168a9f4eb24349deb369a99fd0bde35b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "71e8447401164da0beb3c1a69aa70622": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f8feb2a56dcb476da9c4cf346df908b6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_46441cfd20af428d8d3c9066e4100956",
              "IPY_MODEL_c0055b1829b44641b2dc4522c03bf4a5"
            ]
          }
        },
        "f8feb2a56dcb476da9c4cf346df908b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "46441cfd20af428d8d3c9066e4100956": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cac0e858532b4b47bfd4ddf048915a08",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2e8e6d96ab074f898e18641f6d9e932c"
          }
        },
        "c0055b1829b44641b2dc4522c03bf4a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_05ee0270614b400f9bd9bb29cdab057f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 2.15MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_235510309d4b42248d5d66cf773daa26"
          }
        },
        "cac0e858532b4b47bfd4ddf048915a08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2e8e6d96ab074f898e18641f6d9e932c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "05ee0270614b400f9bd9bb29cdab057f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "235510309d4b42248d5d66cf773daa26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/stephan-strate/sentiment-fine-foods/blob/master/bert_amazon_fine_food_score_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QQnZJNoqzVSJ",
        "outputId": "a8773981-0aa0-4bc0-e0f3-d206161db8b8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "!pip install -q ktrain"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 25.3MB 129kB/s \n",
            "\u001b[K     |████████████████████████████████| 983kB 49.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 245kB 40.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 45.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 471kB 47.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 48.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.0MB 37.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 50.7MB/s \n",
            "\u001b[?25h  Building wheel for ktrain (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-bert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for syntok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-transformer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-pos-embd (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-multi-head (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-layer-normalization (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-position-wise-feed-forward (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-embed-sim (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras-self-attention (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kO18CO-TzaGP"
      },
      "source": [
        "MODEL_NAME = \"distilbert-base-uncased\"\n",
        "BATCH_SIZE = 32\n",
        "EPOCHS = 2\n",
        "LEARNING_RATE = 5e-5\n",
        "MAX_LEN = 128\n",
        "RANDOM = 42\n",
        "\n",
        "TU_GREEN = \"#639a00\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y6VwK2mwzqqS"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import urllib.request\n",
        "import zipfile\n",
        "\n",
        "url = \"https://storage.statc.de/datasets/amazon-fine-food/amazon-fine-food.zip\"\n",
        "filehandle, _ = urllib.request.urlretrieve(url)\n",
        "zip_file_object = zipfile.ZipFile(filehandle, \"r\")\n",
        "zip_file_object.extractall()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c1jd2V_40NJV",
        "outputId": "4db4bc59-4776-4f20-fe0d-93a6a167b682",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df = pd.read_csv(\"amazon-fine-food-cleaned.csv\", delimiter=\",\", header=0)\n",
        "print(\"Dataset has {:,} rows\".format(df.shape[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dataset has 393,670 rows\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sd5vx2s-46qD"
      },
      "source": [
        "import math\n",
        "\n",
        "def calculate_helpfulness(percentage):\n",
        "  if math.isnan(percentage):\n",
        "    return -1\n",
        "  if percentage > 0.5:\n",
        "    return 1\n",
        "  return 0\n",
        "\n",
        "df[\"Helpfulness\"] = (df[\"HelpfulnessNumerator\"] / df[\"HelpfulnessDenominator\"]).apply(calculate_helpfulness)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XjV3s2mMuWZN",
        "outputId": "e7cc922f-382b-4a05-e83b-2cd17c6c2895",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train, test = train_test_split(df, test_size=0.3, random_state=42)\n",
        "\n",
        "print(\"Train dataset has {:,} rows\".format(train.shape[0]))\n",
        "print(\"Test dataset has {:,} rows\".format(test.shape[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train dataset has 275,569 rows\n",
            "Test dataset has 118,101 rows\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iXU_AQuL0Fez"
      },
      "source": [
        "import ktrain\n",
        "from ktrain import text\n",
        "from pathlib import Path\n",
        "import random\n",
        "from os import path\n",
        "import os.path\n",
        "import itertools\n",
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "random.seed(RANDOM)\n",
        "\n",
        "def train_and_score(X_train, X_test, X_validate, y_train, y_test, y_validate, model_name=None):\n",
        "  print(\"train size: {:,}\".format(len(X_train)))\n",
        "  print(\"test size: {:,}\".format(len(X_test)))\n",
        "  print(\"validate size: {:,}\".format(len(X_validate)))\n",
        "\n",
        "  model_path = \"/content/drive/My Drive/Colab Notebooks/models/{}\".format(model_name)\n",
        "  # check if model was already trained, load model without training\n",
        "  if path.exists(model_path):\n",
        "    predictor = ktrain.load_predictor(model_path)\n",
        "    print(predictor.get_classes())\n",
        "  else:\n",
        "    # text transformation configuration\n",
        "    t = text.Transformer(MODEL_NAME, maxlen=MAX_LEN, class_names=[0, 1, 2, 3, 4])\n",
        "\n",
        "    # pre-process text (input vectors, truncation, ...)\n",
        "    trn = t.preprocess_train(X_train, y_train)\n",
        "    val = t.preprocess_test(X_test, y_test)\n",
        "\n",
        "    # learn model\n",
        "    model = t.get_classifier()\n",
        "    learner = ktrain.get_learner(model, train_data=trn, val_data=val, batch_size=BATCH_SIZE)\n",
        "    learner.fit(LEARNING_RATE, EPOCHS)\n",
        "\n",
        "    # save predictor\n",
        "    predictor = ktrain.get_predictor(learner.model, preproc=t)\n",
        "    predictor.save(model_path)\n",
        "\n",
        "  score_path = model_path + \"/score\"\n",
        "  if path.exists(score_path):\n",
        "    score = Path(score_path).read_text()\n",
        "    predictions = np.load(model_path + \"/predictions.npy\")\n",
        "    y_validate = np.load(model_path + \"/true_labels.npy\")\n",
        "  else:\n",
        "    predictions = predictor.predict(X_validate)\n",
        "    np.save(model_path + \"/predictions\", predictions)\n",
        "    np.save(model_path + \"/true_labels\", y_validate)\n",
        "    comparison = list(map(lambda x: x[0] == x[1], zip(predictions, y_validate)))\n",
        "    score = comparison.count(True) / len(y_validate)\n",
        "    Path(score_path).write_text(str(score))\n",
        "\n",
        "  print(\"Validation score: {}\".format(score))\n",
        "\n",
        "  return predictor, score, predictions, y_validate\n",
        "\n",
        "def train_test_validate_split(X, y, train_size=0.8, sample_size=15000):\n",
        "  shuffled = random.sample(list(zip(X, y)), len(X))\n",
        "  validate = shuffled[sample_size:]\n",
        "  assert len(validate) == len(X) - sample_size\n",
        "  \n",
        "  train_test = shuffled[:sample_size]\n",
        "  assert len(train_test) == sample_size\n",
        "  train_test_cut = int(train_size * len(train_test))\n",
        "  train, test = train_test[:train_test_cut], train_test[train_test_cut:]\n",
        "\n",
        "  X_validate, y_validate = zip(*validate)\n",
        "  X_train, y_train = zip(*train)\n",
        "  X_test, y_test = zip(*test)\n",
        "\n",
        "  return list(X_train), list(X_test), list(X_validate), list(y_train), list(y_test), list(y_validate)\n",
        "\n",
        "def train_test_validate_split_groups(groups, train_size=0.8, sample_size_group=15000):\n",
        "  X_train, X_test, X_validate, y_train, y_test, y_validate = zip(*list(map(lambda x: train_test_validate_split(x[1].TextClean.values, x[1].Score.values - 1, sample_size=sample_size_group, train_size=train_size), groups)))\n",
        "  f = lambda x: list(itertools.chain.from_iterable(x))\n",
        "\n",
        "  return f(X_train), f(X_test), f(X_validate), f(y_train), f(y_test), f(y_validate)\n",
        "\n",
        "def confusion_matrix_custom(true_labels, predictions, name=None):\n",
        "  cm = confusion_matrix(true_labels, predictions, [0, 1, 2, 3, 4], normalize=\"true\")\n",
        "\n",
        "  fig = plt.figure(figsize=(7, 5))\n",
        "  ax = fig.add_subplot(111)\n",
        "  for (i, j), z in np.ndenumerate(cm):\n",
        "      if i == j:\n",
        "        ax.text(j, i, \"{:0.2f}\".format(z), ha=\"center\", va=\"center\", fontsize=12)\n",
        "  cax = ax.matshow(cm, cmap=plt.cm.Greens, vmin=0, vmax=1)\n",
        "  fig.colorbar(cax)\n",
        "  ax.set_xticklabels([''] + [0, 1, 2, 3, 4])\n",
        "  ax.set_yticklabels([''] + [0, 1, 2, 3, 4])\n",
        "  plt.xlabel(\"Predicted\")\n",
        "  plt.ylabel(\"True\")\n",
        "\n",
        "  plt.savefig(name)\n",
        "  plt.show()\n",
        "  return cm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eWcs-UfkfdcU",
        "outputId": "25eb6263-0647-4398-c4ac-46b34cff85d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "base_line_predictor, base_line_score, base_line_predictions, base_line_true_labels = train_and_score(*train_test_validate_split(train.TextClean.values, train.Score.values - 1, sample_size=200000), model_name=\"base_line_real\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train size: 160,000\n",
            "test size: 40,000\n",
            "validate size: 75,569\n",
            "[0, 1, 2, 3, 4]\n",
            "Validation score: 0.7676031176805304\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zVbmC_b7MS7h",
        "outputId": "a6605d11-e593-495c-af7f-b125fa0544b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "49fd906fd5be40f3a070ac0fbdb8cb7a",
            "e47520e035eb477fbcbb740034abb96b",
            "353f5b1d49df4d5dbafb5b6af1bc8acb",
            "aac730db09f64696a61bdcf2cc8b303a",
            "bb0017909bfd40f3aa2cb6b93231f5ee",
            "223f5d7ce6e4479a97f4c5d863a3295b",
            "423acf71a625430abf154269b2e611f3",
            "0a6b333b7533484ea6a8a9d5a1b4f3c2",
            "4366394a2c434d618955e90c69b7f26a",
            "f3ac9ade05fa4a7bb32f3dcec44e0122",
            "2669d019cf3547169e0af2f15aa5a32e",
            "1148bfb3edc84bd99b5639c41218580d",
            "6cc14cf5e5034f62ab44a6ecae0de519",
            "c1f70d1964a744f6836720b1fd94a489",
            "4bdf22a594fd4826b9568ad90ad4c661",
            "31c2fbad30dc4a3fb41f5a120cb07638"
          ]
        }
      },
      "source": [
        "base_line_predictions = base_line_predictor.predict(test.TextClean.values)\n",
        "true_labels = test.Score.values - 1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "49fd906fd5be40f3a070ac0fbdb8cb7a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4366394a2c434d618955e90c69b7f26a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9sSs7JO1Elwp",
        "outputId": "7eae6e00-f5da-48b2-b8a7-aa74457bc6cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "cm1 = confusion_matrix_custom(true_labels, base_line_predictions, \"base_line_matrix.pdf\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAE9CAYAAAD9HVKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV1b3+8c83EyEJkwYQSRBQRNHKIBcHqlKn4khrrRWt2t7e8tNbrW0dqrbVqm2ttlXrlVsbh5+zVtRWqlScUGudAMEBEE0RBFQQRAQSCEm+949zwOTkZDjx7Jysk+fN67w4e++1114b9Mli7bX3NndHRETClJPpBoiISPspxEVEAqYQFxEJmEJcRCRgCnERkYApxEVEAtblQtzMJprZYjOrNLOLMt2edDGz28xstZm9lem2pJOZlZvZLDNbaGYLzOzcTLcpHcys0MxeNbPX4+d1eabblG5mlmtm88zs0Uy3JZt1qRA3s1xgKnAUMAKYbGYjMtuqtLkdmJjpRkSgFjjP3UcA+wM/yJK/sy3Aoe4+EhgFTDSz/TPcpnQ7F1iU6UZkuy4V4sA4oNLdl7h7DXA/MCnDbUoLd38e+CTT7Ug3d//Q3V+Lf99ALBQGZrZVX5zHbIwv5sc/WXPnnZmVAccAt2S6Ldmuq4X4QGB5g+UVZEEgdBVmNhgYDbyS2ZakR3y4YT6wGnjS3bPivOKuBy4E6jPdkGzX1UJcAmVmJcBDwI/c/bNMtycd3L3O3UcBZcA4M9s7021KBzM7Fljt7nMz3ZauoKuF+EqgvMFyWXyddGJmlk8swO9x94cz3Z50c/dPgVlkzzWN8cDxZraU2JDloWZ2d2ablL26WojPBoaZ2RAzKwBOBqZnuE3SAjMz4FZgkbtfm+n2pIuZ9TWz3vHv3YEjgLcz26r0cPeL3b3M3QcT+3/sGXf/doablbW6VIi7ey1wNjCT2AWyB9x9QWZblR5mdh/wEjDczFaY2fcy3aY0GQ+cRqw3Nz/+OTrTjUqDAcAsM3uDWOfiSXfXVDxJmelRtCIi4epSPXERkWyjEBcRCZhCXEQkYApxEZGAKcRFRDpIaw+qs5gb4g/oe8PMxrRWZ5cNcTObkuk2REHnFZ5sPbdsPa8v6HZavqnrKGBY/DMF+FNrFXbZECf2B5SNdF7hydZzy9bzarc2PKhuEnBn/AFpLwO9zWxAS3V25RAXEelsUn5IX16kzUlRTnG+5/Uu7JBj5fbqRsHAHh1yp9OInXbtiMMAUFZexqh9R3XYHVzWQccpG1TG6A49r47r35QPKmfMvmM67Nzqva5DjlNWPpBRY0Z2yHktf385a9d8krb/HK200KlJ8QGMG7YuADY3WFPh7hXpalNzOlWI5/UupN9/75vpZqTdM+dPy3QTIpNj2fmPuYKcbpluQmQ2bs2Kh0A2cuT4o9JbYU097NcvtX2eWrnZ3cd+wSOn/JC+7Pw/UETkizJL7ZMe04HT47NU9gfWu/uHLe3QqXriIiKdghFJFzf+oLoJQKmZrQAuI/ZWJ9z9JmAGcDRQCVQB322tToW4iEgy6etdb+fuk1vZ7sAPUqlTIS4ikkxHXbX/ghTiIiJNpHWcO1IKcRGRRBGNiUdBIS4ikox64iIiAQsjwxXiIiJNGJATRoorxEVEkgkjwxXiIiJJaUxcRCRgYWS4QlxEpAmNiYuIBC6MDFeIi4g0pTs2RUTCFdBwSiA3loqISDLqiYuIJBNGR1whLiKSlMbERUQCFkaGZ0+I11dtZd1fF7Olch05Rfn0PHIIRSP7Nym35o43qFm2fvuy1zl5pUX0P2cstZ9uZvUNsxuV95p6ek4cSo8vlydWJSLZKqALm5GGuJlNBP4I5AK3uPtvozrWp39/F8vNYaeLDmTrhxtZe9eb5O9UQn7/4kblSs/Yp9Hyx7fMp9vQ3gDk9S5k50sP2r6t9pNqVl33Kt336htVs0Wkswojw6ObnWJmucBU4ChgBDDZzEZEcaz6mjqqF66hx+GDyemWS7fBvSjcY0eq5q9qcb/adZupWbaeotE7Jd1eNX8VBYN7kdenMIpmi0hnlpm33acsyimG44BKd1/i7jXA/cCkKA5Uu6YKyzHyS4u2r8sfUELt6k0t7lc17yMKdkke0u5O1bxVzQa8iGS5nBQ/GRLloQcCyxssr4ivSzuvqcO65TZal9Mtj/otdS3uVzV/FUVjkod0zbL11G+q0VCKSFeUai88gz3xjF/YNLMpwBSA3F7d2ldHQS6eENj1W2rJSQj2hrYsXU/9xuZDumreKgpH9G2xDhHJYl19TBxYCTSc0lEWX9eIu1e4+1h3H5tTnN+uA+WVFuH1Tu2aqu3rtn60ibx+xc3uUzXvo2ZD2rfWUf3WxxSPbjq7RUS6iEB64lGG+GxgmJkNMbMC4GRgehQHyinIpfuIUj57ein1NXVsWbaezYvWUDQqeQi3FtLVC9eQ0z2PgvisFRHpgrr6mLi71wJnAzOBRcAD7r4gquP1Pm4YXlvPR1e9yLoHFtH7+GHk9y9my9JP+eCKfzYqW71wLTmFzYd01bxVFI3qjwVyx5aIpJkRTE880jFxd58BzIjyGNvkFOWz46l7N1nfbXDvRnO/AYpG9qNoZL9m6yr9zj7NbhORLiKQPlzGL2yKiHRKumNTRCRggQynKsRFRBIZwQyn6KUQIiIBU09cRKQJS3l2mkfUktYoxEVEklCIi4gELJDrmgpxEZFEsXdCpJbiLT9uLzoKcRGRRJb6cEqmKMRFRJJQiIuIBCv12SmZohAXEUkikAxXiIuIJIo9xDCMFFeIi4gk0oVNEZGwWSAPT1GIi4gkoZ64iEjAAslwhbiISCLDUr5jM1MU4iIiSYQynKLniYuIJIrPTknl06ZqzSaa2WIzqzSzi5JsH2Rms8xsnpm9YWZHt1anQlxEpAOYWS4wFTgKGAFMNrMRCcV+Djzg7qOBk4H/ba1ehbiISBJmqX3aYBxQ6e5L3L0GuB+YlFDGgZ7x772AD1qrtFONiQ/o3YeLvvaNTDcj7ZZ89k6mmxCZwrzumW5CJMqLB2e6CZGp89pMNyHtPM2vZGjnHZulZjanwXKFu1c0WB4ILG+wvALYL6GOXwJPmNk5QDFweGsH7VQhLiLSWbQjxNe4+9gveNjJwO3u/gczOwC4y8z2dvf65nZQiIuINBHJUwxXAuUNlsvi6xr6HjARwN1fMrNCoBRY3VylGhMXEUkUzeyU2cAwMxtiZgXELlxOTyjzPnAYgJntCRQCH7dUqXriIiJJpLsj7u61ZnY2MBPIBW5z9wVmdgUwx92nA+cBN5vZj4ld5PyOu7c44K8QFxFJENWjaN19BjAjYd2lDb4vBManUqdCXEQkiVDu2FSIi4gkoWeniIiEqu038GScQlxEJIHpRckiImHTm31ERAKmnriISMAU4iIiAQskwxXiIiKJzMLpievZKSIiAVNPXESkCU0xFBEJmkJcRCRggWS4QlxEJBn1xEVEAhXS7BSFuIhIEqGEuKYYBmD9us+44IyLOWjw4Rw35hs8/tATScvddeO9fOvg0zhkyBFMGvtN7rrx3kbbz/z6ORyx57FMGHokp0w4g+f+8c+OaL5IkMxS+2SKeuIBuOaiP5BXkM/Mt6bzzlvv8qNTL2TYXrux6x5DG5Vzdy6/8efsNmJXViz9gHNO+jH9B/bjyK8fDsB5vzqXIcMHk5eXx1tzF/CDE3/EQy/fR2n/0kyclkgnFs4Uw8h64mZ2m5mtNrO3ojpGV1C9qZpnHn2OMy/6L4pKihi1/0gO/uqXmTFtZpOyp59zKnvsM5y8vDwG7zaIQyYexOuvvrl9+7C9diMvL/Zz28yora1j1cpmX6It0qVF8KLkSEQ5nHI7MDHC+ruE95csJzcvl112HbR93bC9dmXJ4vda3M/dmffy6wwdPqTR+h+feiHjyw/lOxOnMObAUew5ao9I2i0Ssm0XNkMI8ciGU9z9eTMbHFX9XUXVpmqKS4obrSvpWULVxqoW96u45ja8vp7jJh/daP1191xD7dZaXnl+NkvfWUZOji6LiCQTyGiKLmx2dkXF3dm0cVOjdZs2bKKopKjZfR649SFmTHuc6+79HQXdCppsz8vPY/xhB/Dys6/y3OMvpL3NItkglJ54xkPczKaY2Rwzm7Nx3cZMN6fTGTS0nLraOt5fsnz7uncXVDYZJtlm+r2PcscNdzP1wevpv3O/Fuuuq61j5dKVaW2vSNYIZHpKxkPc3Svcfay7jy3pU5Lp5nQ63Yu785VjDuHPV99C9aZqXn/lDZ57/AWO/uZXm5T9x4NPMPXXFdw47TrKBg9stG3pu8v419Mvsbl6C7Vba5kxbSbzXn6dMQeO6qhTEQlIar3wrBwTl/T56dXnceW5V3HkXsfRq09PLrrmPHbdYyjzXn6dc08+n+eXPgnATb+9mfXr1nPGkd/fvu9RJx7Jxb+/AHfn5t/dxiWLLyUnN5fyoWX8puJy9thneKZOS6Tz0tvuwczuAyYApWa2ArjM3W+N6njZrFefnvz+zquarB+9/8jtAQ7wyJxpzdYxZPfB3P74zZG0TyTbGOHcsRnl7JTJUdUtIhK1UEI842PiIiLSfhoTFxFJIpSeuEJcRCSJQDJcIS4i0kSGpw2mQiEuIpJAs1NERAKnEBcRCZhCXEQkVLpjU0QkbOqJi4gEygJ6PZtCXEQkCYW4iEjAAslwhbiISBOmnriISNgU4iIi4VJPXEQkUAbkhJHhCnERkabCmWKol0KIiARMIS4iksggxyylT5uqNZtoZovNrNLMLmqmzElmttDMFpjZva3VqeEUEZEEUTyK1sxyganAEcAKYLaZTXf3hQ3KDAMuBsa7+zoz69daveqJi4gkkZPipw3GAZXuvsTda4D7gUkJZb4PTHX3dQDuvrot7RQRkQQRDKcMBJY3WF4RX9fQ7sDuZvYvM3vZzCa2VmmnGk7p3a03XxuS+INJOrPp703PdBMiccjATvW/Rlo5nukmpF2d16W1vnYOp5Sa2ZwGyxXuXpFiHXnAMGACUAY8b2ZfcvdPW9pBREQaafvFygbWuPvYFravBMobLJfF1zW0AnjF3bcC75nZO8RCfXZzlWo4RUQkUfzZKal82mA2MMzMhphZAXAykPhP2b8R64VjZqXEhleWtFSpeuIiIgmM9Pdw3b3WzM4GZgK5wG3uvsDMrgDmuPv0+LYjzWwhUAdc4O5rW6pXIS4ikkQ7hlNa5e4zgBkJ6y5t8N2Bn8Q/baIQFxFJIpTb7hXiIiIJYg/AUoiLiAQrjAhXiIuIJNGuKYYZoRAXEUlgpuEUEZGg6cKmiEjAQumJ645NEZGAqScuIpLA0OwUEZGghTKcohAXEWlCUwxFRIJlptkpIiJBC6Un3ursFIv5tpldGl8eZGbjom+aiEjmWIqfTGnLFMP/BQ4AJseXNxB7Y7OISFba9gCsNL9jMxJtGU7Zz93HmNk8AHdfF38rhYhI1gplOKUtIb7VzHIh9nZVM+sL1EfaKhGRjGrzK9cyri0hfgPwV6Cfmf0aOBH4eaStki5h0/pN3PmLe1n00tuU9C7maz86nnHHNP+e2dqttfzqhN+yuWoLv336yu3rl7+9grsuvZcPl3zEgKE7cdoVp1C+R1lHnIJkqShezxaVVtvp7vcAFwJXAR8CX3P3aa3tZ2blZjbLzBaa2QIzO/eLN1eyyX2/eoC8/Fyuee43/OfVZ3DvlX/hg8oPmy3/xG1PU7JDSaN1tVtr+dM5FYw7dizXvng1+x8/jj+dU0Ht1tqomy/ZLJoXJUeiLbNTBgFVwN+JvZl5U3xda2qB89x9BLA/8AMzG/FFGivZY0vVFuY9+TrHn3MshUXd2G3Mroyc8CVe+furScuvWbGGVx+dzcT/OqLR+ndefZe6unoOO+0r5Bfkc+i3J+AOi195pyNOQ7JYNl3YfIzYeLgBhcAQYDGwV0s7ufuHxHruuPsGM1sEDAQWfpEGS3ZYtWw1OXk59B/cb/u6gcMH8u6cyqTl//KbB5l07nHkd2t8Tf2Df39I2e47N+oJle2+Mx9UfsheX1afQdonpNeztWU45Uvuvk/892HAOOClVA5iZoOB0cAr7WmkZJ8tVVvoXlzYaF33HoVs3rS5Sdl5T71OfX09ow8fmaSeGrqXdG+0rrBHdzZv2pLeBkuXkzXDKYnc/TVgv7aWN7MS4CHgR+7+WZLtU8xsjpnNWbvmk1SbI4HqVtSN6oTA3rxxM4UJwb6lagsPX/sIJ118YjP1FDRTT7f0Nli6GCMnxU+mtDqcYmY/abCYA4wBPmhL5WaWTyzA73H3h5OVcfcKoAJg5Jh9vC31Svj679KP+tp6Vi1bTf9dYkMqKxavZOfddmpUbvX7H7P2g7X84fTrAajdWkf1xmouPOQSLrz3PHbedQBP3TELd9/eG1r5zkomTD6oY09Isk42TTHs0eB7LbEx8oda28lifwK3Aovc/dr2NU+yVbeibow+fCR/v/ExTrv8FJa/vZLXZ73JhXf/pFG5nXcbwFVPfT6dcMm8Jdz/m2lcMu2n9OhTQu9+vcjJMZ65+zkO/tZ4XnjwRQCG77d7h56PSKa0GOLxm3x6uPv57ah7PHAa8KaZzY+vu8TdZ7SjLslCk39xEnf+4h4uOOQSinsVc8ovvsXOuw3g3bmV3Hjmn/jj7D+Qm5dLr9Ke2/cp6lWM5eRsX5eTm8OZN3yfuy+7j79dP52dhvbnzBu+T16+nu0m7ZcVL0o2szx3rzWz8e2p2N1fIJyXY0gGFPcq5qwbpjRZP2zf3fjj7D8k3Wf4uGGNbvQBGLRnOZc8cGEkbZSuywKJr5a6K68SG/+eb2bTgWnApm0bmxvjFhHJBtk0Jl4IrAUO5fP54g4oxEUkK1mWvNmnX3xmylt8Ht7baBaJiGQ1C+TpKS2FeC5QQvJxbYW4iGS1bOiJf+juV3RYS0REOpFsGBMP4wxERNLM4r9C0FKIH9ZhrRAR6UyyYZ64u+tBJiLSZWXDcIqISJcUe7NP+LNTRES6qOx6x6aISJejEBcRCVgmnxGeCoW4iEgCQz1xEZFwBTTFMIzLryIikpR64iIiTWTHHZsiIl2SATkWxkCFQlxEJIlQLmyG8aNGRKSDWYq/2lSn2UQzW2xmlWZ2UQvlvmFmbmZjW6tTPXERkSbS/2af+IvnpwJHACuA2WY23d0XJpTrAZwLvNKWetUTFxFJYETSEx8HVLr7EnevAe4HJiUpdyVwNbC5LZUqxEVEksgxS+nTBgOB5Q2WV8TXbWdmY4Byd3+sre3sVMMpuZZHz4I+mW5G2oUyVak9Ttz1m5luQiTmrnk1002IzA1z/5rpJqTdig2r0luhgaU+O6XUzOY0WK5w94o2HzJ2wGuB76Ry0E4V4iIinUO75omvcfeWLkSuBMobLJfF123TA9gbeDY+M2YnYLqZHe/uDX84NKIQFxFJEJsnnvZ/Qc8GhpnZEGLhfTJwyraN7r4eKN3eBrNngfNbCnBQiIuIJJXueeLuXmtmZwMzgVzgNndfYGZXAHPcfXp76lWIi4gkEcWjaN19BjAjYd2lzZSd0JY6FeIiIgn0KFoRkaBZe2anZIRCXEQkCb3ZR0QkUGYaThERCVooN+mFMegjIiJJqScuItKEaThFRCRkurApIhKo2DzxMEabFeIiIk3oRckiIkHTmLiISMDUExcRCZh64iIigTI0O0VEJFymeeIiIkGzQG5oV4iLiCShnrhIF/XZug1cd8H/MPf5efTaoSff/enpHPr1Q5qUm3bTwzw57RlWr/yYnjv04LjTj+abZ56wffuCOYu46Ze3sLxyBTuV9+PsX5/F3uNGdOSpdFmGZqdgZoXA80C3+HEedPfLojqeSGcx9ec3kZefx1/m3cm/F7zHL75zBUNHDGHw8EGNyrk7F1z/Y4buOZgPln3IJadeRt8BpUyYdDCfrdvAZd/9FT+86izGH3UAzz7yPJf955Xc/sLN9OhdkqEz60osihclRyLKQZ8twKHuPhIYBUw0s/0jPJ5Ixm2u2swL/3iJMy44le7F3dl73AgOOGIcTz88q0nZk876BsO+tCu5ebmU71rGAUfux4I5iwBYOPdt+vTrzcHHfpnc3FwOO+Er9NqhF/96/MWOPqUuy1L8lSmRhbjHbIwv5sc/HtXxRDqDFUtWkpubQ9nQgdvXDdlzCMveeb/F/dydt15dwC67N+ite9MyS99uuR5JH4vPUGnrJ1MivfxqZrlmNh9YDTzp7q9EeTyRTKvetJmiHkWN1hX3LKJ6Y3WL+9117X3U1ztHnnQ4ACP2Hc7aVZ8w62/PUbu1lienPc2Hyz5iy+YtkbVdPhcbE89J6ZMpkR7Z3evcfRRQBowzs70Ty5jZFDObY2Zz1qxZE2VzRCLXvbiQqg1VjdZVbaiie0n3Zvd55PZHeeqhZ7jyjksp6JYPQM8+PfnlLT/j4Zsf4VujT2fOs68x+ssjKd1px0jbL+HpkNkp7v6pmc0CJgJvJWyrACoAxuw7RsMtErSyoQOpq6tn5XsfMHDIzgAsWbS08TBJAzPvf5IHpj7E7x+8ir4DShtt2+eAvfmfx64FoK62jjPGf59vTPlatCcgceHc7BNZT9zM+ppZ7/j37sARwNtRHU+kMygsKmT8xAO48/f3sLlqMwtmL+SlJ17hsBO+0qTsM399lv9/zV1cde8VDNhlpybbK9/6N7Vba9m0oYqKX91G351LGTthTEechkCKgymZC/woe+IDgDvMLJfYD4sH3P3RCI8n0imc/eszufb8Gzhp1Gn07NODc359FoOHD+LNVxbw89Mv55HFDwBw++/u5rN1Gzjn2PO273voCRM496r/BmDanx7m1VlzARh7yBguu/mSjj+Zrkpvuwd3fwMYHVX9Ip1Vzz49+OWtP2uy/kv77bU9wAHufPGWFuu5eOoFaW+btI1u9hERCVyX74mLiITL9AAsEZGQhXLbvUJcRCSBxsRFRAKnMXERkWBl9qFWqVCIi4gkoZ64iEigYi9K1uwUEZEw6Y5NEZGQaUxcRCRo6omLiAQslJ54GCP3IiKSlHriIiIJdMemiEjoNCYuIhIqzU4REQmaZqeIiARMPXERkYCFEuKaYigiksCIDaek8mlTvWYTzWyxmVWa2UVJtv/EzBaa2Rtm9rSZ7dJanQpxEZEmLOVfrdZolgtMBY4CRgCTzWxEQrF5wFh33wd4ELimtXoV4iIiSaQ7xIFxQKW7L3H3GuB+YFLDAu4+y92r4osvA2WtVdrpxsRDefxjKkK5yt0eRXklmW5CJA7sf3CmmxCZr975w0w3If3Wbkhvfe17imGpmc1psFzh7hUNlgcCyxssrwD2a6G+7wH/aO2gnS7ERUQ6g3Zc2Fzj7mPTcmyzbwNjgUNaK6sQFxFJsO3CZpqtBMobLJfF1zU+ttnhwM+AQ9x9S2uVZt/YhYjIF5b+C5vAbGCYmQ0xswLgZGB6o6OajQb+DBzv7qvbUql64iIiSaR7nri715rZ2cBMIBe4zd0XmNkVwBx3nw78DigBpsX/JfC+ux/fUr0KcRGRJKKYkODuM4AZCesubfD98FTrVIiLiCShOzZFRCRy6omLiCTQSyFERILW9uehZJpCXEQkKYW4iEiY2nfbfUYoxEVEktCYuIhIwBTiIiKBMl3YFBEJm3riIiIBU4iLiARMwykiIgFTT1xEJFC6sCkiEjj1xEVEgqYQF5FsU1MHc9fC6mooyIG9+0B5SfJyb3wCq6pjy0N6wIg+jctUfhb7bKmDojzYvx/0yI/+HNoojAhXiItIKuZ/EnsLwTHl8GkNvLgKehVAz4LG5d5YB3UOXy2LhfQLq2JBPbhHbPt7G2DpBjgwHtybamM/FDqRUMbEI/9TM7NcM5tnZo9GfSwRiVBtPazcFOtR5+VAaSEMKIL3NzYt+1EVDOsVK1ecD7uUwLJ4OXd4+1PYZ4dY+JtBST4U5Hbs+WSJjuiJnwssAnp2wLFEJCobt0KONR7y6FUAaza3bf/PtsZ+r66LfT7bCnPXxEJ8UDHs2Tv2vdPoTG1pXqQ9cTMrA44BbonyOCLSAWod8hKCLT8n1kNP1L87vLMettbHwn/ZRqiLl6uujf2+qhoO2xkO6g8rNsHSJD36DLIUP5kS9XDK9cCFQJK/ZREJSp7Fgryh2vrYkEmikTtArsETK+Cl1VBWDN3j//DPjUfe7r1iQyjF+bELnx9VR9v+lKQa4ZmL8ciGU8zsWGC1u881swktlJsCTAEoH1QeVXNE5IsqyYd6j/WsS+JDKutroGeSGSUFufAffT9ffmsd9On2eT05dOrRCgvopRBR9sTHA8eb2VLgfuBQM7s7sZC7V7j7WHcfW1paGmFzROQLycuBgUWw8NNYD3ztZvigCgYlmWK4cWtsVop77CLn0g2wR6/P6ykr/ny4pao2NltlQPeOPZ8sEVlP3N0vBi4GiPfEz3f3b0d1PBHpAKN2jF2MfGx5bErg6B1jM0zWbIZ/rYJJu8TKfVoTmye+tR5K8mK98obTEEfuCPPWwD+Wx8bVB/eIzWDpRHTHpohkn4JcOKB/0/WlhZ8HOMR62mXFzdeTnwPj+qW/fWmkEG/A3Z8Fnu2IY4mIdCXqiYuIJKELmyIiEjn1xEVEmjCNiYuIhE0hLiISpEzfSp8KhbiISBKhXNhUiIuIJKUQFxEJVhgRrhAXEWlGGDGueeIiIgFTT1xEpAkL5sKmeuIiIgFTT1xEJEFsnngYPXGFuIhIUgpxEZFghRHhCnERkaRCubCpEBcRaSKcp6coxEVEkggjwjXFUESkGZbipw01mk00s8VmVmlmFyXZ3s3M/hLf/oqZDW6tToW4iEgii42Jp/JptUqzXGAqcBQwAphsZiMSin0PWOfuuwHXAVe3Vq9CXESkY4wDKt19ibvXAPcDkxLKTALuiH9/EDjMWvkJoRAXEUmw7WafVH61wUBgeYPlFfF1Scu4ey2wHtixpUo71YXNea/NW1OUX7Ksgw5XCqzpoGN1JJ1XeLL13DryvHZJZ2WvzZ03s3tecWmKuxWa2ZwGyxXuXpHOdiXTqULc3ft21LHMbI67j+2o43UUnVd4svXcQj4vd58YQbUrgfIGy2XxdTRI3ZAAAALiSURBVMnKrDCzPKAXsLalSjWcIiLSMWYDw8xsiJkVACcD0xPKTAfOiH8/EXjG3b2lSjtVT1xEJFu5e62ZnQ3MBHKB29x9gZldAcxx9+nArcBdZlYJfEIs6FvUlUM88rGqDMnoeZlZHfAmsf+2FgFnuHtVO+u6HXjU3R8Eqs1shLsvbKbsBKDG3V9M8RhLgbHunskxaf232EW4+wxgRsK6Sxt83wx8M5U6u+xwSkdccMiETnBe1e4+yt33BmqAMxtujI/zpczdD2ouwOMmAAe2p+5M6wR/Z5HI1vPqbLpsiEuH+Cewm5lNMLN/mtl0YKGZ5ZrZ78xstpm9YWb/D8Bibozf0fYU0G9bRWb2rJmNjX+faGavmdnrZvZ0/K62M4Efm9l8MzvIzPqa2UPxY8w2s/HxfXc0syfMbIGZ3UI4d1eLJNWVh1MkQvEe91HA4/FVY4C93f09M5sCrHf3/zCzbsC/zOwJYDQwnNjdbP2BhcBtCfX2BW4GDo7XtYO7f2JmNwEb3f338XL3Ate5+wtmNojYOOSewGXAC+5+hZkdQ+wOOZFgKcQl3bqb2fz4938Su1BzIPCqu78XX38ksI+ZnRhf7gUMAw4G7nP3OuADM3smSf37A89vq8vdP2mmHYcDIxrc7NbTzErixzghvu9jZraunecp0ikoxCXdqt19VMMV8SDd1HAVcI67z0wod3Qa25ED7B+/UJTYFpGsoTFxyYSZwFlmlg9gZrubWTHwPPCt+Jj5AOArSfZ9GTjYzIbE990hvn4D0KNBuSeAc7YtmNm2HyzPA6fE1x0F9EnbWYlkgEJcMuEWYuPdr5nZW8Cfif2r8K/Au/FtdwIvJe7o7h8DU4CHzex14C/xTX8Hvr7twibwQ2Bs/MLpQj6fJXM5sR8CC4gNq7wf0TmKdAhr5WYgERHpxNQTFxEJmEJcRCRgCnERkYApxEVEAqYQFxEJmEJcRCRgCnERkYApxEVEAvZ/ccjvihS6Kw8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fmQWrsfQ4EgG",
        "outputId": "611e951f-57d5-43ba-d8c1-a097cffb1fe4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "score_groups = train.groupby(\"Score\")\n",
        "print(score_groups.size())\n",
        "score_balanced_predictor, score_balanced_score, score_balanced_predictions, score_balanced_true_labels = train_and_score(*train_test_validate_split_groups(score_groups, sample_size_group=12000), model_name=\"score_balanced_real\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Score\n",
            "1     25354\n",
            "2     14643\n",
            "3     20948\n",
            "4     39145\n",
            "5    175479\n",
            "dtype: int64\n",
            "train size: 48,000\n",
            "test size: 12,000\n",
            "validate size: 215,569\n",
            "[0, 1, 2, 3, 4]\n",
            "Validation score: 0.7439752469047034\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LuD4ApvwPlFI",
        "outputId": "6295ddf9-fc39-44d3-85f9-f5f35c0b8645",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "score_balanced_predictions = score_balanced_predictor.predict(test.TextClean.values)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yLsgMmWoBg8z",
        "outputId": "1c4970f1-ef7d-41d5-8a8b-662f893ebc5d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "cm2 = confusion_matrix_custom(true_labels, score_balanced_predictions, \"score_balanced_matrix.pdf\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAE9CAYAAAD9HVKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xVdb3/8ddn77lfuA43uasTiqiACJRZ3gMzyHM8haf65e/UQVM7VnZKs+yX1a+sfnUqPXaoOFpa1rGLqCha4sE8qSAgcRFDRG4KDAIKM8wwM5/fH3szzezZc9m41+z57nk/eezHY9Za3/39fpfie75+13etZe6OiIiEKZbrDoiIyLFTiIuIBEwhLiISMIW4iEjAFOIiIgFTiIuIBKzPhbiZzTKzjWa2ycxuyHV/ssXMFprZbjNbm+u+ZJOZjTazpWa23szWmdl1ue5TNphZiZk9a2bPJ8/rK7nuU7aZWdzMVpnZg7nuSz7rUyFuZnHgdmA2MBG43Mwm5rZXWXMnMCvXnYhAI3C9u08EZgLX5Mm/s3rgPHc/HZgMzDKzmTnuU7ZdB2zIdSfyXZ8KcWA6sMndN7t7A3AvMDfHfcoKd18GvJ7rfmSbu7/q7iuTP79JIhRG5rZXb50nHExuFiY/eXPnnZmNAt4L/CTXfcl3fS3ERwLbWm1vJw8Coa8ws3HAFOCZ3PYkO5LTDauB3cBj7p4X55X0b8DngOZcdyTf9bUQl0CZWQXwG+BT7v5GrvuTDe7e5O6TgVHAdDOblOs+ZYOZXQLsdvfnct2XvqCvhfgOYHSr7VHJfdKLmVkhiQC/x91/m+v+ZJu77weWkj/XNM4C5pjZFhJTlueZ2d257VL+6mshvhyoNrPxZlYEzAMW5bhP0gkzM+CnwAZ3/26u+5MtZjbEzAYkfy4FLgReyG2vssPdb3T3Ue4+jsR/Y4+7+4dz3K281adC3N0bgWuBJSQukP3a3dfltlfZYWa/BP4MTDCz7Wb2sVz3KUvOAj5CYjS3Ovm5ONedyoIRwFIzW0NicPGYu2spnmTM9ChaEZFw9amRuIhIvlGIi4gETCEuIhIwhbiISMAU4iIiPaSrB9VZwg+SD+hbY2ZTu6qzz4a4mc3PdR+ioPMKT76eW76e11t0J53f1DUbqE5+5gN3dFVhnw1xEv+A8pHOKzz5em75el7HrBsPqpsL/Cz5gLSngQFmNqKzOvtyiIuI9DYZP6SvINLuZChWXugFA0p6pK14/2KKRlb2yJ1OJw0b1xPNADBy9HGcNvXUHruDK27xHmln1JhRTDljco+dl/Xg+Gb0mNFMPWNqD9511zNNJc5rSo80tvWVrdTU7LVs1WdVJU5Dhg9gfPPIOuBwqz0L3H1BtvrUkV4V4gUDShh69Rm57kbWPXhd/j5SuV/RwFx3IRJFseJcdyEy7vn3dNizZ747uxU2NMOMoZl95w87Drv7tLfYcsYP6dN0iohIOmaZfbJjEfC/kqtUZgIH3P3Vzr7Qq0biIiK9ghHJEDf5oLpzgCoz2w58mcRbnXD3HwGLgYuBTUAt8L+7qlMhLiKSTvZG1y3c/fIujjtwTSZ1KsRFRNLJfoZHQiEuItJOVue5I6UQFxFJFdGceBQU4iIi6WgkLiISsDAyXCEuItKOAbEwUlwhLiKSThgZrhAXEUlLc+IiIgELI8MV4iIi7WhOXEQkcGFkuEJcRKQ93bEpIhKugKZTArmxVERE0tFIXEQknTAG4gpxEZG0NCcuIhKwMDI8f0K8ufYI+363kfpN+4iVFdLvovGUnT6sXbmau9bQ8MqBlm1vcgqqyhj2yWk07j/M7h8sb1PeG5rpN+t4Kt85OrUqEclXAV3YjDTEzWwW8H0gDvzE3b8ZVVv7H/grFo8x/IZ3cOTVg+z9+V8oHF5B4bDyNuWqPnpam+09P1lN8fEDACgYUMJxN5/dcqzx9Tp2fe9ZSk8ZElW3RaS3CiPDo1udYmZx4HZgNjARuNzMJkbRVnNDE3Xra6i8YByx4jjF4/pTctJgalfv6vR7jfsO0/DKAcqmDE97vHb1LorG9adgYEkU3RaR3iw3b7vPWJRLDKcDm9x9s7s3APcCc6NoqLGmFosZhVVlLfsKR1TQuPtQp9+rXfUaRWPTh7S7U7tqV4cBLyJ5LpbhJ0eibHoksK3V9vbkvqzzhiasON5mX6y4gOb6pk6/V7t6F2VT04d0wysHaD7UoKkUkb4o01F4DkfiOb+waWbzgfkA8f7Fx1ZHURxPCezm+kZiKcHeWv2WAzQf7Dika1ftomTikE7rEJE81tfnxIEdQOslHaOS+9pw9wXuPs3dp8XKC4+poYKqMrzZaaypbdl35LVDFAwt7/A7tate6zCk/UgTdWv3UD6l/eoWEekjAhmJRxniy4FqMxtvZkXAPGBRFA3FiuKUTqzijT9uobmhifpXDnB4Qw1lk9OHcFchXbe+hlhpAUXJVSsi0gf19Tlxd28ErgWWABuAX7v7uqjaG/C+aryxmde+8T/s+/UGBsyppnBYOfVb9rPzlifblK1bv5dYScchXbtqF2WTh2GB3LElIllmBDMSj3RO3N0XA4ujbOOoWFkhgz80qd3+4nED2qz9Big7fShlpw/tsK6qK07r8JiI9BGBjOFyfmFTRKRX0h2bIiIBC2Q6VSEuIpLKCGY6RS+FEBEJmEbiIiLtWMar0zyinnRFIS4ikoZCXEQkYIFc11SIi4ikSrwTIrMU7/xxe9FRiIuIpLLMp1NyRSEuIpKGQlxEJFiZr07JFYW4iEgagWS4QlxEJFXiIYZhpLhCXEQklS5sioiEzQJ5eIpCXEQkDY3ERUQCFkiGK8RFRFIZlvEdm7miEBcRSSOU6RQ9T1xEJFVydUomn25VazbLzDaa2SYzuyHN8TFmttTMVpnZGjO7uKs6FeIiIj3AzOLA7cBsYCJwuZlNTCn2ReDX7j4FmAf8e1f1KsRFRNIwy+zTDdOBTe6+2d0bgHuBuSllHOiX/Lk/sLOrSnvVnPjoQUP45ryP57obWbdk26O57kJk3j58Zq67EImxFcfnuguRCWX9cy4d4x2bVWa2otX2Andf0Gp7JLCt1fZ2YEZKHf8HeNTMPgmUAxd01WivCnERkd7iGEK8xt2nvcVmLwfudPf/Z2ZvB35uZpPcvbmjLyjERUTaieQphjuA0a22RyX3tfYxYBaAu//ZzEqAKmB3R5VqTlxEJFU0q1OWA9VmNt7MikhcuFyUUmYrcD6AmZ0MlAB7OqtUI3ERkTSyPRB390YzuxZYAsSBhe6+zsxuAVa4+yLgeuDHZvZpEhc5r3D3Tt/BrBAXEUkR1aNo3X0xsDhl382tfl4PnJVJnQpxEZE0QrljUyEuIpKGnp0iIhKq7t/Ak3MKcRGRFKYXJYuIhC2UO1sV4iIiaWgkLiISMIW4iEjAAslwhbiISCqzcEbienaKiEjANBIXEWlHSwxFRIKmEBcRCVggGa4QFxFJRyNxEZFAhbQ6RSEuIpKGQlyy5tD+Q9z1pXtY/z8bqBhQzqWfnsuMS85sV+6xux5n6T1PcHDfIYrLipk2eyqXffZS4gVxAF5atZlfffM+Xn3pNapGDeYfv/RBqs84sadPRyQIgWS4QjwEv/jarygojPOdZd9g2wvb+eEn7mD0hJEcV31cm3Knn3sqZ106k7J+ZRzaf4gfffonPH73E1x4xfkc2n+I267+ER/68jymXjiZZx9awe3X/IivL7mF8v5lOTozkd4qnCWGkd3sY2YLzWy3ma2Nqo2+oL62npWPrmbuv1xCSXkJ1WecyOnnnsrTDzzbruzQMUMo65cIZMcxM3ZvTbxj9aXVm+lXVcm0WVOJxWPMnDOdioGVrPrD6h49H5FQRPCi5EhEORK/E7gN+FmEbeS9XVt2EyuIMWzcsJZ9oyeM4sUVf01b/pkHl3PPV+7l8KHDVAys4B8+93d/O5j6ulV3dvx1ZwS9FgmbLmwC7r7MzMZFVX9fUV9bT2l5SZt9pZWlHD5Un7b8jEvOZMYlZ7Jry26eXvQM/QZXAnD85PHs33OAZx9awdSLpvDsQ8vZs62GhrqGyM9BJESBZLiendLbFZcVU3focJt9dQfrKCkv7vR7w8YNZcSJI/jFV38FQMWACq6+7Uoeu+uPfPbsG1j3p/Wc9PYJDBw+MLK+i4RM0yndZGbzgfkAVccNznFvep9h44bS3NjMri27GTZuKADbN+7guBNHdPnd5sZm9myradmecGY1N/368wA0NTbxhYu+zEVXnB9Nx0VCF8hQPOcjcXdf4O7T3H1av0H9ct2dXqe4rJgpF05m0W0PUl9bz6aVL7H68TXMfN/0dmWfvO8p3tj7JgA7N73Kwz9ewkkzJ7Qc37p+G41Hmqg7WMd93/4dg0YM5JR3TuyxcxEJR2aj8D49EpeufehLH+TOL97N9WffQHn/cj508zyOqz6Ov67YxA+uvJ0fPvc9AF5auZnff/8B6mvrqRxYwRnvmcLcf3lfSz1LFj7GX5atA2DSOyfyiR/Mz8n5iPR6ets9mNkvgXOAKjPbDnzZ3X8aVXv5rHxAOdfcdmW7/dXTTmwJcIAr/u9HOq3nn7/zT1nvm0g+MrQ6BXe/PKq6RUSiFkqI53xOXEREjp3mxEVE0ghlJK4QFxFJI5AMV4iLiLST42WDmVCIi4ik0OoUEZHAKcRFRAKmEBcRCZXu2BQRCZtG4iIigbKAXs+mEBcRSUMhLiISsEAyXCEuItKO3rEpIhI4hbiISLg0EhcRCZQBsTAyXCEuItJeOEsM9VIIEZGAKcRFRFIZxMwy+nSrWrNZZrbRzDaZ2Q0dlPmAma03s3Vm9ouu6tR0iohIiigeRWtmceB24EJgO7DczBa5+/pWZaqBG4Gz3H2fmQ3tql6NxEVE0ohl+OmG6cAmd9/s7g3AvcDclDL/DNzu7vsA3H13d/opIiIpIphOGQlsa7W9PbmvtbcBbzOzp8zsaTOb1VWlvWo6paKwgneOeFeuu5F1NYe7/GUarN9uejDXXYjEhyd8INddiMyg4qpcdyHrHM9qfcc4nVJlZitabS9w9wUZ1lEAVAPnAKOAZWZ2qrvv7+wLIiLSRvcvVrZS4+7TOjm+AxjdantUcl9r24Fn3P0I8LKZvUgi1Jd3VKmmU0REUiWfnZLJpxuWA9VmNt7MioB5wKKUMr8nMQrHzKpITK9s7qxSjcRFRFIY2R/hunujmV0LLAHiwEJ3X2dmtwAr3H1R8thFZrYeaAL+1d33dlavQlxEJI1jmE7pkrsvBhan7Lu51c8OfCb56RaFuIhIGqHcdq8QFxFJkXgAlkJcRCRYYUS4QlxEJI1jWmKYEwpxEZEUZppOEREJmi5siogELJSRuO7YFBEJmEbiIiIpDK1OEREJWijTKQpxEZF2tMRQRCRYZlqdIiIStFBG4l2uTrGED5vZzcntMWY2PfquiYjkjmX4yZXuLDH8d+DtwOXJ7TdJvLFZRCQvHX0AVpbfsRmJ7kynzHD3qWa2CsDd9yXfSiEikrdCmU7pTogfMbM4JN5EamZDgOZIeyUiklPdfuVaznUnxH8A/A4YamZfBy4Dvhhpr6RPqD1Qy/1fu5+XnnmJsgFlXHD1BZw267QOyzceaeSOD91BQ20D1z94fcv+5qZmli5YyqoHVlFfW8+gUYO44o4rKK0s7YnTkDwUxevZotJliLv7PWb2HHA+iXN7v7tv6Op7ZjYa+BkwjMQofoG7f/8t9lfyyEPffoh4YZx/feRfee3F17jn0/cwvHo4Q08Ymrb8Uz9/ivIB5TTUNrTZv3TBUrat2cbHf/px+g/vz+7Nuyko0sIreQsCWmLYndUpY4Ba4AESb2Y+lNzXlUbgenefCMwErjGziW+ls5I/Guoa2PD4Bs678jyKy4oZO3ksE941gecffj5t+X079rHmkTWcfcXZbfbXvVHH0/c+zZyb5jBgxADMjGEnDKOwuLAnTkPyWD5d2HyIxEjagBJgPLAROKWzL7n7q8CryZ/fNLMNwEhg/VvpsOSHvVv3EovHqBpb1bJvePVwtqzckrb84u8s5oJPXEBBcdu/srs27SIWj7Huj+t4+pdPU1xezIx5M5jxDzOi7L7kubx6PZu7n9p628ymAldn0oiZjQOmAM9k8j3JXw21DRSXF7fZV1xR3G6qBGDD0g00Nzdz8rkn8/JzL7c59sbuNzh88DB7t+7lU7//FHu37eWua+6iakwVJ8w4IdJzkPyWN9Mpqdx9JdDtYY6ZVQC/AT7l7m+kOT7fzFaY2Yq9Na9n2h0JVFFZEfWH6tvsqz9UT1FZ29WrDXUNPHrbo1x8/cVp6zk6bXLOx8+hsKSQ4dXDmXThJF586sVoOi59hBHL8JMrXY7EzewzrTZjwFRgZ3cqN7NCEgF+j7v/Nl0Zd18ALAA4fepp3p16JXyDxwymuamZvVv3MnjMYAB2vbiLoce3vai5d9te9u/cz8L5CwFoamzi8MHDfHvWt/n4wo8zrHpYomCr/4ZCGUFJ7xbK36PujMQrW32KScyRz+3qS5b4J/BTYIO7f/etdFLyT1FpESefezKPL3ichroGtj6/lReWvcDps09vU27o8UP5zAOf4aq7r+Kqu69izk1zqBhUwVV3X0X/Yf0ZNGoQYyePZdl/LqOxoZE9L+9h7WNrmfDOCTk6M5Ge1elIPHmTT6W7f/YY6j4L+AjwFzNbndz3BXdffAx1SR567+fey/1fvZ9vvedblPUv45LPX8LQE4byyqpXuPtTd3PTf99EvCBOZVVly3dK+5ViMWuz77KvXcb9X7ufWy+8lfKB5Zx75bkcP/34XJyS5Im8eFGymRW4e6OZnXUsFbv7nwjn5RiSA2X9y7j8O5e32z92ylhu+u+b0n5n/Bnj29zoA9BvaD8+8oOPRNJH6bsskPjqbCT+LIn579Vmtgj4L+DQ0YMdzXGLiOSDUObEu7NOvATYC5zH39aLO6AQF5G8ZHnyZp+hyZUpa/lbeB+lVSQiktcskKendBbicaCC9PPaCnERyWv5MBJ/1d1v6bGeiIj0IvkwJx7GGYiIZJkl/4SgsxA/v8d6ISLSm+TDOnF314NMRKTPyofpFBGRPinxZp/wV6eIiPRR+fWOTRGRPkchLiISsFw+IzwTCnERkRSGRuIiIuEKaIlhGJdfRUQkLY3ERUTayY87NkVE+iQDYhbGRIVCXEQkjVAubIbxq0ZEpIdZhn+6VafZLDPbaGabzOyGTsr9vZm5mU3rqk6NxEVE2sn+m32SL56/HbgQ2A4sN7NF7r4+pVwlcB3wTHfq1UhcRCSFEclIfDqwyd03u3sDcC8wN025rwK3Aoe7U6lCXEQkjZhZRp9uGAlsa7W9PbmvhZlNBUa7+0Pd7Wevmk6JWZzywn657kbW9SsckOsuRObDEypy3YVI3PfS73PdhciM7zcq113Iujca3shuhQaW+eqUKjNb0Wp7gbsv6HaTiQa/C1yRSaO9KsRFRHqHY1onXuPunV2I3AGMbrU9KrnvqEpgEvBEcmXMcGCRmc1x99a/HNpQiIuIpEisE8/6EsPlQLWZjScR3vOAfzx60N0PAFUtfTB7AvhsZwEOCnERkbSyvU7c3RvN7FpgCRAHFrr7OjO7BVjh7ouOpV6FuIhIGlE8itbdFwOLU/bd3EHZc7pTp0JcRCSFHkUrIhI0O5bVKTmhEBcRSUNv9hERCZSZplNERIIWyvPEw5j0ERGRtDQSFxFpxzSdIiISMl3YFBEJVGKdeBizzQpxEZF29KJkEZGgaU5cRCRgGomLiARMI3ERkUAZWp0iIhIu0zpxEZGgWSA3tCvERUTSCGUkHsavGpGA1B6o5WfX/5wvvuNmvnHxrax6eHWn5RuPNPKdv/suX5/1jbTHn3twJZ+feiPP/m55FN2VNIyjK8W7/ydXIhuJm1kJsAwoTrZzn7t/Oar2RHqL33/zfuIFcb70h5vYufFV/vO6OxnxthEMP2FY2vLL7nqSioHl1NfWtztW+0YdSxc+wbAOvitRsShelByJKEfi9cB57n46MBmYZWYzI2xPJOca6hpY+8d1XHT1hRSXFTN+yjgmvutkVj20Km3513e8zsrFqzjnn85Je/yRHz7CWfPeTvmAsug6LWmFMhKPLMQ94WByszD58ajaE+kN9rxSQ6wgxpCxQ1r2jXjbCHa9tCtt+ftvXcSsa99DYXFhu2Pb1m5j+/odzLhsRmT9lY5ZcoVKdz+5EumcuJnFzWw1sBt4zN2fibI9kVxrqK2nuLy4zb6SipK0UyVrH19Hc7Mz6bxT2h1rbmrmd9+4n7mfn0MspktXPS0xJx7L6JMrkbbs7k3uPhkYBUw3s0mpZcxsvpmtMLMVe2tqouyOSOSKyoqpP9Q2sOsP1VNc1jbYG+oaWPz9h5n7ufelrefP//U0I6qHM/a0MZH1VfJDjywxdPf9ZrYUmAWsTTm2AFgAMOWMKZpukaANGVtFc2MzNVtrqBpTBcCrL77a7sJkzdYa9r26jzs+9h8ANB1p4vDBw3z1wq9zzV1Xs+nZl3j5uc288NRGAOoO1LFj4052btzJ+2+Y27Mn1SfpZh/MbAhwJBngpcCFwK1RtSfSGxSVFnHKeafw6B2PcdnNf8/OjTtZ99/rufo/P9Gm3LAThvGFxTe0bL/y/Cv8/tZFXPeLT1I+sJwPfOUyGusbW47/7LN3c+oFkzhz7pk9di59XSi33Uc5nTICWGpma4DlJObEH4ywPZFe4dIb53KkvpFbzv8av/jCvVx64/sZfsIwXl75Ml86K7HKNl4Qp7KqsuVT2r8MixmVVZXE4jFKK0vbHC8ojFNSXkJpZUmOz66PsHAubEY2Enf3NcCUqOoX6a3K+pfx0e9+pN3+8VPH89WnvpL2OydMO56bHrmxwzqv/PH8rPVPunb0Zp8Q6LZ7EZE0+vycuIhIuEwPwBIRCVkot90rxEVEUmhOXEQkcJoTFxEJVm4fapUJhbiISBoaiYuIBCrxomStThERCZNpJC4iEjDNiYuIBE0jcRGRgIUyEg9j5l5ERNLSSFxEJIXu2BQRCZ3mxEVEQqXVKSIiQdPqFBGRgGkkLiISsFBCXEsMRURSGNG8KNnMZpnZRjPbZGY3pDn+GTNbb2ZrzOyPZja2qzoV4iIi7VjGf7qs0SwO3A7MBiYCl5vZxJRiq4Bp7n4acB/wra7qVYiLiKSR7RAHpgOb3H2zuzcA9wJzWxdw96XuXpvcfBoY1VWlvWpOPPH4xzDmoTIRs3iuuxCZoaXH5boLkfjQ2+bluguROfWbl+e6C1n3xt7d2a3w2J5iWGVmK1ptL3D3Ba22RwLbWm1vB2Z0Ut/HgIe7arRXhbiISG9xDBc2a9x9WlbaNvswMA14d1dlFeIiIimOXtjMsh3A6Fbbo5L72rZtdgFwE/Bud6/vqlLNiYuItJP9C5vAcqDazMabWREwD1jUplWzKcB/AHPcvVtzRBqJi4ikke114u7eaGbXAkuAOLDQ3deZ2S3ACndfBHwbqAD+K/l/AlvdfU5n9SrERUTSiOK2e3dfDCxO2Xdzq58vyLROhbiISBq6Y1NERCKnkbiISAq9FEJEJGjdfx5KrinERUTSUoiLiITp2G67zwmFuIhIGpoTFxEJmEJcRCRQpgubIiJh00hcRCRgCnERkYBpOkVEJGAaiYuIBEoXNkVEAqeRuIhI0BTiIpJnmusaqX3wJY5sPkCstIDS88ZQNKmqXTlvbKb20S0c2bgPmpopGF1J2ezjifUrAuDw8tdoWLOHpt21FJ0ymPI5J/b0qXQpjAjX88RFJAO1D78McWPAp8+g/P0ncujhl2naU9uuXP2zr9G0/SD9/vlU+n/qDKykgNolL7ccj1UWUfLOkRSdPqQnu58RM8vokyuRh7iZxc1slZk9GHVbIhIdb2jiyAuvU/ru0VhRnIIx/SiqHkj9X2ralW3af5iCE/oTqyjCCmIUTRxM0566luNFJw2iaMIgYqWFPXkKeaknRuLXARt6oB0RiVDT64chZsQHl7bsiw8roznNSLx48lAat71J85sN+JEm6tfWUHjigJ7sbhZYhp/ciHRO3MxGAe8Fvg58Jsq2RCRa3tCEFcfb7LPiOF7f1K5sfFAJsX7FHPj+SjCIDy2jbNb4nupqVmhOPOHfgM8BzRG3IyIRs6L2gZ0u2AFqH9kCTc30v34aAz4/ncKTBnHwlyH9D3mmo/A8nBM3s0uA3e7+XBfl5pvZCjNbUVOzN6ruiMhbFB9UAs1O0+t/m9tu2lVLbEhZu7KNuw5RdNoQYqUFWEGM4jOH07TzEM21R3qyy8fMTBc2Ac4C5pjZFuBe4Dwzuzu1kLsvcPdp7j6tqmpwhN0RkbfCiuIUnjSIuie24w1NNG57k4YX91F8avslhgUjKmj4Sw1+uBFvaqZ+xS6sspBYWeJCpjc73tgM7tCcWJLozd7Tp5QXIpsTd/cbgRsBzOwc4LPu/uGo2hOR6JXNHk/tAy+x/3vPYaUFlM8eT3xIGUe2vsHBX77AwM9PB6D0gjHULdnCgX9fDU1ObEgZFZdNaKnn8JPbOfzkjpbthrU1lJw9ktJ3j+7xc+qI7tgUkbwTKy2g4gMT2u0vHNOvJcABYmWFlF9a3WE9pe8e3asCOx2FeCvu/gTwRE+0JSLSl2gkLiKSRihPMdRt9yIiAdNIXESkHdOcuIhI2BTiIiJByu09mJlRiIuIpBHKhU2FuIhIWgpxEZFghRHhCnERkQ6EEeNaJy4iEjCNxEVE2snt42UzoZG4iEjANBIXEUmRWCcexkhcIS4ikpZCXEQkWGFEuEJcRCStUC5sKsRFRNoJ5+kpCnERkTTCiHAtMRQR6YBl+OlGjWazzGyjmW0ysxvSHC82s18ljz9jZuO6qlMhLiKSyhJz4pl8uqzSLA7cDswGJgKXm9nElGIfA/a5+4nA94Bbu6pXIS4i0jOmA5vcfbO7NwD3AnNTyswF7kr+fB9wvnXxG0IhLiKS4ujNPpn86YaRwLZW29uT+9KWcfdG4AAwuLNKe9WFzVUrV9dUFg14pYeaqwJqeqitnqTzCk++nltPntfYbFa28rlVS0oLyqsy/FqJma1otb3A3Rdks1/p9KoQd/chPdWWmZKstssAAAMDSURBVK1w92k91V5P0XmFJ1/PLeTzcvdZEVS7AxjdantUcl+6MtvNrADoD+ztrFJNp4iI9IzlQLWZjTezImAesCilzCLgo8mfLwMed3fvrNJeNRIXEclX7t5oZtcCS4A4sNDd15nZLcAKd18E/BT4uZltAl4nEfSd6sshHvlcVY7k9LzMrAn4C4m/WxuAj7p77THWdSfwoLvfB9SZ2UR3X99B2XOABnf/nwzb2AJMc/dczknr72If4e6LgcUp+25u9fNh4B8yqbPPTqf0xAWHXOgF51Xn7pPdfRLQAFzV+mByni9j7n52RwGedA7wjmOpO9d6wb+zSOTrefU2fTbEpUc8CZxoZueY2ZNmtghYb2ZxM/u2mS03szVmdiWAJdyWvKPtD8DQoxWZ2RNmNi358ywzW2lmz5vZH5N3tV0FfNrMVpvZ2WY2xMx+k2xjuZmdlfzuYDN71MzWmdlPCOfuapG0+vJ0ikQoOeKeDTyS3DUVmOTuL5vZfOCAu59pZsXAU2b2KDAFmEDibrZhwHpgYUq9Q4AfA+9K1jXI3V83sx8BB939O8lyvwC+5+5/MrMxJOYhTwa+DPzJ3W8xs/eSuENOJFgKccm2UjNbnfz5SRIXat4BPOvuLyf3XwScZmaXJbf7A9XAu4BfunsTsNPMHk9T/0xg2dG63P31DvpxATCx1c1u/cysItnG3yW/+5CZ7TvG8xTpFRTikm117j659Y5kkB5qvQv4pLsvSSl3cRb7EQNmJi8UpfZFJG9oTlxyYQnwCTMrBDCzt5lZObAM+GByznwEcG6a7z4NvMvMxie/Oyi5/02gslW5R4FPHt0ws6O/WJYB/5jcNxsYmLWzEskBhbjkwk9IzHevNLO1wH+Q+L/C3wF/TR77GfDn1C+6+x5gPvBbM3se+FXy0APApUcvbAL/AkxLXjhdz99WyXyFxC+BdSSmVbZGdI4iPcK6uBlIRER6MY3ERUQCphAXEQmYQlxEJGAKcRGRgCnERUQCphAXEQmYQlxEJGAKcRGRgP1/BBMCbR1DTGEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EPqdHoEHHVxz",
        "outputId": "11199ef8-5dbf-4c2b-946e-0be4d770a65e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        }
      },
      "source": [
        "score_helpfulness_groups = train.groupby([\"Score\", \"Helpfulness\"])\n",
        "print(score_helpfulness_groups.size())\n",
        "score_helpfulness_balanced_predictor, score_helpfulness_balanced_score, score_helpfulness_balanced_predictions, score_helpfulness_balanced_true_labels = train_and_score(*train_test_validate_split_groups(score_helpfulness_groups, sample_size_group=3500), model_name=\"score_helpfulness_balanced_real\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Score  Helpfulness\n",
            "1      -1              6385\n",
            "        0              9359\n",
            "        1              9610\n",
            "2      -1              5302\n",
            "        0              4242\n",
            "        1              5099\n",
            "3      -1              9261\n",
            "        0              4637\n",
            "        1              7050\n",
            "4      -1             20204\n",
            "        0              4030\n",
            "        1             14911\n",
            "5      -1             88220\n",
            "        0             11342\n",
            "        1             75917\n",
            "dtype: int64\n",
            "train size: 42,000\n",
            "test size: 10,500\n",
            "validate size: 223,069\n",
            "[0, 1, 2, 3, 4]\n",
            "Validation score: 0.712331162106792\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AfgBfnpXPwX7",
        "outputId": "d53c8bd6-68cd-4a06-b6c8-44119490aea1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "score_helpfulness_balanced_predictions = score_helpfulness_balanced_predictor.predict(test.TextClean.values)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HNB_qp_7B0PF",
        "outputId": "3e4bb41b-573d-4a4e-e072-5398e57e6109",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "cm3 = confusion_matrix_custom(true_labels, score_helpfulness_balanced_predictions, \"score_helpfulness_balanced_matrix.pdf\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAE9CAYAAAD9HVKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhc1X3/8fdXo93yLnlBkjdsFrF4xTiQAGGLWYJJSVJIyVYamoStCSEPNH2g0CclabPRH6SJA4SSpGEndcDBJARKQoNjY5vFBoPAm4yxLW9gWdb6/f0xY0UajSWNmavRGX1eeuZh7pkz555rw0eHc8+919wdEREJU162OyAiIodOIS4iEjCFuIhIwBTiIiIBU4iLiARMIS4iErBBF+JmNt/M1ppZrZldn+3+ZIqZ3W1m28zslWz3JZPMrNrMnjazNWa22syuyXafMsHMis3sz2b2YuK4bs52nzLNzGJmttLMHst2X3LZoApxM4sBdwDnADXAJWZWk91eZcw9wPxsdyICrcC17l4DzAOuyJG/sybgdHefDswA5pvZvCz3KdOuAV7Ndidy3aAKcWAuUOvub7l7M3AfsCDLfcoId38W2JntfmSau29x9xWJ9+8RD4XK7Pbq/fO4vYnNgsQrZ668M7Mq4Dzgzmz3JdcNthCvBDZ12q4jBwJhsDCzScBMYGl2e5IZiemGVcA24LfunhPHlfAD4OtAe7Y7kusGW4hLoMysDHgY+Ad3fzfb/ckEd29z9xlAFTDXzI7Ndp8ywczOB7a5+wvZ7stgMNhCfDNQ3Wm7KlEmA5iZFRAP8F+4+yPZ7k+muftu4Gly55zGycAFZrae+JTl6Wb28+x2KXcNthBfBkwzs8lmVghcDCzKcp+kB2ZmwF3Aq+7+vWz3J1PMrMLMRiTelwBnAa9lt1eZ4e43uHuVu08i/t/Y79390ix3K2cNqhB391bgSmAJ8RNkD7j76uz2KjPM7JfAn4AjzazOzC7Ldp8y5GTg08RHc6sSr3Oz3akMGA88bWYvER9c/NbdtRRP0ma6Fa2ISLgG1UhcRCTXKMRFRAKmEBcRCZhCXEQkYApxEZF+0tuN6izuPxI36HvJzGb11uagDXEzuzzbfYiCjis8uXpsuXpc79M99HxR1znAtMTrcuA/e2tw0IY48T+gXKTjCk+uHluuHtch68ON6hYA9yZukPY8MMLMxvfU5mAOcRGRgSbtm/TlR9qdNOWXFXrh6NJ+2VfBqBJKJ47olyudJo3s8RdpRo2rGkfNjKP77QquolhRv+ynekIVM2fP6Lfjsn4c31RPqGbW7Fn9eNVd/+wqflwz+2VnGzdspL5+h2WqPSsvdprTvAHjey2rgf2dSha6+8JM9elgBlSIF44u5Yh/PDXb3ci4n150Q7a7EJkpw47IdhciUZBXmO0uRMY99+4O+6F5Gc6N5nY4cUx63/nd5v3uPud97jntm/RpOkVEJBWz9F6ZsQj4TGKVyjxgj7tv6ekLA2okLiIyIBiRDHETN6o7DSg3szrgJuJPdcLdfwQsBs4FaoF9wOd7a1MhLiKSSuZG1x3c/ZJePnfginTaVIiLiKSS+QyPhEJcRKSbjM5zR0ohLiKSLKI58SgoxEVEUtFIXEQkYGFkuEJcRKQbA/LCSHGFuIhIKmFkuEJcRCQlzYmLiAQsjAxXiIuIdKM5cRGRwIWR4QpxEZHudMWmiEi4AppOCeTCUhERSUUjcRGRVMIYiCvERURS0py4iEjAwsjw3Anx1oZmNt27ir2vbidWVsj4C49m5NyqbvXe+n/P01C7o2PbW9spGlvGkTd+GIDm+n1svHcl+9btpnBUCZUXH8fQoyv67ThEZAAI6MRmpCFuZvOB24AYcKe7fyuqfW3+5ctYfh41//YRGuv2sO72pZRUDaP4sGFd6k25al6X7drvPkfZUeUd2xvueoHSKSOZcuU83n1lK+sXLuPoW84gf2hRVF0XkYEojAyPbnWKmcWAO4BzgBrgEjOriWJfbU2t7Fn5NuMvOIpYcT5lU0czfPo4di6t6/F7zfX7aKjdwagTqwFo2rqXxk17GPfRo8grjDFi1mGUVA5j98oeHzYtIrkoO0+7T1uUSwznArXu/pa7NwP3AQui2FHT1gbIy6NobFlHWXHlMJrefq/H7+1cuokhU0dTWF4KwP6336OwvJRY8V/+B6W4chj7e2lHRHJQXpqvLIly15XApk7bdYmyjGtvaiVW0nVmKFZSQNv+1h6/t+v5TYz6QHXHdltTK7GSgm7ttPfSjojkmHRH4VkciWf9xKaZXQ5cDlAwquSQ2sgryqetsWvQtu1v6TKiTra3dget7zYxfNZhHWWxonzaGluS2mklr4d2RCRHDfY5cWAzUN1puypR1oW7L3T3Oe4+J7+s8JB2VDR2CLS307R1b0fZ/rp3KTps6EG/s+tPmxg+Y3zXqZPDhtJcv6/LCH5/3R6Ke2hHRHJUICPxKEN8GTDNzCabWSFwMbAoih3FivIZPnM87/x6LW1NrTTU7mDPi+8w6sTuSwwB2pvb2P3C24z8QHWX8qKxZZRUD2frY2tpb2ljz8otNG5+lxEzx0fRbREZyAb7nLi7twJXAkuAV4EH3H11VPurvOR42lvaWHPdEjbctYKqTx1P8WHD2PvGDl6+5vEudfe8uIVYaQFlR5Z3a2fiZbPZt2E3r3z1N2z51RomXX6ClheKDDZGMCPxSCd73X0xsDjKfRyQP6SQyV+a2628bNpojrvtvC5lI0+oYuQJqUfpheWlTL325Ej6KCIBCWROXGfsRERS0RWbIiIB0w2wREQCZQQznaKHQoiIBEwjcRGRbgxLczrFI+pJbxTiIiIpKMRFRAIWyHlNhbiISLL4MyHSS/G2aLrSK4W4iEgyS386JVsU4iIiKSjERUSClf7qlGxRiIuIpBBIhivERUSSxW9iGEaKK8RFRJLpxKaISNgskJunKMRFRFLQSFxEJGCBZLhCXEQkmWFpX7GZLQpxEZEUQplO0f3ERUSSJVanpPPqU7Nm881srZnVmtn1KT6fYGZPm9lKM3vJzM7trU2FuIhIPzCzGHAHcA5QA1xiZjVJ1f4JeMDdZwIXAz/srV2FuIhICmbpvfpgLlDr7m+5ezNwH7AgqY4DwxLvhwNv99bogJoTnzBiLLcvuDrb3ci4h954LNtdiMynj/5ktrsQifGlVdnuQmRiNqD+sx+QDvGKzXIzW95pe6G7L+y0XQls6rRdB5yY1MY/A0+a2VXAEODM3naqv00RkRQOIcTr3X3O+9ztJcA97v5dM/sA8DMzO9bd2w/2BYW4iEg3kdzFcDNQ3Wm7KlHW2WXAfAB3/5OZFQPlwLaDNao5cRGRZNGsTlkGTDOzyWZWSPzE5aKkOhuBMwDM7GigGNjeU6MaiYuIpJDpgbi7t5rZlcASIAbc7e6rzewWYLm7LwKuBX5iZl8hfpLzc+7e4zOYFeIiIkmiuhWtuy8GFieV3djp/Rrg5HTaVIiLiKQQyhWbCnERkRR07xQRkVD1/QKerFOIi4gkMT0oWUQkbHqyj4hIwDQSFxEJmEJcRCRggWS4QlxEJJlZOCNx3TtFRCRgGomLiHSjJYYiIkFTiIuIBCyQDFeIi4ikopG4iEigQlqdohAXEUlBIS4Z0/huI4//62Os+/M6SoaX8OEvfZhjzj72oPXbWtq48zM/oXlfM1f9z9Ud5euXr+ep23/HrrpdlI4o5QOXfoCZF87qj0MQCU4gGa4QD8GS7zxBrCDGNY/9A1vf2MoDX7ufMVPHUjGlImX953/xJ0pHlNK8r7mjrK21jYdveIgPX3E6MxfMZMurW/jFVT/nsGMqGTttbH8dikggwlliGNnFPmZ2t5ltM7NXotrHYNDc2Mxrz7zGKV84lcLSQqqnVzPtg9N45YmXU9bf/fZuXlnyCid95qQu5Y3vNtLU0MRx84/DzDis5jDKJ5ZTv66+Pw5DJDgRPCg5ElFesXkPMD/C9geFnRt3khfLY/SE0R1lY6aNZfu61A/AfvJ7Szjti6eRX1TQpbxsVBk1Zx3Di4+/SHtbO3Uv17HnnT1UT6+OtP8iIbJonnYficimU9z9WTObFFX7g0VzYzNFQ4q6lBUNKeoyVXLA2v99jfb2do489Sg2rNjQ7fNjzqrh8VsX89sfPAnA/K+dw7Cxw6LpuEjgAplN0Zz4QFdYUkhTQ1OXsuaGJgpLC7uWNTbz+zt+z19/9+KU7dSvr+dXN/6Ki/71IibPncLOTTt54Lr7GVpextSTp0XWf5FQhTInnvUQN7PLgcsBxlaOyXJvBp5RE0bR3tbOzk07GVU9CoCttduomNz1pOauTTvZs2UPP/vSvUB8hUpTQxO3nf8DPvuTz7H9re2Mqh7FlHmHAzB64mimnjSVN59/UyEukopCvG/cfSGwEOCo6Ud6lrsz4BSWFHLkqUfx7E/+l3NvOI+tb2zljT+8zmd+/Nku9SqmjOHKX13VsV33ch1Pfm8Jf/vTyygdUYq3OTvrdrJ++Xomzp7I7s27qX2ulnl/M6+/D0kkAOGsTsl6iEvv5l83n8e/+Ri3nfcDSoaXMP+6+VRMqWDjqo3cf+19XPfU18nLz6NsdFnHd0qGlWBmHWUjq0Zy3j+ez5Pff5J3t+6haEgRx5x9LDMumJmtwxIZuPS0ezCzXwKnAeVmVgfc5O53RbW/XFYyrISPf/sT3conzJjAdU99PeV3Js6a2OVCH4CaM2qoOaMmkj6K5BJDc+K4+yVRtS0iErVQQlxP9hERCZjmxEVEUghlJK4QFxFJIZAMV4iLiHST5Uvp06EQFxFJotUpIiKBU4iLiARMIS4iEipdsSkiEjaNxEVEAmW6AZaISNgU4iIiAQskwxXiIiLdmEbiIiJhU4iLiIRLI3ERkUAZkBdGhivERUS6C2eJoR4KISISMIW4iEgygzyztF59atZsvpmtNbNaM7v+IHU+aWZrzGy1mf13b21qOkVEJEkUt6I1sxhwB3AWUAcsM7NF7r6mU51pwA3Aye6+y8zG9NauRuIiIinkpfnqg7lArbu/5e7NwH3AgqQ6XwDucPddAO6+rS/9FBGRJBFMp1QCmzpt1yXKOjsCOMLMnjOz581sfm+NDqjplOL8Yo4acUy2u5Fx+YcPqD/mjPrtpqey3YVInFr5wWx3ITITyw7PdhcyzvGMtneI0ynlZra80/ZCd1+YZhv5wDTgNKAKeNbMjnP33T19QUREuuj7ycpO6t19Tg+fbwaqO21XJco6qwOWunsLsM7MXice6ssO1qimU0REkiXunZLOqw+WAdPMbLKZFQIXA4uS6vyK+CgcMysnPr3yVk+NaiQuIpLEyPwI191bzexKYAkQA+5299Vmdguw3N0XJT4728zWAG3Ade6+o6d2FeIiIikcwnRKr9x9MbA4qezGTu8d+Gri1ScKcRGRFEK57F4hLiKSJH4DLIW4iEiwwohwhbiISAqHtMQwKxTiIiJJzDSdIiISNJ3YFBEJWCgjcV2xKSISMI3ERUSSGFqdIiIStFCmUxTiIiLdaImhiEiwzLQ6RUQkaKGMxHtdnWJxl5rZjYntCWY2N/quiYhkj6X5ypa+LDH8IfAB4JLE9nvEn9gsIpKTDtwAK8PP2IxEX6ZTTnT3WWa2EsDddyWeSiEikrNCmU7pS4i3mFkM4k8iNbMKoD3SXomIZFWfH7mWdX0J8f8AHgXGmNk3gY8D/xRpr2RQaNjTwC9vup/X/u91howcwkevPpc5580+aP3Wlla+/fHvsL+hiX/53U0d5ffd/AC1y99k+8Z6PnXLX3PiAp2ykfcnisezRaXXEHf3X5jZC8AZxI/tQnd/tbfvmVk1cC8wlvgofqG73/Y++ys55MFvPkKsIJ9vPnMzda9t5sdX3knlkZWMnzouZf2nfvo0ZSPL2N/Q1KX8sCMPY+ZHZrDoB4/1R7dlMAhoiWFfVqdMAPYBvyb+ZOaGRFlvWoFr3b0GmAdcYWY176ezkjua9jXx4u9e4rwr5lNUWsThs6Zw7GnHsOyx5Snr76jbwfLHX+Csy87o9tkpF3+QI+cdQUFRQdTdlkEkl05sPk58JG1AMTAZWAsc09OX3H0LsCXx/j0zexWoBNa8nw5Lbti2YTt5+XmMmTSmo6zyiMOoXf5myvoPfetRzr/6XAqKFdQSvZAez9brSNzdj3P34xP/nAbMBf6Uzk7MbBIwE1h6KJ2U3NO8r5niIcVdykrKimna19St7otPvUR7WzvTzzi+v7ongpml9cqWtOfu3X0FcGJf65tZGfAw8A/u/m6Kzy83s+VmtnzH9h3pdkcCVVhayP6G/V3K9jc0UVRa1KWsaV8Ti77/GBdd/7H+7J4MekZemq9s6XU6xcy+2mkzD5gFvN2Xxs2sgHiA/8LdH0lVx90XAgsBZsye7n1pV8I3ZmIF7a3tbNuwnTETKwDYvPbtbic1t2+sZ8fbO7ntc7cD0NbSSuPe/Xzjwzfx1Z9fw+jKUf3edxkcQjmx2Zc58aGd3rcSnyN/uLcvWfxP4C7gVXf/3qF1T3JVUWkR0888jsV3PMEl//xJNq99m5efeYWv3Ht1l3rjp47jlidv7Nhet2o9D976CF+//6uUjSwD4ksPvd1xd9pa2mhpaiFWECMvL5RFYiKHrscQT1zkM9Tdv3YIbZ8MfBp42cxWJcr+0d0XH0JbkoM+8Y2L+O8b7+cbp93EkBGlfPIbFzF+6jjefOEt/vPLC/nO0m8Ry48xrHxYx3dKh5diZl3Kfvj3P+44Ibpu1Xruu+VBrrrry0w7YWq/H5Pkhpx4ULKZ5bt7q5mdfCgNu/sfCefhGJIFQ4YP4Qu3/W238sNnT+E7S7+V8jvTTpja5UIfgKvvviKS/sngZoHEV08j8T8Tn/9eZWaLgAeBhgMfHmyOW0QkF+TSnHgxsAM4nb+sF3dAIS4iOcly5Mk+YxIrU17hL+F9gFaRiEhOs0DuntJTiMeAMlLPayvERSSn5cJIfIu739JvPRERGUByYU48jCMQEckwS/yEoKcQ7367OBGRwSAX1om7+87+7IiIyECSC9MpIiKDUvzJPuGvThERGaRy6xmbIiKDjkJcRCRg2bxHeDoU4iIiSQyNxEVEwhXQEsMwTr+KiEhKGomLiHSTG1dsiogMSgbkWRgTFQpxEZEUQjmxGcavGhGRfmZp/vSpTbP5ZrbWzGrN7Poe6l1kZm5mc3prUyNxEZFuMv9kn8SD5+8AzgLqgGVmtsjd1yTVGwpcAyztS7saiYuIJDEiGYnPBWrd/S13bwbuAxakqPcvwLeB/X1pVCEuIpJCnllarz6oBDZ12q5LlHUws1lAtbs/3td+DqjplDzyKI6VZLsbGVcz4vhsdyEyVUMmZLsLkbht1Y+y3YXInDHhpGx3IeMaWvZmtkEDS391SrmZLe+0vdDdF/Z5l/Edfg/4XDo7HVAhLiIyMBzSOvF6d+/pRORmoLrTdlWi7IChwLHAM4mVMeOARWZ2gbt3/uXQhUJcRCRJfJ14xpcYLgOmmdlk4uF9MfCpAx+6+x6gvKMPZs8AX+spwEEhLiKSUqbXibt7q5ldCSwBYsDd7r7azG4Blrv7okNpVyEuIpJCFLeidffFwOKkshsPUve0vrSpEBcRSaJb0YqIBM0OZXVKVijERURS0JN9REQCZabpFBGRoIVyP/EwJn1ERCQljcRFRLoxTaeIiIRMJzZFRAIVXycexmyzQlxEpBs9KFlEJGiaExcRCZhG4iIiAdNIXEQkUIZWp4iIhMu0TlxEJGgWyAXtCnERkRQ0EhcZpBrfbWTJrU+wftl6SoaXcMrfn8LRZ9d0q/fcXX9k6b3PEyuMdZR99p7PM6JyBADb3tjKklufYMeGHYyeOJqP3DCfMdPG9ttxDGaGVqdgZsXAs0BRYj8PuftNUe1PZKB46ru/JVaQx5cXXcG2N7bxyNcfomLqGMqnlHere+QZR3Hejed3K29raePR6x9l9idnM+NjM3npf17k0esf5e/u+wKxgli3+pJpFsWDkiMR5aRPE3C6u08HZgDzzWxehPsTybrmxmZe/9/XOfnvPkRhaSFV06uY+sGprFmyOq12Nq3ciLe1M/uTc8gvzGfWJ2aDOxtf2BBRzyWZpfmTLZGFuMftTWwWJF4e1f5EBoJdm3aRF8tj1IRRHWUVh4+hfl19yvpvPlfL7ef8Bz+99C5WPbqyo7x+XT3lh1d0mZetmFpx0HYk8yyxQqWvr2yJdE7czGLAC8BU4A53Xxrl/kSyraWxmcIhRV3KisqKaN7X3K3ukacfxfQF0ykdOYQta7aw6J9+RVFZEUefVUNLYwtFZUntDEndjmRefE48jNUpkfbS3dvcfQZQBcw1s2OT65jZ5Wa23MyW19fviLI7IpErKCmkuaGpS1lTQxOFpYXd6pZPLqesfCh5sTwqj6tk1sdn8/ozryfaKaC5oTmpneaU7cjg1i+/atx9N/A0MD/FZwvdfY67zykvH90f3RGJzMjqkbS3tbNr086Osu212ymf3P2kZjdm4PEZx/LJ5Wx/cxvuf5mB3P5mH9uRDEhvKiWb0ymRhbiZVZjZiMT7EuAs4LWo9icyEBSWFDLt1CN47s4/0tzYzOaX6qj94xvUfOSYbnVr//AG+9/dj7uzZc0WVj70Aod/aBoA1TMnYHl5rHjwBVqbW1nx8AoAJsye2K/HM5jlYWm9siXKOfHxwH8l5sXzgAfc/bEI9ycyIJx57VksufU3/PCjd1AyrJizrj2b8inl1L24iYe/9hDX/PYrALz2u1d54tbf0NbSxtCKocz9mxM59pz4jGOsIMaFt36MJ7/1BH/40bOMmjSKC2/9mJYX9hc97R7c/SVgZlTtiwxUJcNKuPDWv+pWXjW9uiPAAc6/+YIe2xl7xFg+ffdnM94/6Z0u9hERCdygH4mLiITLglliqBAXEUkhlMvuFeIiIkk0Jy4iEjjNiYuIBCu7N7VKh0JcRCQFjcRFRAIVf1CyVqeIiIRJV2yKiIRMc+IiIkHTSFxEJGChjMTDmLkXEZGUNBIXEUmiKzZFREKnOXERkVBpdYqISNC0OkVEJGAaiYuIBCyUENcSQxGRJEZ8OiWdV5/aNZtvZmvNrNbMrk/x+VfNbI2ZvWRmT5nZxN7aVIiLiHRjaf/02qJZDLgDOAeoAS4xs5qkaiuBOe5+PPAQ8G+9tasQFxFJIdMhDswFat39LXdvBu4DFnSu4O5Pu/u+xObzQFVvjQ7AOfEw5qHSEcsbgH/MGTKicHS2uxCJLx33+Wx3ITKn3v6lbHch47bu2JrZBg/tLoblZra80/ZCd1/YabsS2NRpuw44sYf2LgN+09tOczddRETeh0M4sVnv7nMysm+zS4E5wKm91VWIi4gkOXBiM8M2A9WdtqsSZV33bXYm8A3gVHdv6q1RzYmLiHST+RObwDJgmplNNrNC4GJgUZe9ms0Efgxc4O7b+tKoRuIiIilkep24u7ea2ZXAEiAG3O3uq83sFmC5uy8C/h0oAx5M/J/ARne/oKd2FeIiIilEcdm9uy8GFieV3djp/ZnptqkQFxFJQVdsiohI5DQSFxFJoodCiIgEre/3Q8k2hbiISEoKcRGRMB3aZfdZoRAXEUlBc+IiIgFTiIuIBMp0YlNEJGwaiYuIBEwhLiISME2niIgETCNxEZFA6cSmiEjgNBIXEQmaQlxEckz7vhZ2PrKWptqd5A0pYPjZUyidPrZbve33vETzht0d297m5JeXMu7qE2jdvZ+tt/25S31vbmf4OYcz9IPVyU1lTRgRrhAXkTTs+vUbWL4x/oaTaNmyl/p7X6ZgXBkFY4d0qVfxueO7bG+7cyXFU0YCkD+imMqbTun4rHVnI+98byklx5RHfwBpCGVOPPKHQphZzMxWmtljUe9LRKLT3txG4+rtDDtzMnlF+RRNGkHJ0eXsW/VOj99r3dVI8/o9lM7sPmIH2LdyK0WTRpA/siSKbue8/niyzzXAq/2wHxGJUGv9PizPKCgv7SgrGDeElq37evzevpVbKZw0PGVIuzsNq96hdFbqgM8uS/OVHZGGuJlVAecBd0a5HxGJnje3YUWxLmVWnE97c2uP32tY+Q5DZo1L+Vnzhj20722m5JiKjPUzU8KI8OhH4j8Avg60R7wfEYmYFcbwprYuZd7URl7hwU+tNa3f3WNIN6x4h5JjKsgrGmin59KN8BwciZvZ+cA2d3+hl3qXm9lyM1teX78jqu6IyPuUX16Ktzst9X+ZPmnZspeCsaUH/c6+lVspqUkd0t7SRuMr2ymdmXqUnk2WeChEOq9siXIkfjJwgZmtB+4DTjeznydXcveF7j7H3eeUl4+OsDsi8n7kFcYoqSnn3afW097cRtOGPTS+Wk/pjNQh7C1t7Ht5G6UHmUppXFNPXkk+RVNGRNntnBdZiLv7De5e5e6TgIuB37v7pVHtT0SiN/KCI/CWNrb863PsvH8NIxccQcHYITSt383mm5/tUre3kG5Y8Q6lM8YN2KV8luZPtgy0iSgRGcDySgsov/S4buVFk0Z0WfsNUDp9bMoLgQ6o+Pz0jPcvk3TZfSfu/gzwTH/sS0RkMNFIXEQkhYE6zZOsPy72ERGRiGgkLiLSTXZPVqZDIS4ikpJCXEQkSNm+lD4dCnERkRRCObGpEBcRSUkhLiISrDAiXCEuInIQYcS41omLiARMI3ERkW6ye3vZdGgkLiISMI3ERUSSxNeJhzESV4iLiKSkEBcRCVYYEa4QFxFJKZQTmwpxEZFuwrl7ikJcRCSFMCJcSwxFRA7C0nz1oUWz+Wa21sxqzez6FJ8Xmdn9ic+Xmtmk3tpUiIuIJLP4nHg6r16bNIsBdwDnADXAJWZWk1TtMmCXu08Fvg98u7d2FeIiIv1jLlDr7m+5ezNwH7Agqc4C4L8S7x8CzrBefkMoxEVEkhy42Cednz6oBDZ12q5LlKWs4+6twB5gdE+NDqgTmytXrKofWjh8Qz/trhyo76d99ScdV3hy9dj687gmZrKxFS+sXFKSP6Q8za8Vm9nyTtsL3X1hJvuVyoAKcXev6K99mdlyd5/TX/vrLzqu8OTqsYV8XO4+P4JmNwPVnbarEmWp6tSZWT4wHNjRUy8ZLCIAAALaSURBVKOaThER6R/LgGlmNtnMCoGLgUVJdRYBn028/zjwe3f3nhodUCNxEZFc5e6tZnYlsASIAXe7+2ozuwVY7u6LgLuAn5lZLbCTeND3aDCHeORzVVmS1eMyszbgZeL/br0KfNbd9x1iW/cAj7n7Q0CjmdW4+5qD1D0NaHb3/0tzH+uBOe6ezTlp/bs4SLj7YmBxUtmNnd7vBz6RTpuDdjqlP044ZMMAOK5Gd5/h7scCzcAXO3+YmOdLm7t/6GABnnAacNKhtJ1tA+DvLBK5elwDzaANcekXfwCmmtlpZvYHM1sErDGzmJn9u5ktM7OXzOzvASzu9sQVbb8DxhxoyMyeMbM5iffzzWyFmb1oZk8lrmr7IvAVM1tlZh8yswozezixj2VmdnLiu6PN7EkzW21mdxLO1dUiKQ3m6RSJUGLEfQ7wRKJoFnCsu68zs8uBPe5+gpkVAc+Z2ZPATOBI4lezjQXWAHcntVsB/AQ4JdHWKHffaWY/Ava6+3cS9f4b+L67/9HMJhCfhzwauAn4o7vfYmbnEb9CTiRYCnHJtBIzW5V4/wfiJ2pOAv7s7usS5WcDx5vZxxPbw4FpwCnAL929DXjbzH6fov15wLMH2nL3nQfpx5lATaeL3YaZWVliH3+V+O7jZrbrEI9TZEBQiEumNbr7jM4FiSBt6FwEXOXuS5LqnZvBfuQB8xInipL7IpIzNCcu2bAE+JKZFQCY2RFmNgR4FvjrxJz5eODDKb77PHCKmU1OfHdUovw9YGinek8CVx3YMLMDv1ieBT6VKDsHGJmxoxLJAoW4ZMOdxOe7V5jZK8CPif9f4aPAG4nP7gX+lPxFd98OXA48YmYvAvcnPvo18LEDJzaBq4E5iROna/jLKpmbif8SWE18WmVjRMco0i+sl4uBRERkANNIXEQkYApxEZGAKcRFRAKmEBcRCZhCXEQkYApxEZGAKcRFRAKmEBcRCdj/Byke+ghgiJNJAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BG-jq6r6_9t1",
        "outputId": "7ee632d5-639a-481a-dfff-11e1d10522a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "helpfulness_groups = train.groupby([\"Helpfulness\"])\n",
        "print(helpfulness_groups.size())\n",
        "helpfulness_balanced_predictor, helpfulness_balanced_score, helpfulness_balanced_predictions, helpfulness_balanced_true_labels = train_and_score(*train_test_validate_split_groups(helpfulness_groups, sample_size_group=30000), model_name=\"helpfulness_balanced_real\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Helpfulness\n",
            "-1    129372\n",
            " 0     33610\n",
            " 1    112587\n",
            "dtype: int64\n",
            "train size: 72,000\n",
            "test size: 18,000\n",
            "validate size: 185,569\n",
            "[0, 1, 2, 3, 4]\n",
            "Validation score: 0.7645350247077907\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ojs0MNBGP2l5",
        "outputId": "4f591eac-ec9b-4440-e66c-3e20eab8c953",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "helpfulness_balanced_predictions = helpfulness_balanced_predictor.predict(test.TextClean.values)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C2SMBV9vNjGw",
        "outputId": "a341121a-7434-4ad0-aabd-f745f4f7bd32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "cm7 = confusion_matrix_custom(true_labels, helpfulness_balanced_predictions, \"helpfulness_balanced.pdf\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAE9CAYAAAD9HVKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1f3/8dcnk4Uk7JsVEmQRRQRFpGilVtyB+pPaasXW7l/tZn/uLe23P9v6/dV+rdWudkFL1VpF6/L7oaLYWq12UUFFNrVFRAkuiCACCYQkn+8fM8RkMiSZOHduzuT95DGPx9wzZ849F/Gdk3PPvdfcHRERCVNR3B0QEZGuU4iLiARMIS4iEjCFuIhIwBTiIiIBU4iLiASsx4W4mc0wsxfMbI2ZzY27P7liZvPNbKOZrYy7L7lkZtVm9rCZrTazVWZ2ftx9ygUz62VmT5rZs6nj+l7cfco1M0uY2TNmdm/cfSlkPSrEzSwBXAvMBMYDZ5nZ+Hh7lTM3ADPi7kQEGoCL3X08cCTw1QL5b7YLOM7dDwUmATPM7MiY+5Rr5wPPxd2JQtejQhyYCqxx97XuXg8sAGbH3KeccPdHgc1x9yPX3P01d3869X4byVAYHm+v3jtP2p7aLEm9CubKOzOrAj4MXB93XwpdTwvx4cD6Fts1FEAg9BRmNhI4DHgi3p7kRmq6YRmwEfiTuxfEcaX8BPg60BR3RwpdTwtxCZSZ9QbuBC5w93fi7k8uuHuju08CqoCpZjYh7j7lgpmdAmx096fi7ktP0NNCfANQ3WK7KlUm3ZiZlZAM8D+4+11x9yfX3P1t4GEK55zGNOBUM1tHcsryODO7Od4uFa6eFuJLgLFmNsrMSoE5wMKY+yTtMDMDfgs85+7XxN2fXDGzIWbWP/W+HDgReD7eXuWGu3/T3avcfSTJ/8f+4u5nx9ytgtWjQtzdG4DzgMUkT5Dd7u6r4u1VbpjZrcA/gQPNrMbMvhB3n3JkGvApkqO5ZanXrLg7lQP7Ag+b2XKSg4s/ubuW4knWTLeiFREJV48aiYuIFBqFuIhIwBTiIiIBU4iLiARMIS4ikicd3ajOkn6WukHfcjOb3FGbPTbEzezcuPsQBR1XeAr12Ar1uN6jG2j/oq6ZwNjU61zgVx012GNDnORfUCHScYWnUI+tUI+ryzpxo7rZwE2pG6Q9DvQ3s33ba7Mnh7iISHeT9U36iiPtTpaKKkq8qH9ZfvbVt5TiYb3zcqXThPeNycduAKiqHs6kyYfm7Qous/yMA6pHVHHY4Yfl7biKzPK1K6pHVHP4lMl5O7Ymz8+NBauqqzjs8El5Oa5XXl7PW5veytl/NBvcy6nP8u9p2+5VwM4WJfPcfV6u+rQ33SvE+5fR9wsT4+5Gzv3pW3fG3YXIlBVXxN2FSJQWlcbdhcjsbKiNuws5d+xRJ+S2wfomOGJodt/584ad7j7lPe4565v0aTpFRCQTs+xeubEQ+HRqlcqRwFZ3f629L3SrkbiISLdgRDLETd2objow2MxqgO+QfKoT7v5rYBEwC1gD1AKf66hNhbiISCYRnBdx97M6+NyBr2bTpkJcRCST/J3bfk8U4iIibeR0njtSCnERkXQRzYlHQSEuIpKJRuIiIgELI8MV4iIibRhQFEaKK8RFRDIJI8MV4iIiGWlOXEQkYGFkuEJcRKQNzYmLiAQujAxXiIuItKUrNkVEwhXQdEogF5aKiEgmGomLiGQSxkBcIS4ikpHmxEVEAhZGhhdOiDfVNVB774vsXruVovJiyo8bQemEwW3qeUMTtQ+uY/cLW6CxieLqPlTMHE1R3+SDcXcueZ365W/SuLGW0oMHUXnq/vk+FBGJm05sJpnZDDN7wczWmNncKPdVe/9LkDD6X3g4lR/Znx33v0Tjm22f6r3ryddprNlO33Mm0u+Cw7FexdQufqn586I+pfT64HBKDx0SZXdFpLuzLF8xiSzEzSwBXAvMBMYDZ5nZ+Cj25fWN7H5+M+XHVGOlCYpH9KV07AB2rdjUpm7j2zspHtOPot6lWHERpeMH0fhmXfPnpeMGUnrgQIrKS6LoqoiEIp6n3WctypH4VGCNu69193pgATA7ih01bt4JRUZiUHlzWWKfCpoyjMTLJg2lYf02mrbV47sb2bVyEyX794+iWyISsqIsXzGJck58OLC+xXYNcEQUO/L6Rqws0arMyhL4rsY2dRMDe1HUt4ytP30aDBJDK6iYMSqKbolIqGIeXWcj9hObZnYucC7QfHIx6zZK2wZ2pmAHqH1gHTQ20e/iKVhJETv/+Srbb32Ovp+f2KV9i0iBCiPDI/0lYANQ3WK7KlXWirvPc/cp7j7FKrs2D50Y2AuanMbN785tN75RS9GQijZ1G97YQekhQygqL8aKiyh7//tofHUHTbW7u7RvESlQmhNnCTDWzEaZWSkwB1gYxY6sNEHJuIHUPVKD1zfSsH4b9f/aQtnEtksMi/ftTf2KTfjOBryxiV1L38D6lFBUkfwB4k2ONzSBOzQllyR6k0fRbRHpznr6nLi7N5jZecBiIAHMd/dVUe2vYuYoau95kbd//BRWXkzlzFEkhlSw+5V32H7r8wz4xlQAyk8YQd3idWz95TJodIqGVND79AOb29n5WA07H3v3F4b6lZvodfRwyo+pbrNPESlQhubEAdx9EbAoyn3sUVReTO+PH9imvGRE3+YAByiqKKHytLF7baf8mGoFtogEMyce+4lNEZFuKZArNhXiIiKZaDpFRCRQMV9Knw09FEJEJGAaiYuItGFYltMpcS1EVoiLiGSgEBcRCVgg5zUV4iIi6ZLPhMguxdvebi8/FOIiIuks++mUuCjERUQyUIiLiAQr+9UpcVGIi4hkEEiGK8RFRNIlb2IYRoorxEVE0unEpohI2CyQm6coxEVEMtBIXEQkYIFkuEJcRCSdYVlfsRkXhbiISAahTKfofuIiIulSq1OyeXWqWbMZZvaCma0xs7kZPh9hZg+b2TNmttzMZnXUpkJcRCQPzCwBXAvMBMYDZ5nZ+LRq3wZud/fDgDnALztqVyEuIpKBWXavTpgKrHH3te5eDywAZqfVcaBv6n0/4NWOGu1Wc+L7DhjAxaefHnc3cm7lluVxdyEyg3sNjrsLkRjTd1zcXYhMgzfE3YWc8xw/kqGLV2wONrOlLbbnufu8FtvDgfUttmuAI9La+C7woJl9DagETuhop90qxEVEuosuhPgmd5/yHnd7FnCDu19tZh8Afm9mE9y9aW9fUIiLiLQRyV0MNwDVLbarUmUtfQGYAeDu/zSzXsBgYOPeGtWcuIhIumhWpywBxprZKDMrJXnicmFanVeA4wHM7CCgF/Bme41qJC4ikkGuB+Lu3mBm5wGLgQQw391XmdnlwFJ3XwhcDFxnZheSPMn5WXdvd8JfIS4ikiaqW9G6+yJgUVrZZS3erwamZdOmQlxEJINQrthUiIuIZKB7p4iIhKrzF/DETiEuIpLG9KBkEZGw6ck+IiIB00hcRCRgCnERkYAFkuEKcRGRdGbhjMR17xQRkYBpJC4i0oaWGIqIBE0hLiISsEAyXCEuIpKJRuIiIoEKaXWKQlxEJAOFuOTMO1u2cdUlP+apvz5N34H9OOebn+X4045tU2/Br+7gwT/+mTdqNtJvYF9O/cwpzPny6c2fr1n5Ij//P79i7XMvUV5Zzv86exafuvAT+TwUkWAEkuEK8RD89D+vpaSkhDufvZU1q17kW5/+DqPHj2bUgfu1rujO3J9ewpiDRrFh3Wt8/RP/ydBhgzlu9nQAvn/elXxwxlFcc8eVvL7+Dc4/7RJGHzyaaScdmf+DEunWwlliGNnFPmY238w2mtnKqPbRE9TV7uSxRX/nc5d+ivLKciZOncAHTjySP935UJu6c75yBgdM3J9EcYIR+1cx7eQjWblkdfPnr6/fyPEfPZZEIsHwkcOY8P6DefmFl/N5OCLBiOBByZGI8orNG4AZEbbfI9SsrSGRSFA9pqq5bMzBo1jXQfi6OyueWMnIA94drX/sPz7Cg3c8RMPuBl5ZU8Pqp55j8tGTIuu7SKgsmqfdRyKyEHf3R4HNUbXfU9Tt2ElFn4pWZZV9KqnbUdfu9268+maampwZZ57YXHbkCVN59L6/MWPMbD57zDnMOutkxk06MJJ+i4TOLLtXXHTvlG6uvLIXtdtqW5XVbqulvLJ8r9+5+3cLefCOh/jBTd+jtKwUSJ4cnXv2t/n0BZ9g8dqF3LbkJpY88hT//4Z7I+2/SKh6/Ei8s8zsXDNbamZLd2zZEXd3up2q0VU0NjZSs3ZDc9mLq19iZPpJzZT7Fyzm1l/czo9u+wFDhg1pLn/tldcoSiQ46YwTSBQnGDJsCMfOPoYn/rIk8mMQCVIgQ/HYQ9zd57n7FHefUjmgMu7udDvlFb04euZR/O5Hv6eudicrl6ziHw/+kxM/dnybun++6y9c/983ctWCKxi2376tPqsaPRx356G7H6apqYnNGzfzyMJHGX3QqHwdikhAshuF9+iRuHTs/CvOo35nPR87ZA7/9ytXcsEPzmPUgfux/ImVzBp7WnO9+T+8iXe2vMOXZ53PrLGnMWvsafz4Gz8HkvPo37vu29xx3d3MHn8G55x0HiPHjeTsC+bEdVgi3VeWg/A458QjWyduZrcC04HBZlYDfMfdfxvV/gpZ3wF9+K/5l7UpP+SICSz6993N27c8fkO77Uz+4CR+tehnue6eSMExdMUm7n5WVG2LiEQtlBDXdIqISMB02b2ISAahjMQV4iIiGQSS4QpxEZE2Yl42mA2FuIhIGq1OEREJnEJcRCRgCnERkVDFfBVmNhTiIiIZaCQuIhIoC+jxbApxEZEMFOIiIgELJMMV4iIibZhG4iIiYVOIi4iESyNxEZFAGVAURoYrxEVE2gpniaEeCiEiEjCFuIhIOoMis6xenWrWbIaZvWBma8xs7l7qfNzMVpvZKjO7paM2NZ0iIpImilvRmlkCuBY4EagBlpjZQndf3aLOWOCbwDR332JmQztqVyNxEZEMirJ8dcJUYI27r3X3emABMDutzjnAte6+BcDdN3amnyIikiaC6ZThwPoW2zWpspYOAA4ws7+b2eNmNqOjRrvVdEr/sn7MHn1K3N3IuZKi0ri7EJl5K38Xdxci8bVDhsXdhcjsaNgedxdyrskbc9peF6dTBpvZ0hbb89x9XpZtFANjgelAFfComU1097fb+4KIiLTS+ZOVLWxy9yntfL4BqG6xXZUqa6kGeMLddwMvmdm/SIb6kr01qukUEZF0qXunZPPqhCXAWDMbZWalwBxgYVqd/0dyFI6ZDSY5vbK2vUY1EhcRSWPkfoTr7g1mdh6wGEgA8919lZldDix194Wpz04ys9VAI3Cpu7/VXrsKcRGRDLowndIhd18ELEoru6zFewcuSr06RSEuIpJBKJfdK8RFRNIkb4ClEBcRCVYYEa4QFxHJoEtLDGOhEBcRSWOm6RQRkaDpxKaISMBCGYnrik0RkYBpJC4iksbQ6hQRkaCFMp2iEBcRaUNLDEVEgmWm1SkiIkELZSTe4eoUSzrbzC5LbY8ws6nRd01EJD6W5SsunVli+EvgA8BZqe1tJJ/YLCJSkPbcACvHz9iMRGemU45w98lm9gyAu29JPZVCRKRghTKd0pkQ321mCcABzGwI0BRpr0REYtXpR67FrjMh/jPgbmComX0fOB34dqS9kh6hbmsd91xxD2ufWEtF/wqO/fKxTDx54l7rN+5u5Ddn/4b62nouuOcCAF5Z9gq3XHhLq3q763Zz+hWnc9BxB0XafylcUTyeLSodhri7/8HMngKOJ3lsH3H35zr6nplVAzcB+5Acxc9z95++x/5KAbn/R/eTKE5w0aKLeP1fr7Pg4gXsM3Yfho4emrH+P27+B5UDKqmvrW8uGzFpBHMfntu8ve6pddx26W2M+cCYyPsvBSygJYadWZ0yAqgF7iH5ZOYdqbKONAAXu/t44Ejgq2Y2/r10VgpHfV09zz38HNO/OJ3SilJGTBrBAUcfwIr7V2Ssv+XVLax4YAXTPj2t3XaXL1rOQcceRGm5TtvIe1NIJzbvIzmSNqAXMAp4ATi4vS+5+2vAa6n328zsOWA4sPq9dFgKw1uvvEVRoohBIwY1l+0zdh9efvrljPUXX72Y4758HMVle/8nu+cHw5lXnZnz/krPUlCPZ3P3VpOUZjYZ+Eo2OzGzkcBhwBPZfE8K1+663ZRVlrUqK6ssazVVssfzjzxPU2MT46aPY91T6/ba5vOPPE9Fvwr2m7xfrrsrPVDBTKekc/engSM6W9/MegN3Ahe4+zsZPj/XzJaa2dLNm7Zk2x0JVEl5Cbt27GpVtmvHLkorWk+D1NfV89AvHuLki07usM3l9y1n4qyJwfzPJ92ZUZTlKy4djsTN7KIWm0XAZODVzjRuZiUkA/wP7n5XpjruPg+YBzBx8gTvTLsSvkEjBtHU2MRbr7zVPKXyxpo3GDJ6SKt6m9dv5u3X3ubGL90IQGNDI7u27+KaWdfw+es/T/9h/QHY+sZW1j2zjllzZ+X3QKRghTIY6MyceJ8W7xtIzpHf2dGXLPk38FvgOXe/pmvdk0JVWl7KuOnj+Ot1f+WUb53CG/96g389+i8+e91nW9UbOnoo5y88v3l7/fL1PHD1A5xz4zlU9K9oLl9x/wqqJ1YzsGpgvg5BpFtoN8RTF/n0cfdLutD2NOBTwAozW5Yq+5a7L+pCW1KAZl06i4XfX8g1M6+hvF85M78+k6Gjhzav/Z778FyKiovoPah383fK+5ZjZq3KAJbfv5wPfPID+T4EKVAF8aBkMyt29wYza39N1164+98I5+EYEoPyfuWc+cO2K0nS1363NPLwkc0X+rT0lduyOtcu0iELJL7aG4k/SXL+e5mZLQT+COzY8+He5rhFRApBIc2J9wLeAo7j3fXiDijERaQgWYE82WdoamXKSt4N7z20ikRECpoFcveU9kI8AfQm87y2QlxEClohjMRfc/fL89YTEZFupBDmxMM4AhGRHLPUnxC0F+LH560XIiLdSSGsE3f3zfnsiIhId1II0ykiIj1S8sk+4a9OERHpoQrrGZsiIj2OQlxEJGBx3iM8GwpxEZE0hkbiIiLhCmiJYRinX0VEJCONxEVE2iiMKzZFRHokA4osjIkKhbiISAahnNgM40eNiEieWZZ/OtWm2Qwze8HM1phZ5mcQJut9zMzczKZ01KZG4iIibeT+yT6pB89fC5wI1ABLzGyhu69Oq9cHOB94ojPtaiQuIpLGiGQkPhVY4+5r3b0eWADMzlDvv4ArgZ2daVQhLiKSQZFZVq9OGA6sb7FdkyprZmaTgWp3v6+z/exW0ynFVsKgXvvE3Y2cK7Zu9decU58ff3bcXYjEi9v+HXcXIvOrZX+Muws59+r2jblt0MCyX50y2MyWttie5+7zOr3L5A6vAT6bzU4LN11ERLqsS+vEN7l7eyciNwDVLbarUmV79AEmAI+kVsa8D1hoZqe6e8sfDq0oxEVE0iTXied8ieESYKyZjSIZ3nOAT+z50N23AoOb+2D2CHBJewEOCnERkYxyvU7c3RvM7DxgMZAA5rv7KjO7HFjq7gu70q5CXEQkgyhuRevui4BFaWWX7aXu9M60qRAXEUmjW9GKiATNurI6JRYKcRGRDPRkHxGRQJlpOkVEJGih3E88jEkfERHJSCNxEZE2TNMpIiIh04lNEZFAJdeJhzHbrBAXEWlDD0oWEQma5sRFRAKmkbiISMA0EhcRCZSh1SkiIuEyrRMXEQmaBXJBu0JcRCQDjcRFeqitW97higt/yJOPLKX/oH586VvncPLHTmhT7+ZrF3D/bQ/wes0b9BvYj49+7iOc/dU5zZ+fNuVMNr+5hURRckQ48f0T+OntP8rbcfRkhlanYGa9gEeBstR+7nD370S1P5Hu4uq5P6GkpIT7Vt3Fv1eu4eJPfpOxB49h9LhRrSu6c9kvvsWY8aPZsO5VLjjzUvYZNoQTTzu+ucpVN13B1GPae4C6RMOieFByJKKc9NkFHOfuhwKTgBlmdmSE+xOJXd2OOh6+71HOnft5KiorOPSIQzj65KN44I8Ptql79nlnceAhB1BcXMx++4/g6JOnsfzJlTH0WjKxLP/EJbIQ96Ttqc2S1Muj2p9Id/DK2hoSxQlGjKluLtv/4DGsfWFdu99zd5Y9sZxR40a2Kv/uV77PzPGzOf/jl/DvVWsi6LHsjaVWqHT2FZdIT7+aWcLMlgEbgT+5+xNR7k8kbnU76qjsXdGqrHef3tTuqG33e9dfdQPe5JwyZ2Zz2Xd/+W3uWrqAu5fexuRph3HBmZeybeu2SPotrSXnxIuyesUl0j27e6O7TwKqgKlmNiG9jpmda2ZLzWzpW5s2RdkdkciVV5azY3vrwN6xfQcVlRV7+Qb88bd3cf/ti7n6Dz+gtKy0ufzQqRPpVV5Gr4pefOb8T9K7X2+WPb4isr5LmPLy48Pd3wYeBmZk+Gyeu09x9ymDBg/OR3dEIjNidBWNDY2sX1vTXPbvVS8y+sCRGevfc8sifv/zW/j5HdcwdNjQdts2M3DNSOZHdlMpBTmdYmZDzKx/6n05cCLwfFT7E+kOyivLmT7raK67cj51O+p49skVPPbA35lxxklt6i6+40/8+orr+NntVzN85LBWn71e8wbPPrmC3fW72bVzFzdfu4Ctm7dyyNQ2v8xKRLKbTIkvxKNcJ74vcKOZJUj+sLjd3e+NcH8i3cIlV17IFRdcyayDT6PfwL5ceuWFjB43imWPL+eis77OX156AIDf/Pdv2brlHT5/8hebv3vy6SfyjasupnZ7LVd9/cdsWPcqpb1KGXvw/lxzy5X0G9gvrsPqWfS0e3D35cBhUbUv0l31G9CXK2/8fpvySUce0hzgAHctXbDXNkaPG8XNj8yPpH/SMV3sIyISuB4/EhcRCZfpBlgiIiEL5bJ7hbiISBrNiYuIBE5z4iIiwYr3plbZUIiLiGSgkbiISKCSD0rW6hQRkTDpik0RkZBpTlxEJGgaiYuIBCyUkXgYM/ciIpKRRuIiIml0xaaISOg0Jy4iEiqtThERCZpWp4iIBEwjcRGRgIUS4lpiKCKSxkhOp2Tz6lS7ZjPM7AUzW2NmczN8fpGZrTaz5Wb2kJnt11GbCnERkTYs6z8dtmiWAK4FZgLjgbPMbHxatWeAKe5+CHAH8MOO2lWIi4hkkOsQB6YCa9x9rbvXAwuA2S0ruPvD7l6b2nwcqOqo0W41J24YCUvE3Y2cK7LC/Vk5pHzfuLsQiUG9hsbdhcjcdMN/xN2F3Hvrndy217W7GA42s6Uttue5+7wW28OB9S22a4Aj2mnvC8D9He20W4W4iEh30YUTm5vcfUpO9m12NjAFOKajugpxEZE0e05s5tgGoLrFdlWqrPW+zU4A/hM4xt13ddRo4f6eLyLSZbk/sQksAcaa2SgzKwXmAAtb7dXsMOA3wKnuvrEzjWokLiKSQa7Xibt7g5mdBywGEsB8d19lZpcDS919IXAV0Bv4Y+o3gVfc/dT22lWIi4hkEMVl9+6+CFiUVnZZi/cnZNumQlxEJANdsSkiIpHTSFxEJI0eCiEiErTO3w8lbgpxEZGMFOIiImHq2mX3sVCIi4hkoDlxEZGAKcRFRAJlOrEpIhI2jcRFRAKmEBcRCZimU0REAqaRuIhIoHRiU0QkcBqJi4gETSEuIoVmVyM8uRFeq4WyBBw6CEb2aVuvvhGe2pSsBzC2L0wc9O7nD22Arbug0aF3CUwcCFW983MMnRRGhCvERSQbS9+EIoPTRsHbu+Cvr8GAUuhX1rre05ugsQlO3Q92NsLDG6CyBEb3TX4+eTD0K022tWln8vNTekF594mkUObEI38ohJklzOwZM7s36n2JSIQamqBme3LUXFIEQ8pheCW8tK1t3Q074KABUFyUHGmP7gtr33n38wFlyQCH5JC3CahtyMdRFJx8/Ng7H3gO6JuHfYlIVN7ZDWbQt/Tdsv6lsLGuc99/u7719l9fhdfroMnhfRUwsCzz92KjkThmVgV8GLg+yv2ISB40NCVH4C2VFCXL0+1bAau3wO4m2FafHIU3ptU7ZhicMRqO2Rf2LU/+gOhGLMtXXKKeTvkJ8HWSvyyJSMiKi5Kh3NLupmR5usOHQMLg3pfh0ddhvz5QkeEX/yKDYZXwWh3U7Iim312SbYTHF+ORTaeY2SnARnd/ysymt1PvXOBcgOoR1VF1R0Teq74l4J4cWfdJTam8XZ88QZmuLAFHve/d7WffgoG99t62O2zfndv+vgcW0EMhohyJTwNONbN1wALgODO7Ob2Su89z9ynuPmXw4MERdkdE3pPiouQywOWbk1Mob9YlT2COyrDEcNvu5HLEJodXd8CarTBhQPKzd+qTZQ1Nyc9f2pZsa2g7IS97FdlI3N2/CXwTIDUSv8Tdz45qfyKSB1OGwBMb4a6XkqPtKUOSyws31iVPVJ4xJllvy87kMsP6JuhTAkft8+4yRAdWboat9ckhb5+S5Ki9vZF6DHTFpogUnrIEfGjftuVDy98NcIARfZKvTPqVwkndf+pUId6Cuz8CPJKPfYmI9CQaiYuIZKATmyIiEjmNxEVE2jDNiYuIhE0hLiISpLgvpc+GQlxEJINQTmwqxEVEMlKIi4gEK4wIV4iLiOxFGDGudeIiIgHTSFxEpA0L5sSmRuIiIgHTSFxEJE1ynXgYI3GFuIhIRgpxEZFghRHhCnERkYxCObGpEBcRaSOcu6coxEVEMggjwrXEUERkLyzLVydaNJthZi+Y2Rozm5vh8zIzuy31+RNmNrKjNhXiIiLpLDknns2rwybNEsC1wExgPHCWmY1Pq/YFYIu77w/8GLiyo3YV4iIi+TEVWOPua929HlgAzE6rMxu4MfX+DuB46+AnhEJcRCTNnot9svnTCcOB9S22a1JlGeu4ewOwFRjUXqPd6sTmM08/s6l3Sd+X87S7wcCmPO0rn3Rc4SnUY8vnce2Xy8aefuqZxeXFlYOz/FovM1vaYnueu8/LZb8y6VYh7u5D8rUvM1vq7lPytb980XGFp1CPLeTjcvcZETS7AahusV2VKstUp/aQNVUAAALfSURBVMbMioF+wFvtNarpFBGR/FgCjDWzUWZWCswBFqbVWQh8JvX+dOAv7u7tNdqtRuIiIoXK3RvM7DxgMZAA5rv7KjO7HFjq7guB3wK/N7M1wGaSQd+unhzikc9VxSTW4zKzRmAFyX9bzwGfcffaLrZ1A3Cvu98B1JnZeHdfvZe604F6d/9HlvtYB0xx9zjnpPVvsYdw90XAorSyy1q83wmckU2bPXY6JR8nHOLQDY6rzt0nufsEoB74UssPU/N8WXP3o/cW4CnTgaO60nbcusF/s0gU6nF1Nz02xCUvHgP2N7PpZvaYmS0EVptZwsyuMrMlZrbczL4IYEm/SF3R9mdg6J6GzOwRM5uSej/DzJ42s2fN7KHUVW1fAi40s2VmdrSZDTGzO1P7WGJm01LfHWRmD5rZKjO7nnCurhbJqCdPp0iEUiPumcADqaLJwAR3f8nMzgW2uvv7zawM+LuZPQgcBhxI8mq2fYDVwPy0docA1wEfSrU10N03m9mvge3u/qNUvVuAH7v738xsBMl5yIOA7wB/c/fLzezDJK+QEwmWQlxyrdzMlqXeP0byRM1RwJPu/lKq/CTgEDM7PbXdDxgLfAi41d0bgVfN7C8Z2j8SeHRPW+6+eS/9OAEY3+Jit75m1ju1j4+mvnufmW3p4nGKdAsKccm1Onef1LIgFaQ7WhYBX3P3xWn1ZuWwH0XAkakTRel9ESkYmhOXOCwGvmxmJQBmdoCZVQKPAmem5sz3BY7N8N3HgQ+Z2ajUdwemyrcBfVrUexD42p4NM9vzg+VR4BOpspnAgJwdlUgMFOISh+tJznc/bWYrgd+Q/K3wbuDfqc9uAv6Z/kV3fxM4F7jLzJ4Fbkt9dA9w2p4Tm8D/BqakTpyu5t1VMt8j+UNgFclplVciOkaRvLAOLgYSEZFuTCNxEZGAKcRFRAKmEBcRCZhCXEQkYApxEZGAKcRFRAKmEBcRCZhCXEQkYP8Dl2ThtzTrqPoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zDdssYtVIC6u",
        "outputId": "3b1bf21c-c67f-4885-bd2e-04aa00a40902",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "only_not_helpful = train[train[\"Helpfulness\"] == 0]\n",
        "not_helpful_predictor, not_helpful_score, not_helpful_predictions, not_helpful_true_labels = train_and_score(*train_test_validate_split(only_not_helpful.TextClean.values, (only_not_helpful.Score.values - 1), sample_size=30000), model_name=\"only_not_helpful_real\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train size: 24,000\n",
            "test size: 6,000\n",
            "validate size: 3,610\n",
            "[0, 1, 2, 3, 4]\n",
            "Validation score: 0.6171745152354571\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pU5aMXHxP84c",
        "outputId": "8ec4cd1d-21eb-4354-970b-ea29cedf5b8d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "not_helpful_predictions = not_helpful_predictor.predict(test.TextClean.values)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jS5Ztkn5CJt4",
        "outputId": "f359b606-61cc-4c88-8061-fa7d3b62b2d0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "cm4 = confusion_matrix_custom(true_labels, not_helpful_predictions, \"only_not_helpful.pdf\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAE9CAYAAAD9HVKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3wV5Z3H8c8vhyQkIUEw3AREULRGVARExbbidaF1pVXratdau91lWy91e1frvZfVdrXtblltqtZLba1Ka7FF8YZrra0SxFqIpUWUuyIX5ZJASPLbP84Bk5NDkhPPZPKcfN+8zsszM8888wzgNw/PPDNj7o6IiISpIO4GiIhI1ynERUQCphAXEQmYQlxEJGAKcRGRgCnERUQC1utC3MymmdlSM1tmZpfH3Z5cMbM7zWy9mS2Ouy25ZGYjzWy+mdWa2RIzuyzuNuWCmfU1sxfN7M+p87o+7jblmpklzGyRmf027rbks14V4maWAGYB04Eq4Dwzq4q3VTlzFzAt7kZEoBH4srtXAccCF+fJn9lO4CR3PxIYD0wzs2NjblOuXQa8Gncj8l2vCnFgMrDM3Ze7ewNwPzAj5jblhLs/C2yKux255u7r3P2l1PetJENheLytev88aVtqsTD1yZs778xsBPBR4Pa425LveluIDwdWtVheTR4EQm9hZgcARwEvxNuS3EgNN7wMrAeecPe8OK+UHwBfA5rjbki+620hLoEys37AbOA/3H1L3O3JBXdvcvfxwAhgspmNi7tNuWBmpwPr3X1h3G3pDXpbiK8BRrZYHpFaJz2YmRWSDPD73P1Xcbcn19z9HWA++XNN43jgDDN7g+SQ5Ulm9rN4m5S/eluILwDGmtloMysCzgXmxNwmaYeZGXAH8Kq73xJ3e3LFzAaZ2T6p7yXAqcBf421Vbrj7Fe4+wt0PIPn/2NPufn7MzcpbvSrE3b0RuASYR/IC2QPuviTeVuWGmf0C+CNwiJmtNrPPxt2mHDke+BTJ3tzLqc9H4m5UDgwD5pvZKyQ7F0+4u6biSdZMj6IVEQlXr+qJi4jkG4W4iEjAFOIiIgFTiIuIBEwhLiLSTTp6UJ0l/XfqAX2vmNmEjurstSFuZjPjbkMUdF7hyddzy9fzep/uov2buqYDY1OfmcCtHVXYa0Oc5G9QPtJ5hSdfzy1fz6vLOvGguhnAPakHpP0J2MfMhrVXZ28OcRGRnibrh/T1ibQ5WUr0K/LCgSXdcqw+A/rSd//+3XKn0yGVo7rjMAAMHzmcIycc0W13cJl1Tz9gxMgRjJ84vtvOq6Ab+zcj9x/JhIkTuu3cmrvpwYLd+We2asUqNm7YaLmqzyr7Og1Z/j5t3bUE2NFiTbW7V+eqTXvTo0K8cGAJB3x5StzNyLnH/u3HcTchMkWJvnE3IRKlibK4mxCZ+qbtcTch506aclpuK2xohmMGZ7fPk2t2uPuk93nkrB/Sp+EUEZFMzLL75MYc4ILULJVjgXfdfV17O/SonriISI9gRNLFTT2obipQaWargWtJvtUJd78NmAt8BFgG1AGf6ahOhbiISCa5613v4e7ndbDdgYuzqVMhLiKSSe4zPBIKcRGRNnI6zh0phbiISLqIxsSjoBAXEclEPXERkYCFkeEKcRGRNgwoCCPFFeIiIpmEkeEKcRGRjDQmLiISsDAyXCEuItKGxsRFRAIXRoYrxEVE2tIdmyIi4QpoOCWQG0tFRCQT9cRFRDIJoyOuEBcRyUhj4iIiAQsjw/MnxJu2N/Dm/YvZvnQjibJCBp1+MBUT92tTbvVtNdQt37xn2ZuaKRpcxuivfxCAlT96kYZ1W/HGZgr3LWXf6QdRfviQbjsPEekBArqwGWmIm9k04IdAArjd3W+M6lhvPVSLJQo46JsnsmPNVtZUL6R4v3KKh5W3Kjfic61fRr3yf16gdOy+e5YHn3koxUPKsEQB9W+8w6pbF1By5Yfo0z8/3+ouInsRRoZHNzvFzBLALGA6UAWcZ2ZVURyreWcjW195i8qPjKWguA+lYwbQb9xgttSsbXe/XRvrqF++mf5Hv9dj77tfOZZI/bYY0OTsemdHFM0WkZ4snrfdZy3KnvhkYJm7Lwcws/uBGUBtrg/U8HYdVmAUDS7bs654v3LqXtvU7n7vLlhLyZgBFO5b2mr96uqF1P1tI97YTOkHKuk7sn+umywiPV0gE7CjDPHhwKoWy6uBY6I4UPPORgr6tj6VgpI+NO9obHe/LTVr2ffUMW3Wj5g5EW9qZvvSjTS8tQ0LZGxMRHIk5t51NmL/WWNmM82sxsxqmrY1dKmOguK2gd28o22wt1S3fDONW3ZSPn5o5nYlCuhXNYjtSzeybfH6LrVLRAJmWX5iEmWIrwFGtlgekVrXirtXu/skd5+U6FfUpQMVDSrFm52Gt7fvWbdzzVaKh/bb6z5bXlxD+RFDKCju4B8jzU7DhroutUtEAhbImHiUIb4AGGtmo82sCDgXmBPFgQqK+1B+xBA2zP07zTsbqVu+mW2L11Mxqe0UQ4Dmhia2vvwmFZOHt1q/861tbKt9m+aGJrypmXdr1lL32iZKDxwQRbNFpCcryPITk8jGxN290cwuAeaRnGJ4p7sviep4Q86u4s1fLGbZ1fNJlBYy5BNVFA9LXtxc/eOFHPzdU/eU3faXtygo6UPp2IFpjYaNjy1j7VvbMDOKBpWy36fH68KmSG9jBDMmHuk8cXefC8yN8hi7JcqKGP6vE9qsLz1wYKsAB6iYuF/GG4GKh/Zj1JeOi6yNIhKQMDI8f+7YFBHJqUBmpSnERUQy0XCKiEigYp42mI3Y54mLiEjXqScuItKGYVkOp3hELemIQlxEJAOFuIhIwAK5rqkQFxFJl3wnRHYp3hRNUzqkEBcRSWfZD6fERSEuIpKBQlxEJFjZz06Ji0JcRCSDQDJcIS4iki75EMMwUlwhLiKSThc2RUTCZoE8PEUhLiKSgXriIiIBCyTDFeIiIukMy/qOzbgoxEVEMghlOEXPExcRSZeanZLNp1PVmk0zs6VmtszMLs+wfX8zm29mi8zsFTP7SEd1KsRFRLqBmSWAWcB0oAo4z8yq0opdBTzg7kcB5wL/21G9CnERkQzMsvt0wmRgmbsvd/cG4H5gRloZBypS3/sDazuqtEeNiY/oX8l/nnFh3M3IuSWbF8fdhMiU9SmNuwmROGzg+LibEJmm5sa4m5B7nttXMnTxjs1KM6tpsVzt7tUtlocDq1osrwaOSavjOuBxM7sUKANO6eigPSrERUR6ii6E+AZ3n/Q+D3secJe732xmxwH3mtk4d2/e2w4KcRGRNiJ5iuEaYGSL5RGpdS19FpgG4O5/NLO+QCWwfm+VakxcRCRdNLNTFgBjzWy0mRWRvHA5J63MSuBkADM7FOgLvN1epeqJi4hkkOuOuLs3mtklwDwgAdzp7kvM7Aagxt3nAF8GfmJmXyR5kfNC9/YH/BXiIiJponoUrbvPBeamrbumxfda4Phs6lSIi4hkEModmwpxEZEM9OwUEZFQdf4GntgpxEVE0phelCwiEja92UdEJGDqiYuIBEwhLiISsEAyXCEuIpLOLJyeuJ6dIiISMPXERUTa0BRDEZGgKcRFRAIWSIYrxEVEMlFPXEQkUCHNTlGIi4hkoBCXnNmyeSu3fOWHLHz2JSoGVvDZyy/kpI9PbVPugVtn88RDT/HW6vX0H1jBP17wUc75/Fl7ti+pqeXWa3/CymWrGDpyCF/4zkWMm3xYN56JSDgCyXCFeAj+56r/pU9RHx54+T5eW7Kcb3z6OsZUjeaAQ0a1KufufO0HX2LMoaNZu2Idl3/yKgbtV8mJM05gy+atXH3hDVx248V8cPoU5j/8f1z9meu55w93UL5PeUxnJtJThTPFMLKbfczsTjNbb2aLozpGb1Bft4Pn5j7PhV/9FCVlJYybfBjHnXoMT85+uk3Zf7robMYefhCJPglGHjiCKacdy5IFtQDULnyVgYMHcMLpHyKRSHDKWSfRf2B/nnv0+e4+JZEgRPCi5EhEecfmXcC0COvvFdYsX0MikWDEmOF71h1YNZoVf1vZ7n7uzl9eXMKoFr319PetusMbS1fktsEiecCiedt9JCILcXd/FtgUVf29Rf32ekrLS1qtKysvo25bfbv73XPzfXiz8w/nnApA1cQPsPGtTTz98DM07mrk8QefZN2Kdeys3xlZ20VCZpbdJy56dkoPV1JWQt3W1oG9fVsdpf1K9rIHPPzTR3hy9tN86+7rKCouBKBiQAXX33E1s3/yMOeM/2cWPLOQoz40nsphlZG2XyRUofTEY7+waWYzgZkAg/ZToKQbPmY4TU1NrF6+Zs+QyvLa1xl18P4Zyz92/+P8ctaD3DL7pja/n0cedzizfvcDAJoam/jUlM9y9syPR3sCIqHq7Rc2O8vdq919krtPqhhYEXdzepyS0r58cPoU7r75Z9TX7WDxglqef/xPnHLWSW3KPvWr+dx5093c+ItvMWzUsDbbly1+jcZdjWzfWsePv3kHg/ar5OipE7vjNEQCk10vvFf3xKVjl377Im7+yg8458hPUj6ggsu+czEHHDKKv7ywmCs/dS2P/G02AHd97162bN7KJR/94p59Tz7zRP7jxksA+OWtD/Hi0zUAHD11ItfdflX3n4xICPS2ezCzXwBTgUozWw1c6+53RHW8fFYxoJzr77i6zfrDjxm3J8AB7v3jne3W841ZX89520TykaE7NnH386KqW0QkaqGEeOxj4iIi0nUaExcRySCUnrhCXEQkg0AyXCEuItJGzNMGs6EQFxFJo9kpIiKBU4iLiARMIS4iEirdsSkiEjb1xEVEAmUBvZ5NIS4ikoFCXEQkYIFkuEJcRKQNU09cRCRsCnERkXCpJy4iEigDCsLIcIW4iEhb4Uwx1EshREQCphAXEUlnUGCW1adT1ZpNM7OlZrbMzC7fS5lzzKzWzJaY2c87qlPDKSIiaaJ4FK2ZJYBZwKnAamCBmc1x99oWZcYCVwDHu/tmMxvcUb3qiYuIZFCQ5acTJgPL3H25uzcA9wMz0sr8GzDL3TcDuPv6zrRTRETSRDCcMhxY1WJ5dWpdSwcDB5vZH8zsT2Y2raNKe9RwSllhGccNmRJ3M3Ku0RvjbkJk7lt6f9xNiMSw0v3ibkJkmrwp7ibkXK7PqYvDKZVmVtNiudrdq7Osow8wFpgKjACeNbPD3f2d9nYQEZFWOn+xsoUN7j6pne1rgJEtlkek1rW0GnjB3XcBr5vZ30iG+oK9VarhFBGRdKlnp2Tz6YQFwFgzG21mRcC5wJy0Mg+T7IVjZpUkh1eWt1epeuIiImmM3Pdw3b3RzC4B5gEJ4E53X2JmNwA17j4nte00M6sFmoCvuvvG9upViIuIZNCF4ZQOuftcYG7aumtafHfgS6lPpyjERUQyCOW2e4W4iEia5AOwFOIiIsEKI8IV4iIiGXRpimEsFOIiImnMNJwiIhI0XdgUEQlYKD1x3bEpIhIw9cRFRNIYmp0iIhK0UIZTFOIiIm1oiqGISLDMNDtFRCRoofTEO5ydYknnm9k1qeX9zWxy9E0TEYmPZfmJS2emGP4vcBxwXmp5K8k3NouI5KXdD8DK8Ts2I9GZ4ZRj3H2CmS0CcPfNqbdSiIjkrVCGUzoT4rvMLAE4gJkNApojbZWISKw6/cq12HVmOOW/gV8Dg83s28BzwHcibZX0CnXv1nHPl3/G1cdfy40f/S4vP/pyu+UbdzVy81nf5zvTb8y4feFvX+LyiVfy4q/3+k5ZkU7Z/Xq2bD5x6bAn7u73mdlC4GSS5/Yxd3+1o/3MbCRwDzCEZC++2t1/+D7bK3nkNzfNoU9hgqueuJJ1S9fx08vuZtjBwxhy4JCM5Z+95/eUDShjZ93ONtvqttTzzJ3PMOTAwVE3W3qDgKYYdmZ2yv5AHfAIyTczb0+t60gj8GV3rwKOBS42s6r301jJHw31DSx+agmnfv5UikuLOeCoA6g64VBe+t2ijOU3rdnEorkvc+JnTsi4fd6P5jHl3CmU7lMWZbOlFwnlwmZn/hXwO+C3qf8+BSwHHu1oJ3df5+4vpb5vBV4Fhne9qZJP3l6xgYJEAYNGVe5ZN2zsMN5avj5j+TnffYR/uPg0+hQXttm2avEqVteu4ZizNfNVciOvZqe4++Etl81sAnBRNgcxswOAo4AXstlP8ldD/U6K+xW3Wte3X9+MQyWLn15Cc7Mz7qTDeK1meattzU3NPHzjHGZ8/R8pKNBDOSV38mY4JV2qd31MZ8ubWT9gNvAf7r4lw/aZZlZjZjUbN2zKtjkSqKKSYnZuax3YO7bvoLi0dbA31Dfw6H8/xhlfPT1jPX988E8MHTuU/Q/vzAifSGcZBVl+4tJhT9zMvtRisQCYAKztTOVmVkgywO9z919lKuPu1UA1wJETjvDO1CvhGzSqkuamZjas3EDl/skhlXV/f5MhY1pfmNywcgOb127mtn+tBqBpVxM7tu3gW6d9h4vu+jyvvfgay196naV/WApA/bv1rP3rWtb9bR0zvn5G956U5JVQeuKdmSde3uJ7I8mx8dkd7WTJ34E7gFfd/ZauNU/yVVFJEYedVMUTtz3JWVefydql66h9ppaLfvq5VuWGHDiEK+Z+fc/yildW8JubHuEL911C2YAyPnH92TTubNyz/d6v3sfhJ4/j6BmTuu1cROLUboinbvIpd/evdKHu44FPAX8xs90TgK9097ldqEvy0Mcun8FD18/mm6d8m9L+pXz8ihkMOXAIry96nZ9eejc3PHcdiT4Jyivf60eUVJRiBbZnXUl5SatuRqIwQXFZMX3L+3bz2Ug+yYsXJZtZH3dvNLPju1Kxuz9HOC/HkBiU9i/lgls+1Wb96KNGc8Nz12Xc58BJY7jy0cv3Wue/V/9bjlonvZ0FEl/t9cRfJDn+/bKZzQEeBLbv3ri3MW4RkXyQT2PifYGNwEkk77y01H8V4iKSlyxP3uwzODUzZTHvhfdumkUiInnNYn0iSue1F+IJoB+Zx7UV4iKS1/KhJ77O3W/otpaIiPQg+TAmHsYZiIjkmKV+haC9ED+521ohItKT5MM8cXfXg0xEpNfKh+EUEZFeKflmn/Bnp4iI9FLhvGNTIS4ikoFCXEQkYHE+IzwbCnERkTSGeuIiIuEKaIphGJdfRUQkI/XERUTayI87NkVEeiUDCiyMgQqFuIhIBqFc2AzjR42ISDezLH91qk6zaWa21MyWmdle3zNoZmeZmZtZh2/8Vk9cRKSN3L/ZJ/Xi+VnAqcBqYIGZzXH32rRy5cBlwAudqVc9cRGRNEYkPfHJwDJ3X+7uDcD9wIwM5b4J3ATs6EylCnERkQwKzLL6dMJwYFWL5dWpdXuY2QRgpLv/rrPt7FHDKQlL0K+wIu5m5FyBJeJuQmTOP+S8uJsQidrNS+JuQmR++dd5cTch597c/nZuKzSw7GenVJpZTYvlanev7vQhkwe8Bbgwm4P2qBAXEekZujRPfIO7t3chcg0wssXyiNS63cqBccAzqZkxQ4E5ZnaGu7f84dCKQlxEJE1ynnjOpxguAMaa2WiS4X0u8MndG939XaByTxvMngG+0l6Ag0JcRCSjXM8Td/dGM7sEmAckgDvdfYmZ3QDUuPucrtSrEBcRySCKR9G6+1xgbtq6a/ZSdmpn6lSIi4ik0aNoRUSCZl2ZnRILhbiISAZ6s4+ISKDMNJwiIhK0UJ4nHsagj4iIZKSeuIhIG6bhFBGRkOnCpohIoJLzxMMYbVaIi4i0oRcli4gETWPiIiIBU09cRCRg6omLiATK0OwUEZFwmeaJi4gEzQK5oV0hLiKSgXriIr3Uls1bueWrP2Ths4voP7CCf/n6pznp41PblHvgttk88eBTrF/zNhUDK/jHCz7COZ87a8/2JTWvctt11axctpqhI4dw6bc/z7jJh3XjmfRehmanYGZ9gWeB4tRxHnL3a6M6nkhP8aOrbqWwsJAHFv2M15Ys56oLr2dM1WgOOGRU64LufO0HX2LMoaNZu2IdV/zz1QwaVsmJM05gy+atXPOZG7jsPy/m+OnHMf83z3LNv9zA3c/dQfk+/eI5sV7FonhRciSiHPTZCZzk7kcC44FpZnZshMcTiV193Q6ee/R5Pv3V8ykpK2Hc5MM47tRjeOpX89uUPefzZzP28INI9Ekw8sARHHfasdTWvApA7cJXGTh4AB8+/YMkEglOOfNE+g/sz3OPPd/dp9RrWZa/4hJZiHvSttRiYerjUR1PpCdYs3wNiUSCEWOG71k35tDRvPG3Fe3u5+4sfnEJow7ev9W69DJv/LX9eiR3LDVDpbOfuER6+dXMEmb2MrAeeMLdX4jyeCJxq99eT2l5Sat1ZRWl1G+rb3e/e2/5Od7czGnnnApA1cQPsPGtTcx/+P9o3NXI4w8+xboVb7Jzx87I2i7vSY6JF2T1iUukR3b3JncfD4wAJpvZuPQyZjbTzGrMrGbDho1RNkckciVlJdRtbR3YdVvrKOlXspc94Dd3PcITs5/mm3dfR1FxIQAVAyq4/varmP2Th/mno86n5pmFHPXB8VQO3TfS9kt4umV2iru/Y2bzgWnA4rRt1UA1wISJR2m4RYI2fMxwmpqaWPP6GoaPTg6pLH/1dQ44eFTG8o/d/zi/nPUQNz90E4OGVbbadsRxh/Oj330fgKbGJi44/l85e+bHoj0BSQnnZp/IeuJmNsjM9kl9LwFOBf4a1fFEeoKS0r4cP+047v6v+6iv28GSBbU8//gLnHzmiW3KPvXr+fz0u/dw48+/xbBRQ9tsX7b4NRp3NbJ9ax3V37qDQftVMmnqxO44DYEsB1PiC/woe+LDgLvNLEHyh8UD7v7bCI8n0iNc+u2LuPkrP+Sc8f9MxYAKvvDtizjgkFH85YXFfOOC65iz9CEA7vrez9iyeSuXnP7FPfuefOZULvvPSwB44NbZvDi/BoCjT5jItT/5RvefTG+lt92Du78CHBVV/SI9VcWAcq6/46o26w8/ZtyeAAe49/k72q3nyllfy3nbpHN0s4+ISOB6fU9cRCRcpgdgiYiELJTb7hXiIiJpNCYuIhI4jYmLiAQr3odaZUMhLiKSgXriIiKBSr4oWbNTRETCpDs2RURCpjFxEZGgqScuIhKwUHriYYzci4hIRuqJi4ik0R2bIiKh05i4iEioNDtFRCRomp0iIhIw9cRFRAIWSohriqGISBojOZySzadT9ZpNM7OlZrbMzC7PsP1LZlZrZq+Y2VNmNqqjOhXiIiJtWNa/OqzRLAHMAqYDVcB5ZlaVVmwRMMndjwAeAr7bUb0KcRGRDHId4sBkYJm7L3f3BuB+YEbLAu4+393rUot/AkZ0VGkPGxM3zPLv50pBHp7TbvsUDYy7CZE4evCxcTchMqdd++W4m5B7m97NbX1de4phpZnVtFiudvfqFsvDgVUtllcDx7RT32eBRzs6aA8LcRGRnqELFzY3uPuknBzb7HxgEnBCR2UV4iIiaXZf2MyxNcDIFssjUutaH9vsFOAbwAnuvrOjSvP33/kiIl2W+wubwAJgrJmNNrMi4FxgTqujmh0F/Bg4w93Xd6ZS9cRFRDLI9Txxd280s0uAeUACuNPdl5jZDUCNu88Bvgf0Ax5M/Utgpbuf0V69CnERkQyiuO3e3ecCc9PWXdPi+ynZ1qkQFxHJQHdsiohI5NQTFxFJo5dCiIgErfPPQ4mbQlxEJCOFuIhImLp2230sFOIiIhloTFxEJGAKcRGRQJkubIqIhE09cRGRgCnERUQCpuEUEZGAqScuIhIoXdgUEQmceuIiIkFTiItIvtnZBL9/E9Zuh+IETBoEB1ZkLvfCeli9Pbn8gX1gQuV72xdugJVb4Z0GOHLf1tt6iDAiXCEuItl4/i1IGJx3EGzcAU+sgYHFMKC4dbkX10NjM5wzBuqb4LFV0K8QDu6f3F5RmPwBsPTd7j+HTgplTDzyl0KYWcLMFpnZb6M+lohEaFczrNia7DUXFsDQUti/Hyzb0rbsyu1w+EDoUwDlqfD+e4vAHtsfRvaDwjCCsifrjjf7XAa82g3HEZEobWkAM+hf9N66gcXwzs6O93VgcyfK9SiW5ScekYa4mY0APgrcHuVxRKQb7GqGorTIKCpIrk83ohRe2ZTctqUh2Qtv9O5pZ46EEeHR98R/AHwNyPCnLCJBKSyAhrT/lRuak+vTHTskOXb+0HJ4cg2MKYeykC7BZRvh8cV4ZL+rZnY6sN7dF5rZ1HbKzQRmAozcf2RUzRGR96uiCNzh3Yb3hlQ27YR9ituWLU7A1P3eW655Gyr7dk87c8ACeilElD3x44EzzOwN4H7gJDP7WXohd69290nuPqmysudNMxKRlMICGFUOL21IDpO8VQcrt8FBGaYYbmmAHU3Q7LBqW3IWyvh939ve7MnZK07yB0Njc3KdZC2ynri7XwFcAZDqiX/F3c+P6ngi0g2mDEnOE//FsmRve8qQ5PTCN+vg8dVwwcHJcht2JOeJNzQne+0nDGs9DfG5N1vPavnzJvjQ0OSslR5Cd2yKSP4pTsApw9uuH1r6XoADjKlIfvbmw8OSnx5MId6Cuz8DPNMdxxIR6U3UExcRyUAXNkVEJHLqiYuItGEaExcRCZtCXEQkSHHfSp8NhbiISAahXNhUiIuIZKQQFxEJVhgRrhAXEdmLMGJc88RFRAKmnriISBsWzIVN9cRFRAKmnriISJrkPPEweuIKcRGRjBTiIiLBCiPCFeIiIhmFcmFTIS4i0kY4T09RiIuIZBBGhGuKoYjIXliWn07UaDbNzJaa2TIzuzzD9mIz+2Vq+wtmdkBHdSrERUTSWXJMPJtPh1WaJYBZwHSgCjjPzKrSin0W2OzuBwHfB27qqF6FuIhI95gMLHP35e7eANwPzEgrMwO4O/X9IeBk6+AnhEJcRCTN7pt9svnVCcOBVS2WV6fWZSzj7o3Au8C+7VXaoy5sLnpp0YZ+hRUruulwlcCGbjpWd9J5hSdfz607z2tULit7aeGieSV9yiqz3K2vmdW0WK529+pctiuTHhXi7j6ou45lZjXuPqm7jtdddF7hyddzC/m83H1aBNWuAUa2WB6RWpepzGoz6wP0Bza2VyhCDGcAAALXSURBVKmGU0REuscCYKyZjTazIuBcYE5amTnAp1Pfzwaedndvr9Ie1RMXEclX7t5oZpcA84AEcKe7LzGzG4Aad58D3AHca2bLgE0kg75dvTnEIx+rikms52VmTcBfSP7dehX4tLvXdbGuu4DfuvtDQL2ZVbl77V7KTgUa3P35LI/xBjDJ3eMck9bfxV7C3ecCc9PWXdPi+w7gE9nU2WuHU7rjgkMcesB51bv7eHcfBzQAn2u5MTXOlzV3/9DeAjxlKjClK3XHrQf8mUUiX8+rp+m1IS7d4vfAQWY21cx+b2ZzgFozS5jZ98xsgZm9Ymb/DmBJP0rd0fYkMHh3RWb2jJlNSn2fZmYvmdmfzeyp1F1tnwO+aGYvm9mHzGyQmc1OHWOBmR2f2ndfM3vczJaY2e2Ec3e1SEa9eThFIpTqcU8HHkutmgCMc/fXzWwm8K67H21mxcAfzOxx4CjgEJJ3sw0BaoE70+odBPwE+HCqroHuvsnMbgO2uft/pcr9HPi+uz9nZvuTHIc8FLgWeM7dbzCzj5K8Q04kWApxybUSM3s59f33JC/UTAFedPfXU+tPA44ws7NTy/2BscCHgV+4exOw1syezlD/scCzu+ty9017accpQFWLm90qzKxf6hhnpvb9nZlt7uJ5ivQICnHJtXp3H99yRSpIt7dcBVzq7vPSyn0kh+0oAI5NXShKb4tI3tCYuMRhHvB5MysEMLODzawMeBb4p9SY+TDgxAz7/gn4sJmNTu07MLV+K1DeotzjwKW7F8xs9w+WZ4FPptZNBwbk7KxEYqAQlzjcTnK8+yUzWwz8mOS/Cn8N/D217R7gj+k7uvvbwEzgV2b2Z+CXqU2PAB/ffWET+AIwKXXhtJb3ZslcT/KHwBKSwyorIzpHkW5hHdwMJCIiPZh64iIiAVOIi4gETCEuIhIwhbiISMAU4iIiAVOIi4gETCEuIhIwhbiISMD+H6IZ8vrU3wJ4AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MIAdT2KaIUNL",
        "outputId": "ef22bf5f-f619-41d0-c45f-250b30bb0195",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "only_helpful = train[train[\"Helpfulness\"] == 1]\n",
        "helpful_predictor, helpful_score, helpful_predictions, helpful_true_labels = train_and_score(*train_test_validate_split(only_helpful.TextClean.values, (only_helpful.Score.values - 1), sample_size=30000), model_name=\"only_helpful_real\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train size: 24,000\n",
            "test size: 6,000\n",
            "validate size: 82,587\n",
            "[0, 1, 2, 3, 4]\n",
            "Validation score: 0.7541743857992178\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BCVkwWHlQD0x",
        "outputId": "e3b09352-093f-4559-e9e2-c9c127e08315",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "helpful_predictions = helpful_predictor.predict(test.TextClean.values)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s4JsgADsCUqS",
        "outputId": "791747e4-eded-4e42-a9e8-cddf8e9f3a89",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "cm5 = confusion_matrix_custom(true_labels, helpful_predictions, \"only_helpful.pdf\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAE9CAYAAAD9HVKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1f3/8dcnk0AgCYssyhIWBVTcgEbEpe62aP1KF63oV7tpab9qa6v+rFVrW/za1rZWbX/Ulqq1i3Wp2kqVVmtd2yqCGxWQNoILEQVEBBKSkOTz/WOGmEyGJBPn5uZM3k8e83DumTPnnov6zsm5595r7o6IiISpIO4OiIhI1ynERUQCphAXEQmYQlxEJGAKcRGRgCnERUQC1utC3MxmmtlKM6s0s0vi7k+umNnNZrbOzF6Muy+5ZGblZvaImS03s2Vmdn7cfcoFMys2s6fN7IXUcX077j7lmpklzOw5M7sv7r7ks14V4maWAOYBxwOTgdPMbHK8vcqZW4CZcXciAg3Ahe4+GZgBnJsn/87qgKPd/QBgCjDTzGbE3KdcOx9YEXcn8l2vCnFgOlDp7qvcvR64HZgVc59ywt0fBzbG3Y9cc/e17v5s6v0WkqEwKt5evX+etDW1WZR65c2Vd2Y2GvgIcGPcfcl3vS3ERwGvt9heQx4EQm9hZuOAqcCieHuSG6nphueBdcBf3T0vjivlOuBioCnujuS73hbiEigzKwXuBr7i7pvj7k8uuHuju08BRgPTzWzfuPuUC2Z2IrDO3Z+Juy+9QW8L8SqgvMX26FSZ9GBmVkQywG9193vi7k+uufsm4BHy55zGocBJZvYKySnLo83st/F2KX/1thBfDEw0s/Fm1geYDSyIuU/SDjMz4CZghbv/KO7+5IqZDTOzQan3/YDjgJfi7VVuuPvX3X20u48j+f/Yw+5+Rszdylu9KsTdvQE4D3iA5AmyO919Wby9yg0zuw14EtjTzNaY2Vlx9ylHDgXOJDmaez71OiHuTuXACOARM1tKcnDxV3fXUjzJmulWtCIi4epVI3ERkXyjEBcRCZhCXEQkYApxEZGAKcRFRLpJRzeqs6Qfp27Qt9TMpnXUZq8NcTObE3cfoqDjCk++Hlu+Htf7dAvtX9R1PDAx9ZoD3NBRg702xEn+BeUjHVd48vXY8vW4uqwTN6qbBfw6dYO0p4BBZjaivTZ7c4iLiPQ0Wd+krzDS7mSpsLSPFw3p3y37KtqlH/3GDuqWK532Gjq+O3YDQPmY0Uz9wNRuu4KryRu7ZT+jykdxwLT9u+24EtZ9/2uUjyln2gemdduxNfj2btnPqPKR7D9tv245rjWvVbFxw0bLVXs2tNipz/IGjFu2LwNqW5TMd/f5uerTzvSoEC8a0p8JX/tg3N3IucfO/nXcXYhMTWN13F2IxICiQXF3ITIb69bH3YWcO+GDJ+W2wfomOGh4dt95qKrW3Sve556zvkmfplNERDIxy+6VGwuAT6VWqcwA3nX3te19oUeNxEVEegQjkiFu6kZ1RwJDzWwN8E2ST3XC3X8GLAROACqBGuCzHbWpEBcRySR3o+tm7n5aB587cG42bSrERUQyyX2GR0IhLiLSRk7nuSOlEBcRSRfRnHgUFOIiIploJC4iErAwMlwhLiLShgEFYaS4QlxEJJMwMlwhLiKSkebERUQCFkaGK8RFRNrQnLiISODCyHCFuIhIW7piU0QkXAFNpwRyYamIiGSikbiISCZhDMQV4iIiGWlOXEQkYGFkeP6EeEN1PVW3vsDWFRsoLOnDrrP2YtCBo9rUe2XeImoqNzZve2MTfXYtZeJlRwDw1p9Wsnnpm9S9uZVhMyew60f27LZjEJEeIqATm5GGuJnNBK4HEsCN7v69qPa19o4XsUQBe333OGrXbObVG56meNQAikeWtao37tyDWm2vuu6flE4a2rzdZ1h/dvvo3mx84tWouioiIQgjw6NbnWJmCWAecDwwGTjNzCZHsa+mugY2P7+WXU/ck0RxISUTdqFsv13Z9PSadr9X/3YNNZUbGXTQ6OaywTPKKdtnOAXFefNLioh0RTxPu89alEsMpwOV7r7K3euB24FZUeyobl01FBh9dy1tLus3egB1a7e0+71Ni9bQf8Iu9BnSP4puiUjICrJ8xSTKXY8CXm+xvSZVlnNNdQ0kiotalRX0K6SxrqHd721atIbBB5VH0SURCVm2o/AYR+KxzxmY2RxgDkDRLv261EZB30Iaa7e3Kmva1kCi784Pr7pyIw2b6xgwdUSX9ikiea63z4kDVUDLYe7oVFkr7j7f3SvcvSJR2qdLO+o7vASanLp1W5vLaqs203dE2U6/s2nR6wyYshsJzX2LSCaBjMSjDPHFwEQzG29mfYDZwIIodlTQt5ABU0aw7r5/01TXQPXLG9m89C0GTR+dsX5TfSPvPruWQTPaTqV4YxNN2xvBHZqcpu2NeJNH0W0R6ckCmROPbBjq7g1mdh7wAMklhje7+7Ko9jfi1H2p+u0LrLjkrxSWFDFy9n4UjyyjuvJtXp33NJOvPb657ualb5LoV0TJpCFt2qm6dSmbFr23qmX9XyoZdcYBDD5Yc+civYahKzYB3H0hsDDKfexQWNKHsV84sE15yYQhrQIcYFDFKAZVZD7HOvpTUxj9qSmR9FFEAhJGhsd/YlNEpEfSFZsiIgHTdIqISKCMYKZT9FAIEZGAaSQuItKGYVlOp8S1EFkhLiKSgUJcRCRggZzXVIiLiKRLPhMiuxRvjKYrHVKIi4iks+ynU+KiEBcRyUAhLiISrOxXp8RFIS4ikkEgGa4QFxFJl7yJYRgprhAXEUmnE5siImGzQG6eohAXEclAI3ERkYAFkuEKcRGRdIZlfcVmXBTiIiIZhDKdovuJi4ikS61OyebVqWbNZprZSjOrNLNLMnw+xsweMbPnzGypmZ3QUZsKcRGRbmBmCWAecDwwGTjNzCanVbscuNPdpwKzgZ921K5CXEQkA7PsXp0wHah091XuXg/cDsxKq+PAgNT7gcAbHTXao+bEdysbyIXH/Ffc3ci5moatcXchMs9sWBJ3FyJx8K6Hxd2FCMX1+IJwdPGKzaFm1vJ/iPnuPr/F9ijg9Rbba4CD0tr4FvCgmX0JKAGO7WinPSrERUR6ii6E+AZ3r3ifuz0NuMXdrzGzg4HfmNm+7t60sy8oxEVE2ojkLoZVQHmL7dGpspbOAmYCuPuTZlYMDAXW7axRzYmLiKSLZnXKYmCimY03sz4kT1wuSKvzGnAMgJntDRQD69trVCNxEZEMcj0Qd/cGMzsPeABIADe7+zIzmwsscfcFwIXAL8zsqyRPXnzG3ds9iaEQFxFJE9WtaN19IbAwreyKFu+XA4dm06ZCXEQkg1Cu2FSIi4hkoHuniIiEqvMX8MROIS4iksb0oGQRkbDpyT4iIgHTSFxEJGAKcRGRgAWS4QpxEZF0ZuGMxHXvFBGRgGkkLiLShpYYiogETSEuIhKwQDJcIS4ikolG4iIigQppdYpCXEQkg1BCXEsMA/DOxk18dvbn2X3YnlTsdTD33PHHjPX+8dg/+cTxpzJpxD4cuPchbT6/eu4POerA4xg9YDw/vOpHUXdbJGhm2b3iohAPwKVfvZw+fYr41+pnmXfz9VzylctYuXxlm3r9S/oz+8xP8o2rLs3Yzvjdx3L5/17KsTOPjrrLIoHL7vmacY7aIwtxM7vZzNaZ2YtR7aM3qKmu4f57/8zF37iIktISDjpkOh864Vjuuu2eNnWnVkzhlNM/wdhxYzK29ckzTuGYDx9FSWlp1N0WCV6vD3HgFmBmhO33Ci//ZxWJwgR7TNy9uWyf/SazcsW/Y+yVSH7bcWIzhBCP7MSmuz9uZuOiar+3qK6upqysrFVZ2cAytm6tjqlHIr1DIOc1tTqlpyspKWHLli2tyrZu3kppaUlMPRLpHbQ6pZPMbI6ZLTGzJVve2Rp3d3qcPSbuTmNDI6sqVzeXLfvXcvbce1KMvRLpBQJZnhJ7iLv7fHevcPeKssE64Zauf0l/Tpg1kx9ceQ011TU8/eRiHrj/r5x82sfb1G1qaqK2tpbt2xtwd2pra6mvr2/+fPv27dTW1uJNTTQ0NFJbW0tjY2N3Ho5IILQ6RXLou9dexbbaWvYdN5X/+cyX+N51V7Hn5D156h+L2GP4Xs31nvr7IsYPmcQZH/80Va9XMX7IJGafdEbz5xed+zXGD5nEH35/L9d//yeMHzKJu37XdpWLSK+X5SA8zpmXyObEzew24EhgqJmtAb7p7jdFtb98NniXQdxyx41tymccehAvr3upefuQww9mbfVrO23n+vk/4vr5ushHpCNGOHPiUa5OOS2qtkVEohZKiGs6RUQkYFpiKCKSQSgjcYW4iEgGgWS4QlxEpI2Ylw1mQyEuIpJGq1NERAKnEBcRCZhCXEQkVDFfhZkNhbiISAYaiYuIBMrQ6hQRkaApxEVEAhZIhivERUTaMI3ERUTCphAXEQmXRuIiIoEyoCCMDFeIi4i0Fc4SQz0UQkQkYApxEZF0BgVmWb061azZTDNbaWaVZnbJTup80syWm9kyM/tdR21qOkVEJE0Ut6I1swQwDzgOWAMsNrMF7r68RZ2JwNeBQ939HTMb3lG7GomLiGRQkOWrE6YDle6+yt3rgduBWWl1Pg/Mc/d3ANx9XWf6KSIiaSKYThkFvN5ie02qrKVJwCQz+4eZPWVmMztqtEdNpwzoM5APl3fY5+AUF/aPuwuRWf3ua3F3IRIj+lfG3YXI9E0Ux92FnGv0xpy218XplKFmtqTF9nx3n59lG4XAROBIYDTwuJnt5+6b2vuCiIi00vmTlS1scPeKdj6vAspbbI9OlbW0Bljk7tuB1Wb2b5KhvnhnjWo6RUQkXereKdm8OmExMNHMxptZH2A2sCCtzh9JjsIxs6Ekp1dWtdeoRuIiImmM3I9w3b3BzM4DHgASwM3uvszM5gJL3H1B6rMPmdlyoBH4f+7+dnvtKsRFRDLownRKh9x9IbAwreyKFu8duCD16hSFuIhIBqFcdq8QFxFJk7wBlkJcRCRYYUS4QlxEJIMuLTGMhUJcRCSNmaZTRESCphObIiIBC2Ukris2RUQCppG4iEgaQ6tTRESCFsp0ikJcRKQNLTEUEQmWmVaniIgELZSReIerUyzpDDO7IrU9xsymR981EZH4WJavuHRmieFPgYOB01LbW0g+sVlEJC/tuAFWjp+xGYnOTKcc5O7TzOw5AHd/J/VUChGRvBXKdEpnQny7mSUABzCzYUBTpL0SEYlVpx+5FrvOhPiPgT8Aw83sKuBk4PJIeyW9QvW71fzum7fz0j9XUjK4hJO+fCIVH/nATus3bG/geyd/n7rqOq586NvN5bd9+w4ql1Sy/rUNnD53NjNmHdQd3Zc8FsXj2aLSYYi7+61m9gxwDMlj+6i7r+joe2ZWDvwa2JXkKH6+u1//PvsreeTOq+4iUZTgO49eyZqXqvjZefMZtedIRkwYkbH+3375MKWDS6mrrmtVPmrPkUz78FTuve5P3dFt6Q0CWmLYmdUpY4Aa4E8kn8xcnSrrSANwobtPBmYA55rZ5PfTWckfdTV1vPDQUk489wT69u/LHtN2Z78j9+Xp+5ZkrL9hzdssvn8Jx511bJvPDp/9QfacMYmivloxK7mTTyc27yc5kjagGBgPrAT2ae9L7r4WWJt6v8XMVgCjgOXvp8OSH9a9up6CwgKGjxveXDZq0kgql7ycsf5d37ub//ryifQpLuquLkovFtLj2Tocibv7fu6+f+qfE4HpwJPZ7MTMxgFTgUVd6aTkn7qaOopLiluVFZf2o7amtk3dF/62lKbGJg44Zv/u6p4IZpbVKy5Zz927+7NAp88cmVkpcDfwFXffnOHzOWa2xMyWvL1hY7bdkUD17d+X2urWgV1bXUtx/9bBXldTx73XLuDkSz7Rnd2TXs8oyPIVlw6nU8zsghabBcA04I3ONG5mRSQD/FZ3vydTHXefD8wHOGDa/t6ZdiV8w8cOo6mhiXWvrmf42GEAVK2sYrcJu7Wqt/619bz9xkau+8yPAWjc3si2rdu49KhvcOFvv8KQUUO6ve/SO4RyYrMzc+JlLd43kJwjv7ujL1nyb+AmYIW7/6hr3ZN81bd/Xw44dn/un7eQ0781m6qVVfzr0Re54Nfnt6o3YsIIrnzwW83bq55fze+/ezdfu+MiSgeXAsmlh97kuEPj9ia2120nUZSgoCCURWIiXdduiKcu8ilz94u60PahwJnAv8zs+VTZpe6+sAttSR765GUnc+sVt3Hpkd+gZFB/Tr3sFEZMGEHlMy9zwzk/55pF3ydRmGDA0AHN3ykZ2J8Cs1Zl875wQ/MJ0dXPr+b2uXfw5ZvOZeKBE7v9mCQ/5MWDks2s0N0bzOzQrjTs7n8nnIdjSAxKBpYw5/qz25RP+MAeXLPo+xm/M/HAia0u9AE4/+YvRdI/6d0skPhqbyT+NMn57+fNbAHwe6B6x4c7m+MWEckH+TQnXgy8DRzNe+vFHVCIi0hesjx5ss/w1MqUF3kvvHfQKhIRyWsWyN1T2gvxBFBK5nlthbiI5LV8GImvdfe53dYTEZEeJB/mxMM4AhGRHLPUnxC0F+LHdFsvRER6knxYJ+7uupGJiPRa+TCdIiLSKyWf7BP+6hQRkV4qv56xKSLS6yjERUQCFuc9wrOhEBcRSWNoJC4iEq6AlhiGcfpVREQy0khcRKSN/LhiU0SkVzKgwMKYqFCIi4hkEMqJzTB+1IiIdDPL8k+n2jSbaWYrzazSzC5pp94nzMzNrKKjNjUSFxFpI/dP9kk9eH4ecBywBlhsZgvcfXlavTLgfGBRZ9rVSFxEJI0RyUh8OlDp7qvcvR64HZiVod6VwNVAbWcaVYiLiGRQYJbVqxNGAa+32F6TKmtmZtOAcne/v7P97FHTKQlLMKBoUNzdyLlEQY/6a86pUyZ8Iu4uRGLRW0/G3YXI3P3vx+LuQs69Vb0htw0aWParU4aa2ZIW2/PdfX6nd5nc4Y+Az2Sz0/xNFxGRLuvSOvEN7t7eicgqoLzF9uhU2Q5lwL7Ao6mVMbsBC8zsJHdv+cOhFYW4iEia5DrxnC8xXAxMNLPxJMN7NnD6jg/d/V1gaHMfzB4FLmovwEEhLiKSUa7Xibt7g5mdBzwAJICb3X2Zmc0Flrj7gq60qxAXEckgilvRuvtCYGFa2RU7qXtkZ9pUiIuIpNGtaEVEgmZdWZ0SC4W4iEgGerKPiEigzDSdIiIStFDuJx7GpI+IiGSkkbiISBum6RQRkZDpxKaISKCS68TDmG1WiIuItKEHJYuIBE1z4iIiAdNIXEQkYBqJi4gEytDqFBGRcJnWiYuIBM0CuaBdIS4ikoFG4iK91JZNW/nJ127g+SeWMmBwGWdefDpHzDqsTb17b7qP+3/1Fza/s4Xi/sUcduLBfPbrZ5IoTLBpw7vcOPeXvLhoBXXbahkzaQyfu+xT7Dl1YgxH1PsYWp2CmRUDjwN9U/u5y92/GdX+RHqKn19xI4VFhfxq8S9YvfwVrjzru4zfeyxjJpW3qjf92AqOOeUoSgeUsGXTVq4+5xruu+XPzDr7RGprapm4/wQ+d/mnGThkIA/d8TBXnvU9fvHEPPqVFMd0ZL2JRfGg5EhEOelTBxzt7gcAU4CZZjYjwv2JxK62ppYn/7KI/77gVPqVFDP5wL2YfkwFj/zh8TZ1R4zdjdIBJQC4O1ZQwNpX3wRgtzG7MuvsE9ll+GASiQI+fPqxNGxvoGrVG916PL2ZZfknLpGNxN3dga2pzaLUy6Pan0hP8MbqtRQkEozafWRz2bi9x7Js0fKM9R+79+/ccPkv2LZ1GwN2KeNzl56Zsd6q5a/QUN/AiLG7RdJvaUtz4oCZJYBngAnAPHdfFOX+ROK2rbqW/qX9WpWVlPVnW3VtxvpHzDqMI2Ydxhur1/LIPY8xaNigNnVqttRw3QU/Yfb5J1MyoH8k/ZbWknPiYaxOibSX7t7o7lOA0cB0M9s3vY6ZzTGzJWa2ZMOGt6Psjkjk+pUUU7N1W6uymq3bOpzHHjl+BOWTyvnZN25sVV5XW8//nn01k6ZM5ORzPpbz/kr4uuVHjbtvAh4BZmb4bL67V7h7xdChQ7qjOyKRGTl+BE2Njbyxem1z2eoVr1KedlIzk6aGRt589a3m7e112/nOnB8wZMQQzvnOnEj6KzuTvNgnm1dcIgtxMxtmZoNS7/sBxwEvRbU/kZ6guH8xMz58EL+79g5qa2pZseQlnn5oMUd97PA2dR+8/W9s2vAuAK/9Zw133fBH9j80+ctqw/YGrj7nGvoWF/GVH55LQUEYv9rnkwIsq1dcopwTHwH8KjUvXgDc6e73Rbg/kR7hi1eezU8u/imfqvg8ZYNL+eKVn2fMpHKWPb2CuZ/9Dncs+w0AK55ZyW9/eDu1NbUM3GUAh5wwg/++8FQAXnrm3yx++Fn6FPfh9AM+09z2Fb+8lH2m7x3HYfUueto9uPtSYGpU7Yv0VGWDSrl0/sVtyveZvndzgAOc/4NzdtrGvjMmc+/qOyPpn3RMF/uIiASu14/ERUTCZcEsMVSIi4hkEMpl9wpxEZE0mhMXEQmc5sRFRIIV702tsqEQFxHJQCNxEZFAJR+UrNUpIiJh0hWbIiIh05y4iEjQNBIXEQlYKCPxMGbuRUQkI43ERUTS6IpNEZHQaU5cRCRUWp0iIhI0rU4REQmYRuIiIgELJcS1xFBEJI2RnE7J5tWpds1mmtlKM6s0s0syfH6BmS03s6Vm9jczG9tRmwpxEZE2LOs/HbZolgDmAccDk4HTzGxyWrXngAp33x+4C/h+R+0qxEVEMsh1iAPTgUp3X+Xu9cDtwKyWFdz9EXevSW0+BYzuqNEeNidumOnnSkjKigbF3YVIHD7y6Li7EJlZ37gs7i7k3tubc9te1+5iONTMlrTYnu/u81tsjwJeb7G9BjionfbOAv7c0U57WIiLiPQMXTixucHdK3Kyb7MzgArgiI7qKsRFRNLsOLGZY1VAeYvt0amy1vs2Oxa4DDjC3es6alRzFyIibeT+xCawGJhoZuPNrA8wG1jQaq9mU4GfAye5+7rONKqRuIhIBrleJ+7uDWZ2HvAAkABudvdlZjYXWOLuC4AfAKXA71O/Cbzm7ie1165CXEQkgyguu3f3hcDCtLIrWrw/Nts2FeIiIhnoik0REYmcRuIiImn0UAgRkaB1/n4ocVOIi4hkpBAXEQlT1y67j4VCXEQkA82Ji4gETCEuIhIo04lNEZGwaSQuIhIwhbiISMA0nSIiEjCNxEVEAqUTmyIigdNIXEQkaApxEck3dY3w1FvwRg0UJ2DKEBg/oG29+kZYsh6qapLbkwbCAUOS72sbYPF6WLcNGhwG9YEPDIOhxd13HJ0QRoQrxEUkG0+vgwKDk3eHd+rgkTdgcF8Y1Ld1vSXrkwH9sXFQ2wgPVUFpIewxELY7DClOBndxAl7eDI9UwUfHQ1HPecRBKHPikf+NmVnCzJ4zs/ui3peIRKihCV7fmhxRFxXA8H4wugRWb2lbt6oaJg+GwgIoLYIJA6Byc/KzsqLkZ/0Lkz8QJg6EJmBzfbceTr7ojh975wMrumE/IhKlzfVgBgP6vFc2uC9s6kT4OvDuTuptrINGT4Z7j2JZvuIRaYib2WjgI8CNUe5HRLpBg7ed7igqgO1NbeuOKIFlG5OfbalPTpk0eNt69Y3wzzdh/12gTyKafndRGBEe/Uj8OuBikr8siUjICq1tYG9vyjyPfeAwSBTAva/Ao2thXGly+qSlhqbkZ0OLYd9dIut212Qb4fHFeGQnNs3sRGCduz9jZke2U28OMAegfEx5VN0RkfdrQB9wT06r7JhSeacuubokXd8EHLbbe9vPbYAhLU5+NjbBY2uTwX7Q8Gj73QUW0EMhohyJHwqcZGavALcDR5vZb9Mruft8d69w94qhQ4dG2B0ReV8KC6C8FF54OzmKXrcN1lTD+LK2dbfUJ5cjNnnyJGflu7BfarTd5PD4m5AwOGTXZGJKl0U2Enf3rwNfB0iNxC9y9zOi2p+IdIPpw+HJt+D3q5Kj7enDkssL122Dh6tg9oRkvY11yWWG9U0woAgO3e29ZYjrtyWDPWFw58vvtX30qOSKlx5CV2yKSP7pm4AjR7YtH97vvQAHGFuWfGWya384Y2I0/cshhXgL7v4o8Gh37EtEpDfRSFxEJAOd2BQRkchpJC4i0oZpTlxEJGwKcRGRIMV9KX02FOIiIhmEcmJTIS4ikpFCXEQkWGFEuEJcRGQnwohxrRMXEQmYRuIiIm1YMCc2NRIXEQmYRuIiImmS68TDGIkrxEVEMlKIi4gEK4wIV4iLiGQUyolNhbiISBvh3D1FIS4ikkEYEa4lhiIiO2FZvjrRotlMM1tpZpVmdkmGz/ua2R2pzxeZ2biO2lSIi4iks+SceDavDps0SwDzgOOBycBpZjY5rdpZwDvuPgG4Fri6o3YV4iIi3WM6UOnuq9y9HrgdmJVWZxbwq9T7u4BjrIOfEApxEZE0Oy72yeZPJ4wCXm+xvSZVlrGOuzcA7wJD2mu0R53YfO7Z5zaUFJW92k27Gwps6KZ9dScdV3jy9di687jG5rKxZ5957oF+hSVDs/xasZktabE9393n57JfmfSoEHf3Yd21LzNb4u4V3bW/7qLjCk++HlvIx+XuMyNotgoob7E9OlWWqc4aMysEBgJvt9eoplNERLrHYmCimY03sz7AbGBBWp0FwKdT708GHnZ3b6/RHjUSFxHJV+7eYGbnAQ8ACeBmd19mZnOBJe6+ALgJ+I2ZVQIbSQZ9u3pziEc+VxWTWI/LzBqBf5H8b2sF8Gl3r+liW7cA97n7XcA2M5vs7st3UvdIoN7d/5nlPl4BKtw9zjlp/bfYS7j7QmBhWtkVLd7XAqdk02avnU7pjhMOcegBx7XN3ae4+75APfDFlh+m5vmy5u4f3FmApxwJHNKVtuPWA/6dRSJfj6un6bUhLt3iCWCCmR1pZk+Y2QJguZklzOwHZrbYzJaa2RcALOn/p65oewgYvqMhM3vUzCpS72ea2bNm9oKZ/S11VdsXga+a2fNm9gUpqBYAAAHCSURBVEEzG2Zmd6f2sdjMDk19d4iZPWhmy8zsRsK5uloko948nSIRSo24jwf+kiqaBuzr7qvNbA7wrrsfaGZ9gX+Y2YPAVGBPklez7QosB25Oa3cY8Avg8FRbu7j7RjP7GbDV3X+Yqvc74Fp3/7uZjSE5D7k38E3g7+4+18w+QvIKOZFgKcQl1/qZ2fOp90+QPFFzCPC0u69OlX8I2N/MTk5tDwQmAocDt7l7I/CGmT2cof0ZwOM72nL3jTvpx7HA5BYXuw0ws9LUPj6e+u79ZvZOF49TpEdQiEuubXP3KS0LUkFa3bII+JK7P5BW74Qc9qMAmJE6UZTeF5G8oTlxicMDwP+YWRGAmU0ysxLgceDU1Jz5COCoDN99CjjczManvrtLqnwLUNai3oPAl3ZsmNmOHyyPA6enyo4HBufsqERioBCXONxIcr77WTN7Efg5yd8K/wD8J/XZr4En07/o7uuBOcA9ZvYCcEfqoz8BH9txYhP4MlCROnG6nPdWyXyb5A+BZSSnVV6L6BhFuoV1cDGQiIj0YBqJi4gETCEuIhIwhbiISMAU4iIiAVOIi4gETCEuIhIwhbiISMAU4iIiAfs/AgWDo1YCKGEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tx44RWj2IUBn",
        "outputId": "57696e9a-5a47-461b-c6d9-f4923cc39507",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "only_not_rated = train[train[\"Helpfulness\"] == -1]\n",
        "not_rated_predictor, not_rated_score, not_rated_predictions, not_rated_true_labels = train_and_score(*train_test_validate_split(only_not_rated.TextClean.values, (only_not_rated.Score.values - 1), sample_size=30000), model_name=\"only_not_rated_real\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train size: 24,000\n",
            "test size: 6,000\n",
            "validate size: 99,372\n",
            "[0, 1, 2, 3, 4]\n",
            "Validation score: 0.754196353097452\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qmXJlbILQJvC",
        "outputId": "7a8a1735-ad52-4b6a-92fd-0c4a0a7715d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "not_rated_predictions = not_rated_predictor.predict(test.TextClean.values)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0QDV0Lr-CvXh",
        "outputId": "78235205-f5d5-44b8-f230-8ff4f9496133",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "cm6 = confusion_matrix_custom(true_labels, not_rated_predictions, \"only_not_rated.pdf\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAE9CAYAAAD9HVKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhV1b3/8fc3YUoCATHMQQaLVoqKgOBcROwFbaWttBdabW31R61D69Bae7XaWnvvbdXWeoteU8WhWoeqVVRaRyzWVmVwYlAvBWQQRRAkkISQ8P39cQ6YnBySnHh2dtbJ58Wzn+fsfdZee22GTxZrr723uTsiIhKmvLgbICIiLacQFxEJmEJcRCRgCnERkYApxEVEAqYQFxEJWLsLcTObZGZvmdlyM7s07vZki5nNMrMNZrY47rZkk5kNNLO5ZrbUzJaY2ffjblM2mFkXM3vZzF5LntfP4m5TtplZvpm9YmaPxd2WXNauQtzM8oGZwGRgODDdzIbH26qsuR2YFHcjIlADXOzuw4EjgHNz5M9sBzDB3Q8FRgKTzOyImNuUbd8HlsXdiFzXrkIcGAssd/cV7l4N3AtMiblNWeHu84AP425Htrn7endflPxcTiIUBsTbqk/OE7YlVzsml5y5887MSoGTgVvibkuua28hPgBYU2d9LTkQCO2FmQ0GDgNeircl2ZEcbngV2AA85e45cV5J1wOXALvibkiua28hLoEys67Ag8AF7r417vZkg7vXuvtIoBQYa2Yj4m5TNpjZ54EN7r4w7ra0B+0txNcBA+uslya3SRtmZh1JBPjd7v5Q3O3JNnffAswld65pHA2cYmarSAxZTjCzu+JtUu5qbyE+HxhmZkPMrBMwDZgdc5ukEWZmwK3AMnf/ddztyRYz62VmPZKfC4ATgTfjbVV2uPuP3b3U3QeT+Df2rLufFnOzcla7CnF3rwHOA54gcYHsfndfEm+rssPM7gH+CRxoZmvN7My425QlRwOnk+jNvZpcToq7UVnQD5hrZq+T6Fw85e6aiicZMz2KVkQkXO2qJy4ikmsU4iIiAVOIi4gETCEuIhIwhbiISCtp6kF1lnBD8gF9r5vZqKbqbLchbmYz4m5DFHRe4cnVc8vV8/qEbqfxm7omA8OSywzgpqYqbLchTuI3KBfpvMKTq+eWq+fVYs14UN0U4M7kA9JeBHqYWb/G6mzPIS4i0tZk/JC+DpE2J0PdexZ779LerXKs3gN6MeyQT7XKnU67vLY1DgNASf992f/gIa12B1dhh6JWOU7/gf04+LARrXZeHfM6tdahGLjfQEaNHtVq57ZzV3WrHKf/wP6t9me2bvU6Pty02bJVn5V0caozfABj+c4lQFWdLWXuXpatNu1Nmwrx3qW9ueGxa+NuRtZt3VkedxMiM7rX6LibEIl+hQObLhSo9yty75lvUz47NbsVVu+CcRl2KJ9eV+XuYz7hkTN+SJ+GU0RE0jHLbMmO2cA3krNUjgA+cvf1je3QpnriIiJtghFJFzf5oLrxQImZrQWuJPFWJ9z9f4E5wEnAcqAC+FZTdSrERUTSyV7veg93n97E9w6cm0mdCnERkXSyn+GRUIiLiDSQ1XHuSCnERURSRTQmHgWFuIhIOuqJi4gELIwMV4iLiDRgQF4YKa4QFxFJJ4wMV4iLiKSlMXERkYCFkeEKcRGRBjQmLiISuDAyXCEuItKQ7tgUEQlXQMMpgdxYKiIi6agnLiKSThgdcYW4iEhaGhMXEQlYGBmeOyFevqWc6y+ZyaJ5r1Lcs5gzLjmN4794XINyf75lNo/ePoePNm+loLALx33hGM78j2+S3yEfgDuv/SP/fPIl1ixfy7Tzv8JpF05r7VMRkbjpwmaCmU0ys7fMbLmZXRrlsW78SRkdOnbgjwtv45LfXsDMy2/mnbdXNyh3xImHc8Pj1/Hgkj9y01O/ZcXSlTxy22N7vu8/uC/f/vE3GDshN9/iLiLNZBkuMYksxM0sH5gJTAaGA9PNbHgUx6qqqOKFv7zI6RdPp6CogM8cPpxxEw/n2Yeea1C236B+dO1eBIC7k5eXx/pV7+35fuLUCRx+/GgKigqiaKqIhCKet91nLMqe+FhgubuvcPdq4F5gShQHWrfiXfLz8ygdOmDPtqEHDeadt9ekLT/34Xmc+pmvMW3kN1mxbBWTv/65KJolIiHLy3CJSZRj4gOAuim6FhgXxYEqK6oo7FZYb1tRcSGV2yvTlj/+i8dx/BePY93Kd3nmwefoUdIjimaJSKhi7l1nIvabfcxshpktMLMFWz/c2qI6Cgq7UFFeUW9bRXllk0MiA4b0Z9ABA7nx8ptbdFwRyWHtfUwcWAcMrLNemtxWj7uXufsYdx9T3LO4RQcaMLQ/tbW7WLfy3T3bVixbxaADBjayV0JtTS3rV7/fouOKSA7TmDjzgWFmNsTMOgHTgNlRHKhLYReOmnQEd/36HqoqqlgyfxkvPvUyE748vkHZv97zFFs2bgFg9dtruP/Ghzj0qIP3fF+zs4bqqmp27XJqa2qprqqmtrY2imaLSFvW3sfE3b3GzM4DngDygVnuviSq45179Qx+88PfMX3UGRTv041zr/4Ogw7Yj8UvL+WKb/6ch5bdA8DShW9y57V3U7m9iu77FnPsSUdx+sVf21PPDZfeyNMPzN2zft/vHuDCa8/nxK9MiKrpItLWGMGMiZu7x92GPYYd8im/4bFr425G1m3dWR53EyIzulduzqfvV9j0UFyo3q9oMKoZvCmfncobryzOWupa7wJn6tDMdrpp6UJ3H5OtNjRXztyxKSKSVYHcsakQFxFJJ5DhFIW4iEiqmKcNZiL2eeIiItJy6omLiDRgWIbDKXFNEVGIi4ikoRAXEQlYINc1FeIiIqkS74TILMXjuq9bIS4iksoyH06Ji0JcRCQNhbiISLAyn50SF4W4iEgagWS4QlxEJFXiIYZhpLhCXEQklS5sioiEzQJ5eIpCXEQkDfXERUQCFkiGK8RFRFIZlvEdm3FRiIuIpBHKcIqeJy4ikio5OyWTpVnVmk0ys7fMbLmZXZrm+/3MbK6ZvWJmr5vZSU3VqRAXEWkFZpYPzAQmA8OB6WY2PKXY5cD97n4YMA24sal6FeIiImmYZbY0w1hgubuvcPdq4F5gSkoZB4qTn7sD7zZVaZsaE+/asStH9jkm7mZk3dPrnoy7CZGZ9+7zcTchEl8ecmrcTYhMKGO9GcnyKbXwjs0SM1tQZ73M3cvqrA8A1tRZXwuMS6njp8CTZnY+UARMbOqgbSrERUTaihaE+EZ3H/MJDzsduN3drzOzI4E/mNkId9+1tx0U4iIiDUTyFMN1wMA666XJbXWdCUwCcPd/mlkXoATYsLdKNSYuIpIqmtkp84FhZjbEzDqRuHA5O6XMauAEADM7COgCfNBYpeqJi4ikke2OuLvXmNl5wBNAPjDL3ZeY2VXAAnefDVwM/N7MLiRxkfMMd2/0HcwKcRGRFFE9itbd5wBzUrZdUefzUuDoTOpUiIuIpBHKLB6FuIhIGnp2iohIqJp/A0/sFOIiIilML0oWEQmb3uwjIhIw9cRFRAKmEBcRCVggGa4QFxFJZRZOT1zPThERCZh64iIiDWiKoYhI0BTiIiIBCyTDFeIiIumoJy4iEqiQZqcoxEVE0gglxDXFMADlW7bxq7N/w2kjvs13j/0ez89+IW25x2b9hXPHX8A3Dj2TGUeey+1X/4HamtoG5Za8tIyv7P917rnu/qibLhIss8yWuKgnHoBbr7ydDh3z+f1LN7Jq2Tv815nXMPjTgxh4QGm9cmNOGMXxU4+jqLiI8i3buO7c3zLnjif4wpkn7SlTs7OG235+J8NG7t/apyESkHCmGEbWEzezWWa2wcwWR3WM9qCqoooXn3iZaRd9hYKiLhw05kDGTBzF3x7+e4OyfQf1oai4KLHiTl6e8d4779cr8+itczj0mIPpP7R/azRfJFgRvCg5ElEOp9wOTIqw/nZh/cr3yM/Pp/+Qfnu2Df70INb+39q05Z+f/QLfOPRMvj3mbFa9uZoTp0/Y890H6z5g7p/+xtTzvxx5u0VCtvvCZgghHtlwirvPM7PBUdXfXlRVVFHQtaDetsJuBVRur0pb/thTjubYU45m/cr3+Nufn6d7Sfc938266k7+/cKpFBR1ibTNIrkgkNEUXdhs67oUdqFyW2W9bZXbKpsM4n5D+jJwWCm3XHEbAAueWUTltiqO/vyRkbVVJJe0+554c5nZDGAGQOnA0iZKtz/9hvSltraW9Svfo9+QvgCsWraa0mFN/17V1tby/uoNALzxjyWsWLyCs8adA0BFeQV5+XmsfnsNP7r54uhOQCRUgXTFYw9xdy8DygAOGz3SY25Om9OlsAvjPnc4913/AGf/11msWvoO859eyC/+9NMGZZ+5by5jThhF95LurPm/tfz5f2dz6LGHADDtwql88ewv7Cl721V30rPPPpx63pda61REAhLO7JTYQ1yadtZV3+KmH5Vx1thz6NqjK//v599i4AGlLJv/Jr/49q+4641ZALy58G3uue5+qip2UNyzG0dMHse0i6YCUNC1oN7Yeqcunehc0JluPbrGck4ibZredg9mdg8wHigxs7XAle5+a1THy2XdenTlkpsvarD9oMM/vSfAAc791XeaXed515ydlbaJ5CIjnDs2o5ydMj2qukVEohZKiGt2iohIwDQmLiKSRig9cYW4iEgagWS4QlxEpIGYb+DJhEJcRCSFZqeIiAROIS4iEjCFuIhIqHTHpohI2NQTFxEJlOkBWCIiYVOIi4gELJAMV4iLiDRg6omLiIRNIS4iEi71xEVEAmVAXhgZrhAXEWkonCmGeimEiEjAFOIiIqkM8swyWppVrdkkM3vLzJab2aV7KfNVM1tqZkvM7I9N1anhFBGRFFE8itbM8oGZwInAWmC+mc1296V1ygwDfgwc7e6bzax3U/WqJy4ikkZehkszjAWWu/sKd68G7gWmpJT5f8BMd98M4O4bmtNOERFJEcFwygBgTZ31tcltdR0AHGBmL5jZi2Y2qalK29RwipFHp/wucTcj647pd1zcTYhM2eJZcTchEmu2r4q7CZFxPO4mZN2uXbVZra+FwyklZragznqZu5dlWEcHYBgwHigF5pnZwe6+pbEdRESknuZfrKxjo7uPaeT7dcDAOuulyW11rQVecvedwEoze5tEqM/fW6UaThERSZV8dkomSzPMB4aZ2RAz6wRMA2anlHmYRC8cMyshMbyyorFK1RMXEUlhZL+H6+41ZnYe8ASQD8xy9yVmdhWwwN1nJ7/7nJktBWqBH7r7psbqVYiLiKTRguGUJrn7HGBOyrYr6nx24KLk0iwKcRGRNEK57V4hLiKSIvEALIW4iEiwwohwhbiISBotmmIYC4W4iEgKMw2niIgETRc2RUQCFkpPXHdsiogETD1xEZEUhmaniIgELZThFIW4iEgDmmIoIhIsM81OEREJWig98SZnp1jCaWZ2RXJ9PzMbG33TRETiYxkucWnOFMMbgSOB6cn1chJvbBYRyUm7H4CV5XdsRqI5wynj3H2Umb0C4O6bk2+lEBHJWaEMpzQnxHeaWT4k3q5qZr2AXZG2SkQkVs1+5VrsmhPiNwB/Bnqb2S+AqcDlkbZK2oWKjyp45OpH+NdL/6KwRyETz5nIIZMO2Wv5mp013PT1m6iuqObixy7es31X7S7mls3llUdfYUfFDnqW9uSMm86goFtBa5yG5KAoXs8WlSZD3N3vNrOFwAkkzu2L7r6sqf3MbCBwJ9CHRC++zN1/+wnbKznk8WseJ79jPj/86w957+33uPvCu+k7rC+99++dtvwLf3iBoh5FVFdU19s+t2wua15fw1m3nkX3vt3ZsGIDHTpp4pV8AgFNMWzO7JT9gArgURJvZt6e3NaUGuBidx8OHAGca2bDP0ljJXdUV1az7NllTPjOBDoXdmbQyEEceNyBvPaX19KW37xuM6//9XWOPePYetsrt1by4r0vcsplp9CjXw/MjD7796Fj546tcRqSw3LpwubjJHrSBnQBhgBvAZ9pbCd3Xw+sT34uN7NlwABg6SdpsOSGTas3kZefR8mgkj3b+g7ry6pFq9KWn3PtHCZ+dyIdOtf/K/v+8vfJy89jyTNLePGeF+lc1Jlx08Yx7ivjomy+5Licej2bux9cd93MRgHnZHIQMxsMHAa8lMl+kruqK6rpXNS53rbOXTs3GCoBWDZ3Gbt27eKg4w9i5cKV9b7bumErVduq2LR6Exc8fAGb1mzijnPvoGS/EvYft3+k5yC5LWeGU1K5+yKg2d0cM+sKPAhc4O5b03w/w8wWmNmCjRs3ZtocCVSnwk7s2L6j3rYd23fQqbD+7NXqymqe/N2TnHTxSWnr2T1sMv6s8XTs0pG+w/oy4sQRvP3C29E0XNoJIy/DJS5N9sTN7KI6q3nAKODd5lRuZh1JBPjd7v5QujLuXgaUAYwaPcqbU6+Eb9/99mVX7S42rd7EvvvtC8D7b79P76H1L2puWrOJLe9uYdaMWQDU1tRSta2KayZdw1mzzqLPsD6JgnX+DYXSg5K2LZS/R83piXers3QmMUY+pamdLPE7cCuwzN1//UkaKbmnU0EnDjr+IJ4te5bqympWv7aaN+e9yaGTD61XrvfQ3lz06EWcfdfZnH3X2Zxy2Sl07dmVs+86m+59utOztCeDRg5i3m3zqKmu4YOVH7D4qcUceMyBMZ2ZSOtqtCeevMmnm7v/oAV1Hw2cDrxhZq8mt/2Hu89pQV2Sg06+5GQe+fkj/OrffkVh90I+/6PP03v/3rzzyjvcdcFdXPa3y8jvkE+3km579ikoLsDyrN62qVdP5ZGrH+GXJ/6Son2KOP47xzN07NA4TklyRE68KNnMOrh7jZkd3ZKK3f3vhPNyDIlBYfdCpl87vcH2QYcN4rK/XZZ2nyGjh9S70QeguHcxp99weiRtlPbLAomvxnriL5MY/37VzGYDfwK27/5yb2PcIiK5IJQx8ebME+8CbAIm8PF8cQcU4iKSkyxH3uzTOzkzZTEfh/dumkUiIjnNAnl6SmMhng90Jf24tkJcRHJaLvTE17v7Va3WEhGRNiQXxsTDOAMRkSyz5K8QNBbiJ7RaK0RE2pJcmCfu7h+2ZkNERNqSXBhOERFplxJv9gl/doqISDuVW+/YFBFpdxTiIiIBi/MZ4ZlQiIuIpDDUExcRCVdAUwzDuPwqIiJpqScuItJAbtyxKSLSLhmQZ2EMVCjERUTSCOXCZhg/akREWpll+KtZdZpNMrO3zGy5mV3aSLlTzczNbExTdaonLiLSQPbf7JN88fxM4ERgLTDfzGa7+9KUct2A7wMvNade9cRFRFIYkfTExwLL3X2Fu1cD9wJT0pT7OfBLoKo5lSrERUTSyDPLaGmGAcCaOutrk9v2MLNRwEB3f7y57WxTwylm4VwRzkRxxx5xNyEyM0Z8O+4mRGLxh6/H3YTI/M/C3HvH+bptG7NboYFlnkUlZragznqZu5c1+5CJA/4aOCOTg7apEBcRaRtaNE98o7s3diFyHTCwznppcttu3YARwHPJmTF9gdlmdoq71/3hUI9CXEQkRWKeeNanGM4HhpnZEBLhPQ342u4v3f0joGRPG8yeA37QWICDQlxEJK1szxN39xozOw94AsgHZrn7EjO7Cljg7rNbUq9CXEQkjSgeRevuc4A5Kduu2EvZ8c2pUyEuIpJCj6IVEQmatWR2SiwU4iIiaejNPiIigTLTcIqISNBCeZ54GIM+IiKSlnriIiINmIZTRERCpgubIiKBSswTD2O0WSEuItKAXpQsIhI0jYmLiARMPXERkYCpJy4iEihDs1NERMJlmicuIhI0C+SGdoW4iEga6omLtFNbN5fz6x/8loXzFlHcs5gzLz2DCV8a36Dc/Tc9yFMPPMP7azfQvWcxX/jGyXz1u6fu+X7JgqXcdOXvWb18DX0H9uF7/3kOI8Z+phXPpP0yNDsFM+sCzAM6J4/zgLtfGdXxRNqK/7n8Rjp06sD9r97Nv5as4LJv/pShw4cw+MBB9cq5O5dcfxFDDxrCu++s59KvXU6v/iUcP+WzbN1czk/OuIrv//e5HDP5KOY+/Dd+8q2fcecLt9KtR7eYzqw9sShelByJKAd9dgAT3P1QYCQwycyOiPB4IrGrrKji73P+wRk/PJ2CogJGjP0MR544jqcffLZB2X8/ZyrDDv4U+R3yGbh/KUd97giWzF8KwNKFy+jZex8++/ljyc/PZ+KpE+jeszt//8s/WvuU2i3L8FdcIgtxT9iWXO2YXDyq44m0BetWrCM/P5/SoQP2bNt/+BDeeXt1o/u5O2+8vIRBdXrr7p5SBla99U52Gyx7ZckZKs1d4hLp5VczyzezV4ENwFPu/lKUxxOJW+X2Sgq7FdTbVtStiIptlY3ud+d1d+O7nH/76okADB/9aTa9/yHPPvwcNTtrePJPT7P+nfXsqNwRWdvlY4kx8byMlrhEemR3r3X3kUApMNbMRqSWMbMZZrbAzBZ88MHGKJsjErmCogIqyusH9vZtFRR2LdjLHvDwbY/y9IPPcvUdP6VT544AFO9TzM9u/QkP/v5hvjry68x/biGHHTuSkn4lkbZfwtMqs1PcfYuZzQUmAYtTvisDygBGjxml4RYJ2oChA6itrWXtinV7hlRWLF3JoAP2S1v+r/c+yX0z/8SvH/wlvfrXD+hDjzyYmY9fD0BtTS2nH3UmU2d8KdoTkKRwbvaJrCduZr3MrEfycwFwIvBmVMcTaQsKCrtwzOSjuOO6u6isqGLx/KX848kXmXjqhAZln3loLrN+eQf/fc/V9BvUr8H3yxf/i5qdNWwvr+Dmn99Kr/4lHD5+dGuchkCGgynxBX6UPfF+wB1mlk/ih8X97v5YhMcTaRPO/8U5XPeD6/nqoV+j2z7FfP8/z2XwgYN446XF/MfpV/Lo2w8CcPs1f2Dr5nLOO/nCPfue8OXjueC/zwPgvpse4OVnFwBw+PjR/PSWy1v/ZNorve0e3P114LCo6hdpq4r36cbPbv1Jg+0HjxuxJ8AB/vDPWY3Wc9nMH2W9bdI8utlHRCRw7b4nLiISLtMDsEREQhbKbfcKcRGRFBoTFxEJnMbERUSCFe9DrTKhEBcRSUM9cRGRQCVelKzZKSIiYdIdmyIiIdOYuIhI0NQTFxEJWCg98TBG7kVEJC31xEVEUuiOTRGR0GlMXEQkVJqdIiISNM1OEREJmHriIiIBCyXENcVQRCSFkRhOyWRpVr1mk8zsLTNbbmaXpvn+IjNbamavm9kzZjaoqToV4iIiDVjGv5qs0SwfmAlMBoYD081seEqxV4Ax7n4I8ADwq6bqVYiLiKSR7RAHxgLL3X2Fu1cD9wJT6hZw97nuXpFcfREobapSjYm3gjzL3Z+V3Tp2j7sJkRjb68i4mxCZ2fd8L+4mZN+H5dmtr2VPMSwxswV11svcvazO+gBgTZ31tcC4Ruo7E/hLUwdViIuIpNGCC5sb3X1MVo5tdhowBvhsU2UV4iIiKXZf2MyydcDAOuulyW31j202EbgM+Ky772iq0tz9f76ISItl/8ImMB8YZmZDzKwTMA2YXe+oZocBNwOnuPuG5lSqnriISBrZnifu7jVmdh7wBJAPzHL3JWZ2FbDA3WcD1wBdgT8l/yew2t1PaaxehbiISBpR3Hbv7nOAOSnbrqjzeWKmdSrERUTS0B2bIiISOfXERURS6KUQIiJBa/7zUOKmEBcRSUshLiISppbddh8LhbiISBoaExcRCZhCXEQkUKYLmyIiYVNPXEQkYApxEZGAaThFRCRg6omLiARKFzZFRAKnnriISNAU4iKSa3bUwssfwHsV0DkfDukJg7s1LFddC4s2wvqKxPqnusPBPRuW21AJz74Lw3vAIftG2/YMhRHhCnERycTCjYm3EHxxMGzZAfPeg306Q/dO9cu9sglqHb4wCKpqYe67UNQBhhZ/XGaXJ4J+386teQbNFsqYeOQvhTCzfDN7xcwei/pYIhKhml2wdluiR90xD3oVQP9CWFXesOy67fDpHtAhD7p2TIT3ipRyb26BvoXQrVPD/aXZWuPNPt8HlrXCcUQkSuU7wQyK64TuPp3ho+rm7V+33PadsLIcPrNPdtuYVZbhEo9IQ9zMSoGTgVuiPI6ItIKduxI98Lo65iW2p+pXCMu2JL4r3wkrtkJtnXKLNn7co2+jwojw6Hvi1wOXAGn+lEUkKOkCO12wA4wqgXyDx1fD8+thUFcoSF6CW7c9sd9+XaNvc4tlGuHxxXhkFzbN7PPABndfaGbjGyk3A5gBMHC/gVE1R0Q+qW4dwR3Kqz8ex96yo+FFTUjMXDmyz8frr22CfbskPr9fCR/ugIdXJdZ37kpk4EfVcGy/KM+g2UwvhQDgaOAUMzsJ6AIUm9ld7n5a3ULuXgaUAYweM8ojbI+IfBId8qC0K7yxGcb2gs07YF0FTBzQsGz5TuiUl+ilv1cB/9oKJyTLHdwTDurxcdlFGxO99DY9Pt52RRbi7v5j4McAyZ74D1IDXEQCM7oEXt4Af16V6G2PLkn0xDdUwrz1MHVootzmHYlw3rkr0YM/ss/HPfaOefWHYPLzoIMl6mtDdMemiOSezvnphzx6F3wc4JAY727umPcRvbPTtixTiNfh7s8Bz7XGsURE2hP1xEVE0gjlwmbbnaQpIiJNUk9cRKQB05i4iEjYFOIiIkGK+1b6TCjERUTSCOXCpkJcRCQthbiISLDCiHCFuIjIXoQR45onLiISMPXERUQasGAubKonLiISMPXERURSJOaJh9ETV4iLiKSlEBcRCVYYEa4QFxFJK5QLmwpxEZEGwnl6ikJcRCSNMCJcUwxFRPbCMlyaUaPZJDN7y8yWm9mlab7vbGb3Jb9/ycwGN1WnQlxEJJUlxsQzWZqs0iwfmAlMBoYD081seEqxM4HN7v4p4DfAL5uqVyEuItI6xgLL3X2Fu1cD9wJTUspMAe5Ifn4AOMGa+AmhEBcRSbH7Zp9MfjXDAGBNnfW1yW1py7h7DfARsG9jlbapC5uLFr6ysaBD0TutdLgSYGMrHas16bzCk6vn1prnNe5pXYcAAAMnSURBVCiblS1a+MoTBR2KSjLcrYuZLaizXubuZdlsVzptKsTdvVdrHcvMFrj7mNY6XmvReYUnV88t5PNy90kRVLsOGFhnvTS5LV2ZtWbWAegObGqsUg2niIi0jvnAMDMbYmadgGnA7JQys4FvJj9PBZ51d2+s0jbVExcRyVXuXmNm5wFPAPnALHdfYmZXAQvcfTZwK/AHM1sOfEgi6BvVnkM88rGqmMR6XmZWC7xB4u/WMuCb7l7RwrpuBx5z9weASjMb7u5L91J2PFDt7v/I8BirgDHuHueYtP4uthPuPgeYk7Ltijqfq4CvZFJnux1OaY0LDnFoA+dV6e4j3X0EUA2cXffL5Dhfxtz92L0FeNJ44KiW1B23NvBnFolcPa+2pt2GuLSK54FPmdl4M3vezGYDS80s38yuMbP5Zva6mX0HwBJ+l7yj7Wmg9+6KzOw5MxuT/DzJzBaZ2Wtm9kzyrrazgQvN7FUzO9bMepnZg8ljzDezo5P77mtmT5rZEjO7hXDurhZJqz0Pp0iEkj3uycBfk5tGASPcfaWZzQA+cvfDzawz8IKZPQkcBhxI4m62PsBSYFZKvb2A3wPHJevq6e4fmtn/Atvc/dpkuT8Cv3H3v5vZfiTGIQ8CrgT+7u5XmdnJJO6QEwmWQlyyrcDMXk1+fp7EhZqjgJfdfWVy++eAQ8xsanK9OzAMOA64x91rgXfN7Nk09R8BzNtdl7t/uJd2TASG17nZrdjMuiaP8eXkvo+b2eYWnqdIm6AQl2yrdPeRdTckg3R73U3A+e7+REq5k7LYjjzgiOSFotS2iOQMjYlLHJ4AvmtmHQHM7AAzKwLmAf+eHDPvBxyfZt8XgePMbEhy357J7eVAtzrlngTO371iZrt/sMwDvpbcNhnYJ2tnJRIDhbjE4RYS492LzGwxcDOJ/xX+Gfi/5Hd3Av9M3dHdPwBmAA+Z2WvAfcmvHgW+tPvCJvA9YEzywulSPp4l8zMSPwSWkBhWWR3ROYq0CmviZiAREWnD1BMXEQmYQlxEJGAKcRGRgCnERUQCphAXEQmYQlxEJGAKcRGRgCnERUQC9v8B3g7iIyRQ32sAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N2SlQQHFKJ0s"
      },
      "source": [
        "def all_confusion(cms):\n",
        "  fig, axs = plt.subplots(2, 3, constrained_layout=True, figsize=(15, 11))\n",
        "  for ax, cm in zip(axs.flat, cms):\n",
        "    pcm = ax.matshow(cms[cm], cmap=plt.cm.Greens, vmin=0, vmax=1)\n",
        "    for (i, j), z in np.ndenumerate(cms[cm]):\n",
        "      if i == j:\n",
        "        ax.text(j, i, \"{:0.2f}\".format(z), ha=\"center\", va=\"center\", fontsize=12)\n",
        "    ax.set_xticklabels([''] + [0, 1, 2, 3, 4])\n",
        "    ax.set_yticklabels([''] + [0, 1, 2, 3, 4])\n",
        "    ax.set_xlabel(\"Predicted\")\n",
        "    ax.set_ylabel(\"True\")\n",
        "    ax.set_title(cm)\n",
        "\n",
        "  fig.colorbar(pcm, ax=axs[1:, :], location=\"bottom\", shrink=0.5)\n",
        "  plt.savefig(\"all_confusion.pdf\")\n",
        "  plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pU6FnhT0Js8T",
        "outputId": "8a13e437-63ec-41d2-9949-76fb01fbacef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 817
        }
      },
      "source": [
        "all_confusion({\"Base line\": cm1, \"Score balanced\": cm2, \"Helpfulness balanced\": cm7, \"Only not helpful\": cm4, \"Only helpful\": cm5, \"Only not rated\": cm6})"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABDoAAAMgCAYAAADCxwLKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5gV5fnG8fs552wvLLD0voAgRREVNWrEjkZFE2LXmPzUqNGIwUI0mmjsNTEajV1jBzsiYsEeQBFEEEF673377nl/f8xh2WWLgJwzy5zv57q4LnbmnZnndeXZ2ftMMeecAAAAAAAAgiDkdwEAAAAAAAC7CkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwCDoAOphZn8zs2djf+9oZpvNLOx3XQAQNGb2lJndvJPbVvXqxsLM5pvZUX7XAWD3siP9zDxPmtk6M5u4HeN3us/uavR8JAJBBxqlWMMojoUL68zsbTPr4Fc9zrmFzrls51ylXzUAwK5gZoeY2RdmtsHM1prZ52a2v991AcDurq5feM3sPDP7LA6HO0TS0ZLaO+cGxGH/wG6NoAON2YnOuWxJbSStkPQvn+sBgN2ameVKGiWvnzaT1E7SjZJKd/FxuPoNAOKrk6T5zrlCvwsBGiOCDjR6zrkSSSMl9dqyzMx+YWaTzWyjmS0ys79VW5duZs+a2RozW29mX5pZq9i6Jmb2uJktM7MlZnbz9pyQm1lnM3NmFol9/ZGZ/T32SegmMxtrZvnVxh8Y+8R0vZl9Y2YDd91/EQDYaXtIknPuBedcpXOu2Dk31jk3dcsAM7vAzGbEett3ZtY/tnzPWO9bb2bTzeykats8ZWYPmdloMyuUdLiZtTWzV8xslZnNM7M//kht+Wb2Xuy4H5tZp2r7/2es1280s0lmdmh9OzGzEWa2PHbFyidm1nubOh+MXSW4ycwmmFnXaut7x2pYa2YrzOza2PKQmQ03szmxny0vm1mzatudY2YLYuuu255vBIDktL29sdq554VmtjR27nplbN3/SXpM0kGxq59vrOvKkdj23erY90AzW2xmw8xsZWzfv622Ps3M7jazhbFe+LCZZcTW5ZvZqNjPgrVm9qmZhWLrromdX28ys5lmdmQD/yno+Ygrgg40emaWKek0SeOrLS6UdK6kPEm/kHSxmZ0cW/cbSU0kdZDUXNJFkopj656SVCGpm6R9JB0j6fydLO1MSb+V1FJSqqQtP3zaSXpb0s3yPjG9UtIrZtZiJ48DALvKLEmVZva0mR1nZk2rrzSzX0v6m7z+mivpJElrzCxF0luSxsrreZdJes7MelTb/ExJt0jKkfRFbPw38q4aOVLSUDM7toHazpL0d0n5kqZIeq7aui8l9ZPXU5+XNMLM0uvZzzuSusfq/Hqb/UjS6fKuYmkqaXasZplZjqT3JY2R1Fbez4kPYttcJulkSYfF1q2T9GBsu16SHpJ0Tmxdc0ntG5gngCQVCwR2tDceLq+nHSPpGjM7yjn3uLzz2//Fbq3+606U01re+XI7Sf8n6cFqPxNulxeM95PXC9tJuiG2bpikxZJaSGol6VpJLvbz4FJJ+zvnciQdK2l+A8en5yOuCDrQmL1uZuslbZB3D+JdW1Y45z5yzn3rnIvGPol8QV4zkqRyeU2nW+wTy0nOuY3mXdVxvKShzrlC59xKSffJa4A740nn3CznXLGkl+U1ZEk6W9Jo59zoWH3vSfoqdmwA8I1zbqO8+7qdpEclrTKzN2P9UfKC3zudc186z2zn3AJJB0rKlnS7c67MOfehvFtgzqi2+zecc58756KS+kpq4Zy7KTZ+bux4DfXbt51znzjnSiVdJ++Tyg6xup91zq1xzlU45+6RlCapR107cc494ZzbFNvP3yTtbWZNqg15zTk30TlXIe+EeEvvPkHScufcPc65ktg+JsTWXSTpOufc4mr7HWLeVX5DJI2qVvv1kqINzBNAsL0eu9phfew89t/V1u2vHe+NN8bOW7+V9KRq9t2folzSTc65cufcaEmbJfUwM5N0oaQrnHNrnXObJN1arcZyebeVd4pt+6lzzkmqlNebe5lZinNuvnNuTgPHp+cjrgg60Jid7JzLk5QuLyH+2MxaS5KZHWBm42KX/W2Q15C23DryX0nvSnoxdqnfnbFPIztJSpG0rNoPn//IS4B3xvJqfy+S90uAYsf59TY/5A6R90MBAHzlnJvhnDvPOddeUh95n0j9I7a6g6S6TkzbSloUCzG2WCDvU74tFlX7eydJbbfpg9fK+/SvPlXbO+c2S1obO67M7ErzbqfZENtXE23t+VXMLGxmt8cuN96orZ8mVh9bX++ub+5b5vNatbnMkHdS3ypWY/XaCyWtaWCeAILtZOdc3pY/ki6ptu4n9UZ5fbftLqpzTeyX/y229MMWkjIlTapW45jYcsn74HG2pLFmNtfMhkuSc262pKHyQoGVZvaimTVUKz0fcUXQgUYvdlXGq/IazCGxxc9LelNSB+dcE0kPS7LY+HLn3I3OuV6SfiYvsT1XXlMqlZRf7QdQrnOut3atRZL+W/2HnHMuyzl3+y4+DgD8JM657+Xd0tcntmiRpK51DF0qqcOW+7BjOkpaUn131f6+SNK8bfpgjnOuoSvbqt6sZWbZ8i5ZXhq7N/tqSadKahr7xWGDYj1/G2dKGizpKHknxp237LKB41avuaCBdcdtM59059wSScu2qT1T3lWFALCtn9Qb5fXdpfWMK5QXUEiStnw4uBNWy7vlu3e1Gps47wUBil35MMw5VyDv9sY/bXkWh3PueefcIfKCAifpju2ZFz0f8UDQgUbPPIPl3Vs3I7Y4R9Ja51yJmQ2Q1+i2jD/czPqa95DRjfIusYs655bJu7/8HjPLjT1oqKuZHaZd61lJJ5rZsbGkOd28hz5x/x4AX5lZT/MePtc+9nUHeZdBb3kG0mOSrjSzfWO9t5t5D4ibIO+TsKvNLMW8ByyfKOnFeg41UdIm8x5MlxHrhX2s4dfYHm/eq29T5d23Pd45t0hev6+QtEpSxMxukPf8kLrkyAu018g74b91O/6zbDFKUhszG2reg/hyzOyA2LqHJd0S+28hM2sR+7kkeQ/LPqFa7TeJ8ysAdduZ3ni9mWWa95DN30p6qZ5x30jqbWb9zHuexd92psDYlXuPSrrPzFpK3vPnLPYcETM7IfazweQFEJWSombWw8yOMLM0SSXywpKGbumg5yOu+KagMXvLzDbLCytukfQb59z02LpLJN1kZpvkPRzp5WrbtZbXhDbKC0Y+lnc7i+Rd2ZEq6Tt5DxYaqV18S0msSQ+WdyniKnmp8FXi3xsA/22SdICkCea9HWW8pGnyHi4n59wIef32+djY1yU1c86VyQs2jpP3ad+/JZ0buyKkFudcpbyr6fpJmhfb5jF5n7jV53lJf5V3+fK+8p53JHm3Io6R9yDVBfJOoBfVtQNJz8TGLJHX58fXM66umjfJex7UifIudf5B3kMAJemf8q4iHBv7uTNe3n9HxX4u/SFW/zJ5P1sWb+9xASSPneyNH8u7VeQDSXc758bWs+9Z8n7pfl9e//qsrnHb6ZrYMcfHbgl5X1ufkdE99vVmSf+T9G/n3Dh5z9G4PTan5fJuDf9zA8eg5yOuzHt2DAAAAACgMTCzzvLCkJRtnqUBYDvwCTMAAAAAAAgMgg4AAAAAABAY3LoCAAAAAAACgys6AAAAAABAYBB0SDKzQWY208xmm9lwv+tJFDN7wsxWmtk0v2tJNDPrYGbjzOw7M5tuZpf7XVOixF53O9HMvonN/Ua/a0q02OvcJpvZKL9rwfZJ1j4t0avp1fRqevXugT5Nn6ZP06cbk6QPOswsLOlBea/M6yXpDDPr5W9VCfOUpEF+F+GTCknDnHO9JB0o6Q9J9H0vlXSEc25vea83G2RmB/pcU6JdLu/Vw9gNJHmflujV9Gp6NRo5+jR9mj5Nn25skj7okDRA0mzn3FznXJmkFyUN9rmmhHDOfSLv3dVJxzm3zDn3dezvm+T9A23nb1WJ4TybY1+mxP4kzcN6zKy9pF/Ie289dg9J26clejW9WhK9Go0ffZo+TZ+mTzcqBB3eP8RF1b5erCT5xwlP7D3l+0ia4G8liRO7zGyKpJWS3nPOJc3cJf1D0tWSon4Xgu1Gnwa9ml6Nxo0+Dfo0fbpRIehAUjOzbEmvSBrqnNvodz2J4pyrdM71k9Re0gAz6+N3TYlgZidIWumcm+R3LQC2H72aXg2gcaNP06cbG4IOaYmkDtW+bh9bhoAzsxR5Dfk559yrftfjB+fceknjlDz3lR4s6SQzmy/vstojzOxZf0vCdqBPJzF6Nb1a9OrdAX06idGn6dNqhH2aoEP6UlJ3M+tiZqmSTpf0ps81Ic7MzCQ9LmmGc+5ev+tJJDNrYWZ5sb9nSDpa0vf+VpUYzrk/O+faO+c6y/u3/qFz7myfy8KPo08nKXo1vVr06t0FfTpJ0afp02qkfTrpgw7nXIWkSyW9K+/hOS8756b7W1VimNkLkv4nqYeZLTaz//O7pgQ6WNI58tLHKbE/x/tdVIK0kTTOzKbKOzF5zznX6F4JBWyRzH1aoleLXk2vRqNHn6ZPiz5Nn25kzLmkeTAsAAAAAAAIuKS/ogMAAAAAAAQHQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHTEmNmFftfgl2Seu5Tc82fu2N0k8/eNuScn5o7dTTJ/35h78krm+TfWuRN0bNUov0EJksxzl5J7/swdu5tk/r4x9+TE3LG7SebvG3NPXsk8/0Y5d4IOAAAAAAAQGOac87uGKqGsFBfJS/fl2NHCcoWyUnw5tiT1at3Vt2OvWbVGzVs09+345tuRPatXr1F+vp/z9y9vXL16tfLz8307ftRV+nbsNavXqLmP3/dvJk9d7Zxr4VsBO8nPPi3526t7tursy3G3WLt6rZrlN/Pl2GEL+3LcLejT/vVpyb/zRL+/75O/nrJ79unMFBfKS/Pt+K6wXOZTn+7j4/m05O+5hZm/n1+vWb1azX3sVSHz9zeKVatWq0ULf+YfdVFfjrvF6lVrlO/T75JTvv6m3j4dSXQxDYnkpavlJfv6XYYvPrxyhN8l+Cbkc2P2W2rIv5MRv20u3+h3Cb5pldlugd817Ixk7tOjLn/M7xJ8k5va1O8SfJXMfdr5fALtp+zUJrtlnw7lpSn3//r6XYYv3rv2Fb9L8E1aJNPvEnyVGkr1uwTflFQU+V2Cb5qmt6i3Tyf3b5gAAAAAACBQCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBQdABAAAAAAACg6ADAAAAAAAEBkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwCDoAAAAAAAAgUHQAQAAAAAAAoOgAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYEb8LiKdoUbnWvTZTpbPXKZSZotxjuihz71a1xq1+eqrKFmyo+tpVOkXyM9Xqsv1Usb5EK+//ssZ4VxZV7qAC5RzSIe5zAIAgo08DQOMWLa5Q0ag5Kp+7QaGMiDKO6KjUPvm1xrmKqIrGzlf5zHVSZVSRDjnKPK5AodxUSVLJl8tVNnWVKlcWKbV3c2Wd1C3RUwGQRAIddKx/6wdZOKTWw3+m8mWbtea/3yqldbZSWmXVGJf/m71qfL3qsSlKK8iTJEXy0tX2hkOr1lWsLdaK+yYqo3eL+E8AAAKOPg0AjVvRO/OksCnvin1VubxQm16aqXCrTIVbZNYYVzpxuSoXb1buBX1l6REVvT1XRe/OU/ave0iSQjmpSj+kncrnrJcqon5MBUASieutK2Y2yMxmmtlsMxsez2NtK1pWqeLvVivnqM4KpYWV1rmJ0ns2V9GUFQ1uV7GuRGULNihzn9Z1ri+askKpnZso0jQ9HmUDQELRpwGg8fOrV7uySpV/v1YZh3WQpYYV6Zir1O5NVfrt6lpjK9eXKNK1iULZqbJISKm9mqtyVXHV+tSezZTao5lCGSmJKh9AEotb0GFmYUkPSjpOUi9JZ5hZr3gdb1sVq4tkIVNK/ta0OaVNtipWFja4XdHk5UrtVPcJsnNORZNX1HtyDQC7E/o0ADR+fvbqyrUlUsgUbp5RtSzcKlPRVUW1xqb1a6mKRZsU3VQmV16p0mmrldItLxFlAkAt8bx1ZYCk2c65uZJkZi9KGizpuzges4orq5SlhWssC6VFFC2tbHC7oikrlDOwU53ryhZsULSwjMuhAQQFfRoAGj/fenVdfdrSwnJ19Olws3SFctO04Z9fSyaFW2Yqc1CXeJcIAHWK560r7SQtqvb14tiyGszsQjP7ysy+ihaW77KDW2rtJhwtrVBom2ZdXen8DYpurv8EuWjyCqX3atHgPgBgN0KfBoDG70d7dfU+7eLcp+sKPySpaMx8qTKqJsP2U941A5TSs5k2vzBjl9UCADvC99fLOucecc7t55zbL5S16+7Zi+RnykWdKlZvvbSufHmhIi2z6t2maPLyek+QXXmliqetUtY+td8GAABBRp8GgMatep+2Xdinw83SpahT5dqtz9qoXFGk0DYPIpWkihWFSt2rhUIZEVkkpLT9W6tyaaGiRbsueAGA7RXPoGOJpOrv9WsfW5YQodSwMnrla+MH8xUtq1Tpgg0qmbFamf3qPgH+sRPk4u9WK5QRUWoB9xoCCAz6NAA0fr71aksNK6VnMxV/tFiurFIVizapbNY6pfWt/XrZSJtslX27Wq6kQq4yqtKvVshyUhTK9IIXF3VyFVHJOSnqvY7WRV0ipgEgCcUz6PhSUncz62JmqZJOl/RmHI9XS96J3eUqolp+2xda9/IM5Z3UXSmtslQ6f72W3vRpjbHF361RKL3+E+SiySuU2a+VzCwRpQNAItCnAaDx87VXZx7XRaqIav19k7T5tR+UdVwXhVtkqnzhRq27Y2LVuIyjOsrCpg3/nqIN905S+ez1yh7So2p9yaeLtf72iSr5YqnKpq32/v7p4kRNA0CSidvDSJ1zFWZ2qaR3JYUlPeGcmx6v49UllJmi5mf1qbU8rXOe2t5waI1lmXu3VObeLevdV/55e+3y+gDAT/RpAGj8/O7VoYyIsk/tUWt5SsdcNb1mwNZxmSnKOqV7vfvJOKyDMg7rUO96ANiV4vnWFTnnRksaHc9jAAB2Hn0aABo/ejUA7BjfH0YKAAAAAACwqxB0AAAAAACAwCDoAAAAAAAAgUHQAQAAAAAAAoOgAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBQdABAAAAAAACg6ADAAAAAAAEBkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwIj4XUB1bfKaavjJv/K7DF/M3TjL7xJ8kx7J8LsEX3XI6ux3Cb6pdBV+l4Ad1KFZC91++vl+l+GLdxeN9bsE3xzU+kC/S/BVp+wCv0vwjcn8LgE7qE3Tpho2ZIjfZfhi2rqpfpfgm/z0fL9L8FXX3J5+l+CbCs6n68QVHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBQdABAAAAAAACg6ADAAAAAAAEBkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwCDoAAAAAAAAgUHQAQAAAAAAAoOgAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAifheA+NiwbqNuHnqbxn/8pfKaNdEfrvu9Bv3qmFrj/vvA8xr18jtavmi58prnach5p+icS8+sWn/RKZdpzvfzVF5aprYd2+j315yvw447NJFTAYBAKlxfqKevf07ffTFD2XlZOuWKwTrghP1rjXvv6Q817rmPtHldodIy07Tfcf015MpTFI6EJUlzJs/VS7eP1LI5y5XfvrnOvP40dd+3W6KnAwCBtHHdJt115X2a9PHXym3WRBf8+Twdecrhtca9+NBIjR3xvlYsXqkmzXJ10m9O0OkXD6laP3vaHP3r+oc0d8Y8ZWRl6MSzj9c5V5xZaz8Adg2CjoC6c/g9iqSm6N1pb2rWtB809Kyr1b13N3XtWVBjnHNONz7wF3Xr1VWL5y/VZadeoVbtWuqYU46SJA27+XJ16dFZkUhE0yZN1x+GDNUr419Qfqt8P6YFAIHx/M0vKZIS1t2f3KZF3y/Wvy5+SB16tFPb7m1rjNv78L46+JQDlZmbqcL1hXr4isf04bMf6ejzjlTh+kI9cMnDOuuvp6v/0f008e2v9OAfHtYt796krCaZPs0MAILjn9c9qJSUFL3yzQuaPX2Orj33ryroVaAuPTrVHOichv/zSnXds4uWzF+mq8+8Ti3b5uuIwQMlSbdceocOGfQz3TvyDi1ftEKXn3KlCnoX6OBjDkz8pIAkELdbV8zsCTNbaWbT4nUM1K24sFgfjvpYFw0/X5nZmep34N76+bGHaPSId2uNPfeys9Rzrx6KRCLq3K2jDht0qL6Z+G3V+u69uykS8fIwM1NFRaVWLFmZsLkAiC96tT9Ki0r19dgpGvzHE5Sela7u+3bT3of31fi3JtYa27JjC2XmeqGFk5OZaeXCVZKkOVPmKjc/R/sN6q9QOKQDTxqg7KY5mvz+lITOB0D80Kf9U1xUok9Hf67fXnWOMrIy1HdAHx109IF675UPao09/ZJfa4++3RSOhNWxW3sdfOyBmvbld1Xrly9aqSN/ebjC4bDadW6rPvv31oKZCxI5HSCpxPMZHU9JGhTH/aMeC+cuUjgSVqeuHauWde/dVXNnzmtwO+ecJo//RgU9utRYfsVZV+vgDkfovEEXqv/P+mnPfj3jUjcAXzwlenXCrZi/UqFISK06t6pa1qFHey2dvazO8RNGfak/7j9Mf/rZNVo8c4l+fuohW1e6bQY7pyU/LI1D1QB88pTo075YPHexwuGwOnRtX7Wsa+8umv8jAYVzTt9OmKbOe2y96uNX55+ssSM/UEV5hRbOXqzvJs1Q/0P7xa12INnF7dYV59wnZtY5XvtH/YoKi5WVnVVjWXZutoo2FzW43SN3PiEXjerEM46vsfy+5+5URXmFJnzypebPWqBQiGfYAkFBr/ZHaVGpMrLSayzLyMlQSWFpneMPOGF/HXDC/loxf6XGvzlBuc1zJEkF/bpo/aoNmvj2V+p/zD6a+PaXWrVotcqKy+I+BwCJQZ/2T3FhiTJzat4GmJWTpeLC4ga3e/qeZxWNOg067eiqZQceNUC3D71bLz/8iqKVUZ17xZnq2a9HXOoG0AjeumJmF5rZV2b21eZ1m/0uJxAyszJUuLmwxrLCTYXKzK7/fu2XH39Fo0eM0X3P36XUtNRa6yMpER185EEa/9FEfTzms11eM4DGq3qf3rh2o9/lBEJaZpqKC0tqLCveXKz0rLQGt2vVuaXadGuj5//+kiQpOy9blzzwe7339Ae68tDhmv7Zd+p5UA81bd00brUDaHyq9+nCdYU/vgG2S0ZWuoo21fygsGhTkTKyMurd5rUn39TYkR/otmdurDqn3rhuk4af/RedO/RMvTv3Tb305TP68qNJeuOpUXGtH0hmvgcdzrlHnHP7Oef2y26a7Xc5gdCxoIMqKyq1cO6iqmU/TJ9d65aULd58fpSevv9ZPTjyH2rVtmWD+66sqNSS+Ut2ab0AGrfqfTq3Wa7f5QRCq84tFa2IasX8rc88Wjxzidp2a/Oj20Yrolq1aHXV1z32767rXr5G/xh/l353+2+0fO4KdenbqYE9AAia6n06q2nWj2+A7dK+oL0qKyu1eO7Wc985381T520fRBrzzovv6oUHXtbdL92mFm1bVC1ftnCZQuGwjvn1UQpHwmrRtoUOH3yYJnz4ZdznACQr34MO7HoZWRk6/BeH6T93PKbiwmJ9M2GqPh7zmY7/9bG1xr4zcqwevOURPTDiPrXv3K7Guvk/LNDnH/xPJcWlqiiv0OgR72ry+G/U/2fcTwgAP0VaZpr2Obqf3nxglEqLSjX76zma8uFUHXjigFpjPx35uTau2SRJWjp7md559F31PHDr5c4Lv1ukivJKFW8u1si7XlOzNk3V+5BeCZsLAARVRma6Dj3uZ3ry7v+quKhE076cri/G/k9H/+rIWmPff/VDPXb707rrxVvVtlPN0Lp9QTs55/TBa+MUjUa1duVaffTmJyrYs+4PIQH8dLxeNqCuuWOY/n75bTqm94lq0jRXw+8cpq49CzR5/De6/PQr9cn89yRJD9/+qDas26DfHHNB1bbHDTlGf777Kjnn9OhdT+jamTcoFA6rQ0F73frIjeq5F/cTAsBPddb1p+mpvzyrYYcOV1aTLJ11w+lq272tfvhqtu7//YP616T7JElzvp6r1//5lkqLSpXTNFv7HruPBv/xxKr9vPvEe/r2k+mSpD6H9NLF91/oy3wAIIguv/VS3TXsPv1qr9OV2zRXQ2+7VF16dNLUCdM0/OzrNfqH1yRJT9z5jDau26iLj7+8atujf3mErrjjMmXlZOnGR/+iR299Qv/48wNKTU/TQUcfoLOHnu7XtIDAM+e2fVz7Ltqx2QuSBkrKl7RC0l+dc483tE2n3h3d8Jeviks9jd3+Lfv7XYJv0iP13+eYDDpkdfa7BN8UVSTvc3naZnWa5Jzbz+86drRXd+1b4G5//aYEVde4rC9L3ueTHNT6QL9L8FWn7NkBirEAACAASURBVAK/S/CNyfwuwTc5qXm7ZZ/u0LuDG/b80ARV17js3aK33yX4Jj893+8SfNU1N3nfCpnM59MtMtrU26fj+daVM+K1bwDArkGvBoDGjT4NADuOZ3QAAAAAAIDAIOgAAAAAAACBQdABAAAAAAACg6ADAAAAAAAEBkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwCDoAAAAAAAAgUHQAQAAAAAAAoOgAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBEfG7gOry0vJ0cpfBfpeBBHtz3pt+l+Crw9o1qn+GCeXk/C4BOyg7JVuHtPm532X4YnXJSr9L8M2rs0f5XYKvzu5xqt8l+KZZWr7fJWAH5aU10eCCE/wuwxcpoVS/S/DNI9Oe9LsEX122V1u/S/BNYcVmv0tolLiiAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBQdABAAAAAAACg6ADAAAAAAAEBkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwPjRoMM8Z5vZDbGvO5rZgPiXBgDYXvRqAGjc6NMAkDjbc0XHvyUdJOmM2NebJD0Yt4oAADuDXg0AjRt9GgASJLIdYw5wzvU3s8mS5JxbZ2apca4LALBj6NUA0LjRpwEgQbYn6Cg3s7AkJ0lm1kJSNK5VAT9B4YZCPXP985rxv++VnZelk4eepAG/2K/e8RXlFbr5l7erpKhUt3/w96rli75frP/e8LyWzV2uNgWtdc5NZ6pDz/aJmAKwM+jV2K0UbSjSGze/oTkT5igzL1NHXXKU9hq0V73jK8or9NBZD6msqEzDRg2rWh6tjGrcI+M0+a3JKi0qVbP2zXTeQ+cpIycjEdMAdgR9GruV4g3FeuvWtzR3wlxl5mXq8IsPV99j+9Y7vrK8Uv85+z8qKyrT0LeGSpIWTlmo5694vsa48uJyDbl1iPY8Ys+41o/ktj1Bx/2SXpPU0sxukTRE0l/iWhXwE7xw88uKpIR158e3avH3i/XAJQ+rfY92atutTZ3jxz7xgbKbZaukqLRqWUV5hR667BEdcc5AHXb6ofr05c/10GWP6KbRNyiSsj3/bICEo1djt/L2XW8rnBLWVWOu0vJZy/XcFc+pdffWatm1ZZ3jP//v58rKy1JZUVmN5eMeGadFUxfp/MfPV5PWTbRy7kpFUunTaJTo09itvHP3OwpHwvrT6D9p+azlenHYi2rVvZVaFtTdp7949gtlNa3Zpzv266jh44ZXfT1/0ny9dNVL6npQ17jXj+T2o8/ocM49J+lqSbdJWibpZOfciB/bzsw6mNk4M/vOzKab2eU/vVygYaVFpZr83jc66bITlJ6Zpm79u2rvgX014a2JdY5fvXi1Jo76UoPOP7rG8lkTf1BlZVRHnnO4UlJTdMTZA+WcNHPCrERMA9hhO9Or6dPwS1lxmWZ8OENH/P4IpWWmqVO/Turx8x765p1v6hy/bsk6TR0zVYeed2iN5cUbizX+xfE66bqTlNcmT2amVl1bKSUtJRHTAHYI59TYnZQVl2nGuBka+PuBSs1MVcd+HbXHoXvo23e+rXP8uqXr9O2Yb3XwuQc3uN+po6dqz8P3VGoGd20hvn70Iw8z6yipSNJb1Zc55xb+yKYVkoY55742sxxJk8zsPefcdz+pYqABKxasVCgSUqvOW5Pmdj3a6YevZtc5/qVbR2rw5ScqJa1ms106Z5na79FWZla1rP0ebbV09jL1PqRXfIoHfoKd7NX0afhizcI1CoVDyu+UX7WsdffWmv/1/DrHj757tI66+ChF0mqetqyYvUKhcEjTP5iu8S+MV1pWmg44/QAd8OsD4lk+sFM4p8buZEufbt6xedWyVt1bacHXC+oc/+497+qIi4+o1aer2xKenHbXabu8XmBb23Nt59vy7iU0SemSukiaKal3Qxs555bJS6vlnNtkZjMktZNEU0bclBaVKiMrvcayjJx0lRSW1Bo7+f1vFI1Gtc9Re2vmxB+22U+ZMrJr3t+dnpOhksJSAY3UDvdq+jT8UlZUprSstBrL0rLTat2WIkkzxs1QNBrVnofvqXmT5tVYt3HlRpVsLtGahWs09PWhWrNojZ7+w9PK75ivrgdwWTQaHc6psdsoLy6v3aez6u7T33/0vaKVUfUc2FPzJ82vd5/ff/S9MptkqlP/Tru6XKCWHw06nHM1njhjZv0lXbIjBzGzzpL2kTRhR7YDdlRaZpqKtwk1SjaXKH2b8KO0qFSv3vuGLn3oonr2k1rPftLqHA/47af2avo0Eik1M1Wl2wTHpYWlSs2seXVdWXGZxj4wVmffd3ad+9lyi8rA8wcqJT1Frbu3Vp+j+2jW57MIOtDocE6N3UlKRsp29+kPHvhAp997+o/uc+rbU9X3+L41rpgG4mWHn9YVu2xuu68JNbNsSa9IGuqc21jH+gslXShJ7Tq029FygBpadWqpaEVUKxasVKtO3u0ri2cuUdturWuMW7lwldYsXaN7zv2HJKmivFLFm4t19WHX6urnh6lt1zZ6/+lxcs5VNeMls5Zo4Bk17w8HGqsd6dX0aSRa847NFa2Mas3CNVWXRa+YtaLWA+7WLFqj9UvX64kLn5AkVVZUqmRzie4adJfOf+J8tereyhtY7ZyZE2jsLnblOXX1Pt22Q9tdWieSU519evYKtShoUWPc2kVrtX7Zej190dOSvD5durlU9x5/r3732O+U1zZPkrRhxQbNnzxfxw8/PrETQdLanmd0/KnalyFJ/SUt3Z6dm1mKvIb8nHPu1brGOOcekfSIJO3dfy+3PfsF6pOWmaZ9jtpbbz3wts658Uwt+n6Jvhn3ra5+9k81xrXt1ka3vb/1VbJzJ8/Vi7eO0LUjrlFO02zltWyiUMj04bMf6+enHazPRn4hSepxwB4JnQ+wvXa2V9On4YfUjFTtefie+vCRDzX4usFaPmu5vv/ke53/2Pk1xrUsaKk/vbX1f+1F3y7S6LtG6/fP/F5ZTbMUCofUqV8nffLkJzp+2PFat2Sdpr03TUP+PiTRUwJ+VDzPqav36b79+9Cn8ZOlZqSq58Ce+vjRj3XCtSdoxawVmvXJLJ336Hk1xrUsaKnL39z6fNxFUxdpzD1jdMHTFygzL7Nq+bfvfKsOfTuoWftmiZoCktyPvnVFUk61P2ny7i8c/GMbmfeRyuOSZjjn7v0pRQI74ozrT1V5abmuOuxaPX71Uzrz+tPUtlsb/TBpti7ff5gkKRwJq0l+btWfzCZZslBITfJzFQqHFEmJ6KL7L9CEtybqTwddoy9eG6+L7r+AV8uiMdvhXk2fhp9+cfUvVFFSoTuPvVMj/zJSJ1xzglp2bakFkxfolsNukeT16pz8nKo/GbkZspApJz9HobB3CjPk5iHasGyD7jj6Dj13xXM6/PeHq2BAgZ9TA+rDOTV2K8dfdbzKS8t173H36tUbXtVxVx+nlgUttXDKQt1++O2SpFAkpOzm2VV/MnIzZGbKbp5d1aclaeo7U7XX8Xv5NRUkIXOu/tDXzMKS7nDOXbnDOzY7RNKnkr6VFI0tvtY5N7q+bfbuv5cb89moHT0UdnNvznvT7xJ8dVi7n/tdgm+ckvdDp15N+01yzu23K/a1s72aPr1jVpes9LsE37w2p97/JZLC2T1O9bsE3zRLy//xQQHVNL2F7306tu0O9eq+/fu4Nz9+Zadr3Z2lhJL3laWPTHvS7xJ8ddledT93LxkUVmz2uwTfdMntUW+frvfjaTOLOOcqzKzhlyHXwzn3mWrcNQsA2NV+Sq+mTwNA/HFODQCJ19B1+BPl3Ts4xczelDRCUuGWlfXdyw0ASCh6NQA0bvRpAEiw7XngQLqkNZKO0NZ3fztJNGUAaDzo1QDQuNGnASBBGgo6WsaeDj1NW5vxFsl7Yz0ANC70agBo3OjTAJBgDQUdYUnZqvueQJoyADQO9GoAaNzo0wCQYA0FHcucczclrBIAwM6gVwNA40afBoAECzWwjqc7A0DjR68GgMaNPg0ACdZQ0HFkwqoAAOwsejUANG70aQBIsHqDDufc2kQWAgDYcfRqAGjc6NMAkHgNXdEBAAAAAACwWyHoAAAAAAAAgUHQAQAAAAAAAoOgAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBEfG7gOrCFlFualO/y/CFyfwuwTdDuv7a7xJ8NWn1RL9L8M39k17zuwTsoJCFlZWS63cZvshNyfO7BN+c3SPb7xJ8NXLO636X4Jsuue39LgE7KGIpap7eyu8yfBGxRvWrTUL9rtfZfpfgqzmbfvC7BN88NGWE3yU0SlzRAQAAAAAAAoOgAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBQdABAAAAAAACg6ADAAAAAAAEBkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwCDoAAAAAAAAgUHQAQAAAAAAAoOgAwAAAAAABEbE7wIA7Dob123SfVf9S5M+mawmzXL122vO1RGnHFZr3IiHX9V7Iz7UyiWrlNssRyeee7x+fdEvq9ZP/2qGHv7bY1o0e7Fad2ipS2+5WH0G9ErkVAAgsIo2FGnkTa9o1v9+UFZelgZddqz2Oa5fveMryiv0j9PuV2lRqa4b8+da6yeN+lov3zBCv7r+lxpwyv7xLB0AksKGdRt16xV3auJHXymveRNddO0FOvZXR9Ua9+yDL+qdl8Zo+eIVatKsiX7525N19h9Or1p/yn6nae2qdQqHvOsL+u7fR/98+e6EzSOZxS3oMLN0SZ9ISosdZ6Rz7q/xOh4A6cG/PKxISkQvTX5Gc6bP0/Xn3aSCXl3UuUfHGuOcc7rqH1eoYM/OWrpgma49669q0SZfAwf/XBvXbdJff3uz/njbxTr4uIP00Ruf6K+/+7ue+uxR5eRl+zQzxAN9GvDH67e/oXAkrOvfv05LZy7Tk5c/pTZ7tFHrrq3qHP/J058qu2mWSotKa60r2liscU98pFb1bIvdH70aSLx7hv9DKSkpenv6q/ph2mwNO+vP6t67qwp6dqk50Dnd8MC16tqrQEvmL9XQ065Sq7YtdPQpR1YNueuZWzXgsP0SPAPE89aVUklHOOf2ltRP0iAzOzCOxwOSWklRiT5753/6zVVnKSMrQ30G9NJBRw/QB6+OqzX21It/pe59uyocCatD1/Y66JgDNP2rGZKk7yZ9r6Yt8/TzEw5ROBzWkb88XE2aNdHnY75I9JQQf/RpIMHKiss07YPpOuaSo5WWmaYu+3RWr5/vqclvT65z/Nola/X16Mka+LuBda4f868xOvj0g5SVlxm/ouE3ejWQQMWFxRr39ie6cPjvlJmVqb0P2EuHHvszjRkxttbYsy89Qz322kORSESdunXUoccerKkTp/lQNbYVt6DDeTbHvkyJ/XHxOh6Q7BbPXaJwOKT2Be2qlnXZs4sWzFrY4HbOOU2bOF2d9qh21YerPWb+9w3vB7sf+jSQeKsWrFYoElKLTi2qlrXZo41WzFlR5/g37nhTgy49VilpKbXWLZq2SIu/W6IDhhwQt3rhP3o1kFgL5y5WOBJWx64dqpZ1691Vc2fOb3A755ymTJiqLj0711j+t0tu0XG9BuvyU6/UD9Nnx6Fi1CWuDyM1s7CZTZG0UtJ7zrkJ8TwekMyKC0uUmVPzE72s3EwVby5ucLv/3vuColGnY0717jvstW8PrVmxVuNe/1gV5RV6b8QHWrZguUpLal8yjd0ffRpIrLKiUqVlpdVYlp6dXudtKdM+nK5o1KnPEb1rrYtWRvXabW9o8DUnKRTi2fJBR68GEqe4sFhZ2TXPqbNzslVUWNTgdo/d9ZRc1OmE04+rWva3f/9Fr371ol776iX1P3gfDT3tKm3asCkudaOmuP5kdM5VOuf6SWovaYCZ9dl2jJldaGZfmdlXq1evjmc5QKBlZKWraFPNBly0qUgZ2Rn1bvPGU6P0/isf6u9P36DU2KeFuU1z9bfHrtOrj76h0/Y5V1999LX2OWRv5bduHtf64Y8d7dNr6NPAT5KamabSwpqhRmlhqdIya4YfZcVlGv3PdzT46hPr3M//RoxXm+6t1WmvjnWuR7D8WK+mTwO7TkZWhgo31zynLtxcqMys+m8RHPH4q3rn5Xd1z3O3KTUttWr53gP6Kj0jTemZ6frN5Wcpu0m2poz/Nm61Y6uEvHXFObfezMZJGiRp2jbrHpH0iCT137c/l+EBO6l9QTtVVka1ZN5StevSVpI0d8b8mrekVPPui+/p5Qdf0d0jb1OLNvk11u11UB/96+17JUmVFZX6zcEX6FcXnhzfCcBX29un99l3H/o08BO06JSvaEVUqxeuVn5Hr/cum7Ws1sNEVy9crXXL1umh//uPJKmyvFIlm0v096Nv0R+evkSzJ87RvElz9f3nMyVJxRuKtWTmUi2duVQnDx+c2EkhYerr1fRpYNfpWNBelRWVWjR3sToUtJck/TB9jgp6dK5z/FvPj9Z///W8Hnr9frVs27LBfZuZ5PgnmgjxfOtKC0nlsYacIeloSXfE63hAskvPTNfBgw7SM3c/pyvuukxzps/V/8ZO0H2v3Vlr7IevfaQn7/yv7nzpFrXp1LrW+tnT5qhzj04qLSnTM/c8pxZt87XfwP6JmAYSiD4NJF5qRqp6H9FbYx96T0Nu+JWWzlyq6R9/p0uevLjGuFZdW+na0cOrvl7wzQK9fsebuvz5y5TVNEun3jhEFaUVVeufufJZ9T2qj/YfzOtlg4ZeDSRWRlaGBh5/qB694wn9+d6rNGv6bH065nM9MuqBWmPfHfmeHr71UT346j/UrnPbGuuWL16hFUtXqle/nopGoxrx+GvasHaD9hpQ6+JZxEE8r+hoI+lpMwvLu0XmZefcqDgeD0h6l95yke698n6d2u8c5TbN0WW3XKzOPTrq2wnT9Zdzb9QbM1+WJD1117PauG6TLjthWNW2R/xyoC6/7RJJ0oiHXtXEcZMkSfsd1l9/ffTaxE8GiUCfBnxwyp8Ha8SNr+imI29WZl6mTvnzyWrdtZXmfT1PT1z2lP7++Y0KR8LKyc+p2iajSaYsZFXLMnIypK2rFUkJKz0rXRk56YmeDuKPXg0k2JV3XKFbh96h43ufoibNcnXVHVeooGcXTRk/VX8642p9OG+MJOk/tz+uDes26nfH/r5q22OHHK1r7hqmos1Fuuvq+7Rk/lKlpqeqe+9uuvf5O9SkWRO/ppVUzDWiS2f679vffTrhY7/L8IXJ/C7BN8UVhX6X4KtJqyf6XYJv7p/0mt8l+Obtk5+c5Jzb7V6qvs+++7iPx9d+ZXEyCMf3sVaN2oripX6X4KuRc173uwTfdMlt73cJvjm12zn06d1MxBJyV36jtDLJ+/Ty4mV+l+Cbh6aM8LsE3zxz7IP19unkPWsDAAAAAACBQ9ABAAAAAAACg6ADAAAAAAAEBkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwCDoAAAAAAAAgUHQAQAAAAAAAoOgAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBQdABAAAAAAACg6ADAAAAAAAERsTvArYVStLsxcz8LsE3mZFsv0vw1c9a/dzvEnxz7DN/9LsE7CCTFFJy9quQhf0uwTctM9r6XYKvztrjdL9L8E3f28/wuwTsIJMpnKT9KmTJ+XuEJLXIaON3Cb5qnt7S7xJ888xT5/tdQqOUvN0AAAAAAAAEDkEHAAAAAAAIDIIOAAAAAAAQGAQdAAAAAAAgMAg6AAAAAABAYBB0AAAAAACAwCDoAAAAAAAAgUHQAQAAAAAAAoOgAwAAAAAABAZBBwAAAAAACAyCDgAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAQGAQdAAAAAAAgMAg6AAAAAAAAIFB0AEAAAAAAAKDoAMAAAAAAAQGQQcAAAAAAAgMgg4AAAAAABAYBB0AAAAAACAwCDoAAAAAAEBgEHQAAAAAAIDAIOgAAAAAAACBQdABAAAAAAACg6ADAAAAAAAERsTvAgBglyirlCatkVYWS6khqU9TqUN23eOmrpVWFHtfd8mRejWtOWb2Ru9PaaWUGZEObCnlpMR/DgAQcNHiChWNmqPyuRsUyogo44iOSu2TX2ucq4iqaOx8lc9cJ1VGFemQo8zjChTKTZUklXy5XGVTV6lyZZFSezdX1kndEj0VAAim0kpp4kppWZGUFpb2bi51zqk9rqxSmrTaGydJ3XOlvs23rv9gibShVKp0UnaK1LeZ1L6Oc/M4iXvQYWZhSV9JWuKcOyHexwOQpKas9a5R+0UHaX2Z9MUKqUmqFDsprjJ1nddwj23vNfLPVnhhxpYGPm+TNH+T9LNYuFFY4QUnAUafBpAoRe/Mk8KmvCv2VeXyQm16aabCrTIVbpFZY1zpxOWqXLxZuRf0laVHVPT2XBW9O0/Zv+4hSQrlpCr9kHYqn7Neqoj6MZWEo1cDSIivVkkhk/6fvfsOk6o83zh+PzvbKyx1aUtHERUVUYklKtiVRFHRaGLUmKgxMTGWWBJ7rIkmGo2x/yRW7L2AXUHARhFEemdZYHubeX9/zLDssgVYmDmzZ76f6+LKzjnvnPO8THj2eM8pP+0nbaiWPlgpdUyV8tIaj5tRJAVD0gmFUlVQmrxcykqR+ueG1+/dOXwsnmRSUVV4/XHpUkZszrWIxdH77yXNicF+ACSqupC0vDx8ZkZyktQ5XSrIlJaUNR27qkIalBcel5UiFWZLiyPjnJO+2yDtkR8OSMzCCXRqILbziT36NICoczVB1X5XrIxDestSA0ruk6vUQR1V/W1Rk7HBDVVKHpCnpOxUWXKSUod2UnBtZf361F3ylTokX0kZCXW2Hb0aQHTVhaRlZeGzL1KSpC4ZUs+s8BeBW1peLu0aOfbOjgQcC0o2r++YFg45JMkkhSRV1MViFpKiHHSYWS9Jx0p6MJr7AZDgymrDjbTh5SV5qVJJ7ba9f9O4ymD4T0mt9MZS6c1l0uz14QDEp+jTAGIlWFwlJZkCnTLqlwW6ZSq0tqLJ2LThXVW3tFSh0hq52qCqZxYpZWCHWJYbV+jVAGKipDb8RV/DM6I7pEoba7bt/Ru2GPfBCunpH6S3l0ldM6T8tObfFwXRPm/kLkmXSWrmoh4A2EnqnJRsjZelJDV/OnO3DGneRmmfzuFLVxaXhU+7k6TKSMq8ulI6vIdUG5I+WR0+xa6fb9sYfRpATLiaoCyt8RlylhaQqw42GRvIT1dSbpo23j1DMinQNVOZR/WLVanxiF4NIPrqQuFj6IZaOqYuyAx/Ibh/N6mqLnw2R3CLcYf0kEIufEZ1SU04RImRqJ3RYWbHSVrjnJu+lXHnmdk0M5tWVNT01EUA2KpkC4cdDdWFwqfSbWnPfClg4WT5szVSr6zN1woGIs13cF74cpWslHDAsaqy6XZ8oG19el2MqgPgN5baNNRoLvyQpIo3F0nBkPIuGaEOl49Uyi75KnsyMa/a2JZezfE0gJ0iOSn8RV9DtS0cU+/TJXzs/Opi6cNVUmFO+L53W0oyqUeWtLJSWlYenbqbEc1LV34k6QQzWyTpKUmHmdkTWw5yzj3gnBvhnBvRuXPTu24DwFZlp4TT4rIGl6psrJFym7l2OzUg7dtFOraPNKan5BS+hnDTdpIUvo4wMbShT3facjUAbJNAfroUcgoWbw6Pg6srlLTFjUglqW51uVL36KKkjGRZcpLS9u2u4IpyhSq28ZJEf9lqr+Z4GsBOkZsSvmS7tMElKBtqwpeEbyktII3qHr5p6bF9wsfU+ektb9ttcaweZVELOpxzf3bO9XLO9ZU0XtIk59wZ0dofgASWnCT1zJRmR+6+v65KWlEh9WnmEVZlteFLVlzkNLpFpdIueZu30ysrfGlLbSh8w6SFpVJBRtPt+AB9GkAsWWpAKbvkq/L9ZXI1QdUtLVXNvPVK273pf5gnF2Sr5tsiuao6uWBI1dNWy3JSlJQZDrBdyMnVhcK9PBR+HK0L+fN+SvRqADGTnBR+BOw3xeFj6rWV4ZuONncJd2nkmDrkpBXl0vyN0rCO4XUlNeFldaHw+oWl4W11bSUI2dlTidmeACCahncKP8v7taXhx8Hu1Sl8I6WiqvB9NsYWhsdtqAk3DFnSVgAAIABJREFU79qQlJ0cPruj4Q2X9uwkfVkUvhlpSlL4sbOFsXvmNwD4WebR/VTxyg/a8I/psoxkZR3dT4EumapdUqKyJ79Tx8tHSpIyRvdR5VuLtPHfX0lBp6QumcoeN6R+O1UfLVPVR8vrX9fMLFL6QT2VcUjvmM8JAHxlRBdpyhrp+YXhszZGdAk/WnZNZfjmoicPCI9bXxV+xGxNKPxAgFHdNj+C1kmaWRw+w9oiDwwY1b31Mz52spgEHc659yW9H4t9AUhQqQHpgG5Nl3dO3xxySOEzNnpltbydlCRpZNedX1+co08DiIWkjGRlnzKkyfKUPrn1IYckJWWmKOung1rcTsYhvRMy1KBXA4i6tIB0cEHT5V0zNoccktQnJ/ynOXmp0hHe9uioPl4WAAAAAAAglgg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQAQAAAAAAfIOgAwAAAAAA+AZBBwAAAAAA8A2CDgAAAAAA4BsEHQAAAAAAwDcIOgAAAAAAgG8QdAAAAAAAAN8g6AAAAAAAAL5B0AEAAAAAAHyDoAMAAAAAAPgGQQcAAAAAAPANgg4AAAAAAOAbBB0AAAAAAMA3CDoAAAAAAIBvEHQAAAAAAADfIOgAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6AAAAAACAbxB0AAAAAAAA3zDnnNc11DOztZIWe7T7zpKKPNq31xJ57lJiz5+5e6fQOdfFw/23icd9WvL+c/MSc09MzN079Om28fpz8xJzT1yJPH8v595in46roMNLZjbNOTfC6zq8kMhzlxJ7/sw9MefeniXy58bcmXuiSeS5t2eJ/Lkx98Scu5TY84/XuXPpCgAAAAAA8A2CDgAAAAAA4BsEHZs94HUBHor7uZtZ0My+MrOZZvasmWXuwLYeNbNxkZ8flPRKK2N/bGaj2rCPRWbWua01xlDcf/ZRlMhzb88S+XOL67nTp6Mmrj/3KEvkubdnify5xf3co9WrJVWa2dBWxtKr/Ssu5849OtAumFmZcy478vMESdOdc39vsD7ZOVe3jdt6VNKrzrnntmHstZLKnHN3bGe9iySNcM4l6k2JACQY+jQAxD96NRIFZ3SgPfpI0sBIMvyRmb0sabaZBczsdjP7wsy+MbNfS5KF3WNmc83sXUldN23IzN43sxGRn48ysxlm9rWZvWdmfSX9RtIfIsn3QWbWxcwmRvbxhZn9KPLeTmb2tpnNinz7aLH9KwGAuEKfBoD4R6+GbyV7XQCwPcwsWdLRkt6MLNpb0jDn3EIzO0/SRufcvmaWJukTM3tb0l6ShkgaKqmbpNmSHt5iu10k/VfSwZFt5Tvnis3sfjVIn83sf5L+4Zz72Mz6SHpL0q6S/irpY+fc9WZ2rKRzovoXAQBxij4NAPGPXg2/I+hAe5FhZl9Ffv5I0kOSRkma6pxbGFl+hKQ9bPO1gnmSBkk6WNKTzrmgpBVmNqmZ7e8v6cNN23LOFbdQx2hJQ83qw+VcM8uO7OPEyHtfM7P1bZwnALRX9GkAiH/0aiQEgg60F5XOueENF0QaY3nDRZIucs69tcW4Y3ZiHUmS9nfOVTVTCwAkMvo0AMQ/ejUSAvfogJ+8Jel8M0uRJDMbbGZZkj6UdGrkesMCSYc2897PJR1sZv0i782PLC+VlNNg3NuSLtr0wsw2/aL4UNLpkWVHS+q402YFAP5BnwaA+EevRrtH0AE/eVDhawVnmNlMSf9R+KylFyR9H1n3uKTPtnyjc26tpPMkPW9mX0t6OrLqFUk/3XTjJEm/kzQicmOm2QrfWEmSrlO4qc9S+HS7JVGaIwC0Z/RpAIh/9Gq0ezxeFgAAAAAA+AZndAAAAAAAAN8g6AAAAAAAAL5B0AEAAAAAAHyDoAMAAAAAAPgGQQcAAAAAAPANgg4AAAAAAOAbBB0AAAAAAMA3CDoAAAAAAIBvEHQAAAAAAADfIOgAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6EDfMbJGZjfa6ji2ZmTOzgdF+r5n9yMy+N7MyM/vJVsb2jWw7uS11AUAstLWvm9m1ZvZEG/e5ze+1sEfMbL2ZTd2G8Y+a2Y1tqQsAvBKvx9ixxvFzYiHowE5jZmeZ2bdmVmFmq8zsPjPr4HVdrYmzg9brJd3jnMt2zr3odTEA0B77+nY6UNIYSb2ccyO9LgYAmtMee3E0jrF3JARH4iHowE5hZpdIulXSpZLyJO0vqVDSO2aW6mVt7UihpFleFwEAUsL09UJJi5xz5V4XAgDNSZBeLM6ywM5G0IEdZma5kq6TdJFz7k3nXK1zbpGkUyT1lXRGZNy1ZvaMmT1uZqVmNsvMRjSzve6RxLpTg2V7m9laM0tpZnyr2zWzXc3sfTPbEFl3QmT5eZJ+JumyyOUir7QyzdGRy0o2mNm9ZmYNtn+2mc2JnPr8lpkVtvD39KiZ3W9m70Tq/GDTWDP7QVJ/Sa9Eaknb8jRDUmwAseJ1X49IbaWv9zCziZH3LzSz37Uwj02nKZ9nZivMbKWZ/Smy7hxJD0o6INJ3r4t8a/rxFtto8+WLALAjvO7F0T7GjvTXC83se0nfR5bdbWZLzazEzKab2UGR5UdJulLSqZFtfh1ZnmdmD0X6+3Izu9HMApF1ATO7w8yKzGyBpGO38yNAO0bQgZ1hlKR0Sc83XOicK5P0usKnBW9ygqSnJHWQ9LKke7bcmHNulaT3FW7im5wp6SnnXG0LNTS73UjTfkXS25K6SrpI0gQzG+Kce0DSBEm3RS4XOb6VOR4naV9Je0TqOjKy/bEKN90TJXWR9JGkJ1vZzs8k3SCps6SvIvuXc26ApCWSjo/UUt3KNgAg2uK5rycp3Ne/ltRT0uGSLjazI1uZz6GSBkk6QtLlZjbaOfeQpN9I+izSd//ayvsBwAvx3It31jH2TyTtJ2lo5PUXkoZLypf0P0nPmlm6c+5NSTdLejqyzT0j4x+VVCdpoKS9FO7z50bW/UrhY/i9JI2QNK6VOuAzBB3YGTpLKnLO1TWzbmVk/SYfO+ded84FJf2fpD2beY8kPabNKXVA0mmR8S1pabv7S8qWdItzrsY5N0nSq5HtbY9bnHMbnHNLJE1WuAFL4YPkvznn5kTmf7Ok4S2d1SHpNefch5Eg4yqFv0nsvZ21AEC0xXNf31dSF+fc9ZG+vkDSfyWNb2Vb1znnyp1z30p6RNv/OwAAvBDPvXhnHWP/zTlX7JyrlCTn3BPOuXXOuTrn3J2S0iQNae6NZtZN0jGSLo70+DWS/qHNvw9OkXSXc26pc65Y0t+2sza0YwQd2BmKJHW25q+tK4is32RVg58rJKW38L6XJA01s34Kp9UbnXOt3RG/pe32kLTUORdqsH6xwt8Cbo8tt58d+blQ0t2RU/Y2SCqWZK1sf+mmHyJpfHGkRgCIJ/Hc1wsl9djUdyO990pJ3VrZ1tIGPy8WfRdA+xDPvXhnHWM37M8ysz9Z+JLwjZH+nqfGgU5DhZJSJK1s8PvgPwqfYaJNNW5RHxIEQQd2hs8kVSt8+UY9M8uWdLSk97Z3g865KknPKJw4n6nWk+bWrJDUO3Kq8yZ9JC3ftKs2bneTpZJ+7Zzr0OBPhnPu0xbG15+9Efn7yY/U2JxySZkNXnffwVoBYFvFc19fKmnhFn03xzl3TCvvaXjmXB9tY981M/ouAC/Fcy/eWcfY9eMi9+O4TOEzMTo65zpI2qjwl4jNbXOpwn8/nRv8Psh1zu0WWb9STfs/EgRBB3aYc26jwjdK+peZHWVmKWbWV+Emukxtb6CPSzpL4WsD27qNKQqnz5dF6vqxpOMVvtZQklYrfBPQtrpf0p/NbDep/oZIJ7cy/hgzO9DCd8m+QdLnzrmlLYz9StL4SN1cVwggZuK8r0+VVGpml5tZRuRmc8PMbN9W3nONmWVGevUvJT3dwrivJe1mZsPNLF3StW2sEQB2WJz34mgcY+cofL+NtZKSzewvknIbrF8tqe+mcMU5t1Lhe4TcaWa5ZpZkZgPM7JDI+Gck/c7MeplZR0lXbPcs0W4RdGCncM7dpvCpw3dIKlG4+S2VdHhbb6zpnPtEUkjSDOdcm041c87VKNx0j1b49L5/S/q5c+67yJCHFD59b4OZvdiG7b+g8CO/njKzEkkzI/tqyf8k/VXhS1b2UeQayRZcI2mApPUK/5L73/bWBwBtFcd9PajwzeWGS1qocG9/UOHTm1vygaT5Cn/7eYdz7u0Wtj1P0vWS3lX4CQAfNzcOAGIljntxNI6x35L0pqR5Cl9mUqXGl548G/nfdWY2I/LzzyWlSpqt8DHzcwpf1iOF79/0lsIh9gxtcVNX+Js5t6Nn7gPRY2aTJP3POfeg17XsKDN7VNIy59zVXtcCAF6JZV+PfPO5UFJKCzfzA4CE5KdjbKA5zd2gBogLkdOQ95Y01utaAAA7jr4OAN6jFyMRcOkK4pKZPabwqcMXO+dKva4HALBj6OsA4D16MRIFl64AAAAAAADf4IwOAAAAAADgGwQdkiKPa5prZvPNLGEeO2RmD5vZGjOb6XUtsWZmvc1sspnNNrNZZvZ7r2uKFTNLN7OpZvZ1ZO7XeV1TrEUeR/mlmb3qdS3YNonapyV6Nb2aXk2vbh/o0/Rp+jR9Op4kfNBhZgFJ9yr8aKShkk4zs6HeVhUzj0o6yusiPFIn6RLn3FBJ+0u6MIE+92pJhznn9lT48YxHmdn+HtcUa7+XNMfrIrBtErxPS/RqejW9GnGOPk2fpk/Tp+NNwgcdkkZKmu+cWxB5HvRTSpA7EDvnPpRU7HUdXnDOrXTOzYj8XKrwP9Ce3lYVGy6sLPIyJfInYW7WY2a9JB0riceptR8J26clejW9WhK9GvGPPk2fpk/Tp+MKQUf4H+LSBq+XKUH+cSLMzPpK2kvSFG8riZ3IaWZfSVoj6R3nXMLMXdJdki6TFPK6EGwz+jTo1fRqxDf6NOjT9Om4QtCBhGZm2ZImKvyIrRKv64kV51zQOTdcUi9JI81smNc1xYKZHSdpjXNuute1ANh29Gp6NYD4Rp+mT8cbgg5puaTeDV73iiyDz5lZisINeYJz7nmv6/GCc26DpMlKnOtKfyTpBDNbpPBptYeZ2RPeloRtQJ9OYPRqerXo1e0BfTqB0afp04rDPk3QIX0haZCZ9TOzVEnjJb3scU2IMjMzSQ9JmuOc+7vX9cSSmXUxsw6RnzMkjZH0nbdVxYZz7s/OuV7Oub4K/1uf5Jw7w+OysHX06QRFr6ZXi17dXtCnExR9mj6tOO3TCR90OOfqJP1W0lsK3zznGefcLG+rig0ze1LSZ5KGmNkyMzvH65pi6EeSzlQ4ffwq8ucYr4uKkQJJk83sG4UPTN5xzsXdI6GATRK5T0v0atGr6dWIe/Rp+rTo0/TpOGPOJcyNYQEAAAAAgM8l/BkdAAAAAADAPwg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQEWFm53ldg1cSee5SYs+fuaO9SeTPjbknJuaO9iaRPzfmnrgSef7xOneCjs3i8gOKkUSeu5TY82fuaG8S+XNj7omJuaO9SeTPjbknrkSef1zOnaADAAAAAAD4hjnnvK6hXiA71aXkZ3iy72BZjQLZqZ7sW5KGdC70bN/riorVqXO+Z/s38zZvW7d2nTp16eTZ/pM8zBuLiorUuXNnz/YfUsizfXv9uX894+si51wXzwpoo+TsVJfSKdOz/XvZq3fp3M+T/W6yrqhInTz69xpyQU/2u4nXv6cCluzZvr3u03Wu1rN9FxcVK9/Dz/3bL2e2yz6dl5/ruvbq6tn+S4pLlJuf68m+ve5VJcWlys3P8WTfmclZnux3k+J1xcrv5N2/15Qk7/47TvK2V9eGajzZ7ybF69Yrv1NHT/Y986tZLfZp735zNyMlP0N9LxnldRmeePNX//G6BM+kBtK9LsFTmQFvfzF5qTJY7nUJnumU3m2x1zW0RUqnTA28/CCvy/DEB+c+7nUJnqlI4H+rkpSb0sHrEjxTXL3W6xI80zu7f7vs0117ddU/X73D6zI8UVJb6nUJntmnyz5el+CpgszeXpfgmdUVy70uwTMD8nZtsU9z6QoAAAAAAPANgg4AAAAAAOAbBB0AAAAAAMA3CDoAAAAAAIBvEHQAAAAAAADfIOgAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQAQAAAAAAfIOgAwAAAAAA+AZBBwAAAAAA8A2CDgAAAAAA4BsEHQAAAAAAwDcIOgAAAAAAgG8QdAAAAAAAAN8g6AAAAAAAAL5B0AEAAAAAAHyDoAMAAAAAAPgGQQcAAAAAAPCNZK8LiKZgeY1WPTVT5XPXKZCVoi7HDVbuPj2ajFt2/zRVLFhf/9oFQ0rtmqV+lx8oSVpyz1TVrCyVqwsppVOmOh09UDm7d4vZPADAr+rKa7R8wtcqm1Ok5KxUdRu7izrs27PJuEX3TlHF/OL61y4YUmq3bA266hBJ0upX5qrkm1WqXlWmLkcNVLdjh8RsDgDgZ6UbSnXXZfdqxodfKTc/V2dddoYO/cnBTca98ODLeuXR17VxfYkyMtN18PEH6pwrf6FAckCS9Pgd/9Nnb0/R0vnLNP6ik3XGH8bHeioAEkhUgw4zO0rS3ZICkh50zt0Szf1tafVzs2WBJA284VBVLS/V8gemK61HjtIKchqN6/WbEY1eL/nXFGUO6lT/uuuJuyqtW5YskKTKRRu09L4vlHHlQUrOS4/JPAAgWrzu0yufnikLJGmXv41R1bISLb5vqtJ75iq9R+M+3ffC/Rq9XnDXp8oe3Ln+dWqXTHX/ya4q/mhxTOoGgFjyslf/+5oHlJySrP9Nf0QLZi/UX395k/oP7avCwX0ajdt/zL4ac/Lhys7LUumGUt30m9v00iOv6sRfjZUk9ejbXWf/+ed6Y8JbsSodQAKL2qUrZhaQdK+koyUNlXSamQ2N1v62FKquU+k3q9X5mEFKSktWZv+Oyh7WVSXTVrT6vtp1FapcsF55+24+8yO9R44sEPmrMklBp9oNVVGsHgCiLx76dMlXK9XtuCEKpCcra2C+cnbvpg1Tl7X6vpp1FaqYX6wO+/WqX9Zx/97K2a2rktJ9faIigATkZa+uqqjSJ298rjMvOU0ZWRnabd+h2m/0vpr0/PtNxhYUFig7L0uS5JxTUlKSVi5aVb9+9LjDtO+h+ygjKyMWpQNIcNE8Ihwpab5zboEkmdlTksZKmh3FfdarWVshSzKlds2qX5bWI0cVPxS38i5p4xcrlNG/o1I6ZTZavuyB6aqYt06uLqTMXTorvXdeVOoGgBjytE9XrymXkkxp3bLrl2X0ylX59+tafd+GKcuUOTBfqVv0aQDwKc969fIFKxQIJKlX/82XFPbfta++nTKr2fGTX/xQ91x1vyrLKpWbn6tzrz4r2iUCQLOiGXT0lLS0wetlkvZrYexOF6qua/LNXlJGskJVda2+r2TaCnUa07/J8l7n7SMXDKl87jrVrC6TJdlOrRcAPOB5nw6kpzRalpSRrGB16316w5Rl6nLUoGiWBgDxxLNeXVlRpcycxqFyVm6mKssrmx1/6E8O1qE/OVjLF67QexPfV4fOHWJRJgA04flTV8zsPDObZmbTgmU1O227SWlNQ41QVdPwo6GKBetVV1KtnOHdm681kKTsoV1UPnedymau2Wm1AkA8i2afDlbVNloWqqxTIK3lPl0+v1h1JdXK3atgp9UBAO1dwz5dUlyy07abkZmuitKKRssqSiu3evlJz349VDi4t/599X92Wi0AsD2iGXQsl9S7wetekWWNOOcecM6NcM6NCGSn7rSdp3bJlAs51awtr19WvbxUad2zW3xPydTlytmjm5JaOciWJIWcaooqWh8DAPHP0z6d1jVLCjlVrymrX1a1vKTJDaMb2jBlqXKHd1eAe3EASBxb7dUN+3Rufu5O23HP/j0UDIa0fOHme9wtmLNIhYN7t/KusGBdUCuXrN5ptQDA9ohm0PGFpEFm1s/MUiWNl/RyFPfXSFJasnL26Kai179XqLpOFQvWq2zmGuWOaPp4WUkK1QRV+tUq5Y5s/FjD6tVlKpu9VqGaoFwwpI3TVqjih2JlDugYi2kAQDR53qdzhxdozavzFKquU/kPxSr5ZrU6jOzV7PhQTVAbZ6xUh/2bHmC7YEih2qDknBRyCtUG5UIu2lMAgFjwrFenZ6Zr1FH764m/P6mqiirN+mKOPn9nqg478cdNxr755DvaULRBkrRk3lI98+/nteeo3evX19XWqaaqRqGQU7AuqJqqGgWDwVhMA0ACitpXYs65OjP7raS3FH4U1sPOuebvXBQl3cYN1aonZ2r+NZMVyExRt5OHKq0gfEPSZf+ZrsG3jakfW/btaiVlJCtzUP4WE5HWvTlfK1aXycyU2iVTPX4xnJuRAmj34qFPF5w6TMuf+FpzrnhHyVkp6jF+d6X3yFH5/HVafO9UDf3H0fVjS75ZpUBGirIGd2qyneUTvtGGKZuf1rL2zfnqecae6njA1r91BIB45nWvvvDG8/SPS+/RaXufpdyOObrwxl+rcHAfzZw6W3/5xQ16fs6TkqTZ07/T43dMUGV5lfI65eqgY0bpzEtOr9/OP6/4t959bnL966fveU5/uOMijTn5sFhNBUACMefi5xuv9D55ru8lo7wuwxOTf5W41zCmBtK9LsFTmYGsrQ/yqcpg+dYH+VSn9G7TnXMjvK5je2UUdnADLz/I6zI88cm5j3tdgmcqEvjfqiTlpiTuDRWLq9d6XYJnemf3b5d9etAeA90/X73D6zI8UVJb6nUJntmnyz5el+CpgszE/WJjdUWTq44TxoC8XVvs057fjBQAAAAAAGBnIegAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQAQAAAAAAfIOgAwAAAAAA+AZBBwAAAAAA8A2CDgAAAAAA4BsEHQAAAAAAwDcIOgAAAAAAgG8QdAAAAAAAAN8g6AAAAAAAAL5B0AEAAAAAAHyDoAMAAAAAAPgGQQcAAAAAAPANgg4AAAAAAOAbBB0AAAAAAMA3CDoAAAAAAIBvEHQAAAAAAADfIOgAAAAAAAC+kex1AQ31yuusv51wltdleGLW+plel+CZrORMr0vw1G75w70uwTPBUJ3XJWA7dc/J0yWHH+91GZ6oqCvzugTPTC+a5nUJnjqg24Fel+Ah53UB2E7ZKdkJ+//Zd5e/7XUJnvlwxUdel+CpE/ud5HUJnjEzr0uIS5zRAQAAAAAAfIOgAwAAAAAA+AZBBwAAAAAA8A2CDgAAAAAA4BsEHQAAAAAAwDcIOgAAAAAAgG8QdAAAAAAAAN8g6AAAAAAAAL5B0AEAAAAAAHyDoAMAAAAAAPgGQQcAAAAAAPANgg4AAAAAAOAbBB0AAAAAAMA3CDoAAAAAAIBvEHQAAAAAAADfIOgAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQAQAAAAAAfIOgAwAAAAAA+AZBh0+VrC/VtefcqOMHnaif7XeWJr3wfrPjnrlvon51+AU6Ycg4nXnA2XrmvomN1s+aNlu/PfYPOmHIOJ03+kLNnDorBtUDgP+tL96gX47/lfp3GaIRuxyg559+sdlxn3zwqU46+lQNLthN++46qsn6W6+/Q4fuO0a9cvvpjpv+Hu2yASChlG4o022/+YfOGHa2zj/od/ro5U+aHffqw2/owh9frJ/veY7OO+BCPXrj/ylYF2wybtaUOTp5wM/05J3PRLt0IKFFLegws4fNbI2ZzYzWPtCyf139byWnJuuZryboz/+6VHdfea8WzV3cZJxzTpfd9Ue9MOtp3fzE9Xrp0Vc0+aUPJIXDkmvOul4nn3+iXpj9tE45/yRd88vrVLqhNNbTARAl9GrvXPmHq5WamqJvF87QvQ/frSsuvkpzZ89tMi4zK1PjzzxF19x0ZbPb6de/UFffeKVGH3VYtEsG4AH6tLce+uujSk4J6L9T/q3f/eNC/feaR7R03rIm40Ycvrdue/kmPf71Q7rzjVu1aM4Svf7YW43G1NXW6ZEbHteg4QNiVT6QsKJ5Rsejko6K4vbRgsqKKn38+qc669IzlZGVoWEjd9MBY/bTuxMnNRl76gXjNGj3gQokB9R7QC+NOmJ/zfpitiRp9vQ5yu/aUYccd5ACgYBGn3SY8vLz9PEbn8Z6SgCi51HRq2OuorxCr730hi675k/Kys7SfqNG6ohjRuu5J59vMnavEcN18uknqbBvn2a3dcoZJ+vwIw9VVnZ2tMsG4I1HRZ/2RFVFlT5/a6rG//FkZWSla9cRQzRi9N764MWPm4ztXthNWblZ4RfOKSnJtGrx6kZjXnnode154O7q0b9HLMoHElrUgg7n3IeSiqO1fbRs+YLlCgQC6tW/Z/2yAUP7afG8Ja2+zzmnb6fOUuGQwkbLGo9Rs2eGAGif6NXe+OH7BQokBzRgUP/6ZbvtPlRz58zzsCoA8Yg+7Z2VC1cpEAioR7+C+mV9dynUsu+bntEhSR+9/Il+vuc5OnvEb7TouyUac9rmM+3WLl+ryc9+oHEXnRj1ugFIyV4XgJ2vsrxSmTkZjZZl5WSpoqyy1fc9fucEuZDTkaeMkSQN3WcXrVtdrEkvvq+Djz1Qk158XysXr1TRgRnoAAAgAElEQVR1ZXXUageARFBeXq6cnJxGy3LyclRWVu5RRQCALVVVVCkju/ExdWZOhirLq5odf9AJP9JBJ/xIKxeu0gcvfKS8znn16x6+/nGd+odxyshKj2rNAMI8vxmpmZ1nZtPMbFpJcYnX5fhCRlaGKkobhxrlZRXK3KJRN/TiI6/o3YmTdONj1yo1LUWSlNsxV9c9dI0m/vdFnTL8Z/ri/ena66Dh6lzQOar1A4gvDft06foyr8vxhaysLJWWNr7fUVlJmbKzszyqCEB71rBPF61d53U5vpGema7KLb4orCyr3GpYUdCvu3oP6qUH//KIJGnaezNUWValHx13QNRqBdCY52d0OOcekPSAJA3cfYDbynBsg579eyoYDGrZguX1l68smL1QhYObv777zafe1tP3Pqu/T7xVXXo0DjH2PGB33fvaXZKkYF1QZ446R+PO+2l0JwAgrjTs0/2GFdKnd4IBg/orWBfUgvkL1X9gP0nSrG9na8iugz2uDEB71LBP77XPcPr0TlLQr7uCwaBWLlylgn7dJUmL5ixRr0G9tvreYDCo1UvWSJK+/XSWFsxcoHP3u0CSVFFaoaRAkpbMW6rL/3NJ9CYAJDDPz+jAzpeRma4Djx6lx+58QpUVVZr5xWx9+vbnGn1S0zvyv/f8ZD1862O65ckbVVBY0GT9/Jk/qK62TuWlFfrPDQ+pS4/O2vfH+8RiGgDgW5lZmTpm7FG6/YY7VVFeoamffaG3XntH405reu12KBRSVVWVamvr5JxTVVWVampq6tfX1taqqqpKLhRSXV1QVVVVCgabPtIQALB90jPTtd8R++rpu55TVUWVvps2V1+8O12H/OTAJmPfe3qyNhZtlCQt/X6ZXrj/ZQ0btZskafwfxunud+/U7a/erNtfvVkjDt9bo089VBfc+uuYzgdIJNF8vOyTkj6TNMTMlpnZOdHaF5q66KYLVFNVo1P2PF03X3ibfn/zheo7pFDfTpmp4wefVD/u0dv/TyXrS/XbY/+g4wefpOMHn6S7rrinfv3T9z2nk/Y4TaeP/IWK1xTr2gev9mI6AKKEXu2dv/3jJlVWVWlY3710/lkX6Za7btKQoUP0+SdTNKDrLvXjPv94ivp1GqwzTvyFli9drn6dBmv8CWfUr//ThZerX6fBeuHZl3T3bf9Sv06D9dz/mj69BUD7RJ/21rnX/1I1VTU6d+QFuuvie/WrG36p3oN7ac4X3+mM3c+uH/fd9Hm65JgrdMaws/W3c27XXocM1+l/OkWSlJGdoY5dOtT/SU1PVVpGmnI68LQsIFpsy6dqeGng7gPc7S/f7HUZnshJzdn6IJ/KSs70ugRP7ZY/3OsSPFNdV+F1CZ7pmtlzunNuhNd1bK9+wwrdXyde5XUZnjiq99Fel+CZ6UXTvC7BUwd0a/rtbaKoqCvd+iCf6p09oF326b32Ge4mf/qu12V44t3lb3tdgmdKahL336okndjvpK0P8qn1NYl7X57+ubu02Ke5dAUAAAAAAPgGQQcAAAAAAPANgg4AAAAAAOAbBB0AAAAAAMA3CDoAAAAAAIBvEHQAAAAAAADfIOgAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQAQAAAAAAfIOgAwAAAAAA+AZBBwAAAAAA8A2CDgAAAAAA4BsEHQAAAAAAwDcIOgAAAAAAgG8QdAAAAAAAAN8g6AAAAAAAAL5B0AEAAAAAAHyDoAMAAAAAAPgGQQcAAAAAAPCNZK8LaCgrJUsHdBvldRmeqHN1XpfgmQlzn/K6BE8VZPbwugTPBF3Q6xKwnXJT83Rk76O8LsMT6cmZXpfgmYUbl3hdgqcKMud7XYJn0gLpXpeA7WRKUmqCfm4HFhzsdQmeeWDmw16X4Kml5Yu8LsEzTs7rEuISZ3QAAAAAAADfIOgAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQAQAAAAAAfIOgAwAAAAAA+AZBBwAAAAAA8A2CDgAAAAAA4BsEHQAAAAAAwDcIOgAAAAAAgG8QdAAAAAAAAN8g6AAAAAAAAL5B0AEAAAAAAHyDoAMAAAAAAPgGQQcAAAAAAPANgg4AAAAAAOAbWw06LOwMM/tL5HUfMxsZ/dIAANuKXg0A8Y0+DQCxsy1ndPxb0gGSTou8LpV0b9QqAgC0Bb0aAOIbfRoAYiR5G8bs55zb28y+lCTn3HozS41yXQCA7UOvBoD4Rp8GgBjZljM6as0sIMlJkpl1kRSKalUAgO1FrwaA+EafBoAY2ZYzOv4p6QVJXc3sJknjJF0d1aqAHVCxsULPXf+8vv/8e2V1yNJRvz1Cw48e3uL4uto63T3+X6quqNaVb1zRZP30V2fo2b8+pxOv/qlG/nTfaJYO7Ah6NdqV8o3l+t9fn9J3n85VVscsnfC74zTi2H1aHF9XW6dbxt2m6vJq3fDudfXLn7zuac2fNl9rlxTp9OvHa/+x+8WifKAt6NNoVyo2VuilG1/SD1N+UGaHTI2+YLT2OGqPFsfX1dbpvp/dp5qKGl3y6iX1y0PBkCY/MFlfvvKlqiuqld8rX2fdd5YycjJiMQ0kqK0GHc65CWY2XdLhkkzST5xzc6JeGdBGL936spJTArr6nSu1cu5KPfL7x1QwuEDdBnRrdvyHj3+krI5Zqq6obrKuoqRS7z/8vroN6BrtsoEdQq9Ge/PMTc8pkBLQze/foGXfLdf9v31APYf0UMHAgmbHv/fIJGV3zFZ1eeNe3XNID+195F566a5XYlE20Gb0abQ3r93+mgIpAV365qVaNW+VJvxhgroP6q6uLRwXf/J/nyirQ5ZqKmoaLZ/8wGQt/Wapzn3oXOV1z9OaBWuUnLot37cDbbctT13pI6lC0iuSXpZUHlm2tff1NrPJZjbbzGaZ2e93vFygdTWVNZr53iyNOX+M0jLT1Hevvhp6yK6a8dqXzY4vXl6sL1//Sof+8pBm1791z1saNX6UMjtkRbNsYIe1pVfTp+GV6opqff3uNzruwmOUlpmmAXv31+4/Hqapr05rdnzRsnX64rVpGnPO6CbrDh5/kIbsP1gpaRw0I75xTI32pKayRnMmzdFhvz5MaZlpKhxeqCEHD9HXb3zd7Pj1y9frmze/0UFnHdRoeWVJpT5/6nOdcNUJ6lDQQWambgO6KSUtJRbTQALblqOC1xS+ltAkpUvqJ2mupN228r46SZc452aYWY6k6Wb2jnNu9o4UDLRm7eIiJQWS1KWwc/2ygkEFWjBjYbPjX77tFR154RFKbqbZLp25VMtmL9fYK07QN+9+G7WagZ2kLb2aPg1PrFm8VknJSerad/O3gj0H99D8aT80O/65Wybq+N8dp9R0DozRrnFMjXZj3ZJ1SgokqXODY+rug7pr0YxFzY5//Y7XNfr80UreInRePX+1kgJJmvXeLH3+5OdKy0rTfuP3034nc5khomurZ3Q453Z3zu0R+d9BkkZK+mwb3rfSOTcj8nOppDmSeu5owUBraiqrlZad1mhZenZ6s5elzJw0S6GQ07DDmh5fhIIhvXjLyxp7+fFKStqWe/YC3mpLr6ZPwyvVFdVKz0pvtCw9O0NVFVVNxn793jcKBUPa8/CWrwsH2gOOqdGe1FTUKC2r8TF1WnZak8tSJGnO5DkKhULa9dBdm6wrWVOiqrIqrVuyThe/eLFOueUUvf/f9/XDlOaDbWBn2e7/gos02u2K4Mysr6S9JE1pZt15ZjbNzKatKyre3nKARlIz0lRd1jjUqCqvUlpm40ZdU1mjN/75pk649Lhmt/PZs5+r+6Du6rP7Vs8oBeLS9vZq+jRiKS0zTVXljUONqvIqpWc2Dj+qK6r10j9e1rgrTopleUBM7Mxj6oZ9uqioaKfViMSVmpna5J5I1eXVSs1s/ETkmsoavX3P2zrmkmOa3c6mS1R+fO6PlZKeou6DumvYmGGa98m86BQORGz10hUz+2ODl0mS9pa0Ylt3YGbZkiZKutg5V7LleufcA5IekKQ9997Dbet2geZ0KeysUDCkoiVF6twnfKrdyu9XqVv/xjdNKlpSpPUr1uv+cx+QJAVrg6oqq9KNR9ysCx49Xz9M/UELZizU3E/mSpIqN1ZqxXcrtHLeSo29/ITYTgrYBjvSq+nTiLWuhV0UqgtpzeK16lrYRZK0fO5ydR/YvdG4tUvWat2KYt111j8lhXt1ZVmlrjz0Gl3yxMXq1LNTzGsH2iqax9QN+/Te++xNn8YO69Snk0LBkNYtWadOfcK9dvW81eq6xTH1uqXrtGHFBj183sOSpGBd+Jj69qNu17kPn6tugyIPA7DN7zEzAdG2LffoyGnwc53C1xdO3JaNm1lKZOwE59zz218esH1SM1K122FD9c797+qka07UirkrNfv92brgkd80GtdtQDf9+fXL618v/maxXrr1Ff1uwm+V1TFLJ183TnXVdfXr/+/SCdr98GHad+yImM0F2E5t6tX0aXghLTNNe47eQ6/d+7pOv3a8ls9drm/fn6k/Pt74HosFAwt0w9vX1r9e8NVCPfu3ibr86T8pu2O2pPDjDF3IyTkpWBtSbXWtAikBLjtEPOKYGu1Gakaqdj10V016YJLGXjVWq+at0ncffqdzHzy30biu/bvqj69szvCWfrtUr9/+un79+K+V1TFLSYEkFQ4v1IePfKhjLjlG65ev18x3ZmrcDeNiPSUkmFaDDjMLSMpxzv1pezds4ajuIUlznHN/b2N9wHb7yRVj9dx1E3XD6JuUmZepn/55rLoN6KaFXy7UIxc9pus/vlaB5IByOm8+3sjIzZQlWf2yjJyMRocjgZSA0rLSlJ6TLiDetLVX06fhpVOuGqcJf3lSV/74GmV1yNSpV52sgoEFmj/9B913wX9055TbFEgOKLdzbv17svIylWTWaNm9v76v/iamC79aqKeuf1q/e+hCDdp3UMznBLSEY2q0R8dedqxeuuEl3XbkbcrMy9Rxlx+nrgO6avGXi/XExU/oqg+uauaYOqPRMbUkjbtxnF668SXdOuZWZXXM0qG/PlT9R/b3YkpIIOZc82e3mVmyc67OzD5zzh2w3Rs2O1DSR5K+lRSKLL7SOfd6S+/Zc+893Fsfv7a9u/KFOle39UE+NWHuU16X4KlTByXutedBF/S6BM8MzBs63Tm3w6cI7UivbmuffvPjV9tecDuWkZy4j5l+Yu4TXpfgqVEFI70uwTNpgcQN+Ifl7+15n468f7t69d777O0+nvJh2wtux0pqN3hdgmcemPmw1yV46qcDmr9HSCJwStyr1fbsNLLFPt3aGR1TFb528Csze1nSs5LKN63c2mlzzrmP1ehqLABAFLS5V9OnASAmOKYGgBjblnt0pEtaJ+kwbX72t5PE9YEAED/o1QAQ3+jTABAjrQUdXSN3h56pzc14k8Q9PwYA4gu9GgDiG30aAGKstaAjIClbzZ8qR1MGgPhArwaA+EafBoAYay3oWOmcuz5mlQAA2oJeDQDxjT4NADHW2kPmuekRAMQ/ejUAxDf6NADEWGtBx+ExqwIA0Fb0agCIb/RpAIixFoMO51xxLAsBAGw/ejUAxDf6NADEXmtndAAAAAAAALQrBB0AAAAAAMA3CDoAAAAAAIBvEHQAAAAAAADfIOgAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQAQAAAAAAfIOgAwAAAAAA+AZBBwAAAAAA8A2CDgAAAAAA4BsEHQAAAAAAwDeSvS6goYAFlJ2S63UZnkiygNcleOaMIad5XYKnZq+f5XUJnnn6u7e8LgHbKWAB5aZ08LoMTwSS4upXZkydPPAkr0vw1JTVn3ldgmcmzvvA6xKwncykJEvM7zIT9feTJJ037GyvS/DUzOJvvC7BM/+a/rzXJcSlxOyCAAAAAADAlwg6AAAAAACAbxB0AAAAAAAA3yDoAAAAAAAAvkHQAQAAAAAAfIOgAwAAAAAA+AZBBwAAAAAA8A2CDgAAAAAA4BsEHQAAAAAAwDcIOgAAAAAAgG8QdAAAAAAAAN8g6AAAAAAAAL5B0AEAAAAAAHyDoAMAAAAAAPgGQQcAAAAAAPANgg4AAAAAAOAbBB0AAAAAAMA3CDoAAAAAAIBvEHQAAAAAAADfIOgAAAAAAAC+QdABAAAAAAB8g6ADAAAAAAD4BkEHAAAAAADwDYIOAAAAAADgGwQdAAAAAADANwg6AAAAAACAbyR7XQCAnadkfan+fundmv7hl8rLz9XZl/9Ch/30x03GPXP/RL3z7Htas3ytcvNzdfzPj9Epvzmpfv2saXN0/7UPaMn8Zereu5suuul8DRu5WwxnAgD+VbqhTP+6/D599dE3yu2YozMvO12HjD2wybiXHnpVrz32pkrWlyo9M10HHneAfvnnMxVIDmhD0UY9eP0jmjlljqorq9RncB+dfdXPNWSvQR7MCAD8pWR9qf7+p7s1/cMZys3P1TlXnNX8MfV9E/XOc+9p9bI1ysvP1fE/P1annN/wmHq27vvrf7Vk/lJ1791Nv7v5Ao6pY4SgA/CRe66+TykpKXrmyyf0w6wFuvqs69R/aD/1HVLYeKBzuuyuP6r/rv20YvFK/fln16hLQWcdOvYQlawv1V9+eb1+/7cL9aOjD9Dklz7UX86+Xo99/JByOmR7MzEA8JH//OVBJack67Ev/quFsxfphnP+pn67FqrP4N6Nxo0cPUKHn3yosnOzVLqhTLdecKdeffQNjT33OFVVVGnQHgN19tW/UF6nPL379CTdcM4t+u9H9yojK92jmQGAP/zr6n8rOTVZz3w1QT/MWqCrfnFts8fUbotj6itOv1pdemw+pr7mrOv1+1su1IFHj9LkFz/QNb+8To9/8pByOuR4NLPEEbVLV8ws3cymmtnXZjbLzK6L1r4ASJUVVfr4jU/1i0vPUEZWhoaN3E0HjNlP7z0/ucnYU84fp0G7D1QgOaDeA3rpgCP21+xpcyRJs6fPUX7Xjjr4uAMVCAQ0+sRDlZefp4/f/DTWU0KU0aeB2KuqqNJnb07Rz/54qjKy0jV031008vARmvzCh03GFhR2V3ZulqTwwbQlJWnl4lWSpO59umnsuccpv2tHBQJJOvL00aqrrdPyBStiOh9EH70aiK3Kiip9/PqnOuvSMxsdU787cVKTsade0PiYetQR+2vWF7MlbT6mPuS4g8LH1CcdFj6mfoNj6liI5j06qiUd5pzbU9JwSUeZ2f5R3B+Q0JYvWK5AIKBe/XvWL+u/az8tmre41fc55zRz6iwVDu7TaNmWYxZ91/p20C7Rp4EYW7FwpZICAfXs36N+Wd9dC7V03tJmx3/w0scav/svdObe52jRnEU68rTRzY5bMHuR6mrqVFDYPSp1w1P0aiCGmjumHjC0nxbPW9Lq+5xz+nbqLBU2OOuj6TG1tGgux9SxELWgw4WVRV6mRP64Vt4CYAdUllcqMyej0bKs3ExVllW2+r7/+/v/5EIhHXHKGEnS0H120brVxZr84geqq63T28++p5WLV6m6qjpqtcMb9Gkg9irLq5SZvUWvzslUZXlVs+MPGXugnvr2Md036W4ddfoYdejSocmYitIK3fXHf2n878cpKzczKnXDO/RqILaaPabOyVLFVo6pH79zglzI6cgtjqknvfh+5Jj6Xa1cvFLVlRxTx0JUn7piZgEz+0rSGknvOOemNDPmPDObZmbTiorWRbMcwNcysjJUUdq4AVeUVihjiwPqhl569BW9M3GSbnjsWqWmpUiScjvm6roHr9bE/774/+3debRdVZ0n8O/OQAImYUqAIMhQgIqIDJFiUMQZh9bScjm3ZS2Vti3Htrq6qizbaqd2LLUtS0VxbgEFtbHtFlqBAgeGoCUyiCIgEoYkjIHMebv/OC/JG/NuQt69L+d+Pmtl5d1z9jnnt+9+9/fO+71z9snLjn51Fl98VY5+0lGZv8+ekxo/vSFPQ3ft/IjZo06WVz64asJ5NfY9aGH2P2z/fO7dXxy2fM3qtXn/6z+cw446NC9504u2e7xMDRPl6qF5etmy5b0JElpirHPqhx5cOapIPdT3vvz9/OjcC/P+kefUZ7w7537he3npUa/KlRdflaOffFTmL5w/qfHTmNRCR611Q631qCT7JTmulHLEGG1Or7UuqrUumj/fL1KwrR558COzYcOGLLl5yaZlN11/cw487IAx2//wrAty9mfOyUfO/EAWjEi4R57w+PzzDz6Rc685K//lU+/MH39/Wx5z1GGTGj+9IU9Dd+170MIMbNiQ22++Y9Oym6//Q/YfMRHpWAbWb8idf7hr0+t1a9blg6d9NHsu3DNv+uBpkxIvU8NEuXponl6wwC9R8HBsPKe+7aYh59TX3TzsNu+hmnPqb+cjZ30gC/Yd/vl7wgmPz2d+8Ml859qz87ef+uv88cbb8mjn1F0xqYWOjWqt9yW5KMmp3Tge9KOdd5mdk049IV/92P/MqpWrc+2V1+VnF1yep7/4qaPa/vi7F+XLH/laPvTN9495P/eN1/w+69etz0MrVub095+RBfvOz6JTju1GN+gReRq6Y/Yus3P8s/803/zE2Vm9cnWuX/ybXPGjK/PUF508qu0FZ/049y2/P0ly6+9uyzmf/V6OPKn5/Xb9uvX58Js+nlmzZ+btH/urTJvWlVM6ekyuhsm38y6z86TnnJivfvwbWbVyda658rr87ILL8ow/f9qotj/+zkX50oe/mg+d+f4sPGDhqPVDz6k//77mnPqJzqm7YtIeL1tKWZBkXa31vlLKzkmemeTDk3U8IHnLB96Uj//1p/LSo16VebvPy1s/8KYc+OgD8uvLr8m7XvOPOe+Gc5IkX/noN/LAvSvy5ue/Y9O2T3/xKXnbf39zkuaZ4FdctDhJ8sSnHJv3fOFd3e8Mk06eht544/ten0//zb/kNYvekLm7z8kb3/eGPOqw/XPtFdfnvX/5wZx97deTJNdfdUO+8bGzsnrl6uy6x7yc+Nzj86p3vixJ8purfpsrL/xFdpq9U175hNdu2vd//fLf53HHPbYX3WKSyNXQfc059Sfz0ie8MnN3n5e3ffCvNp1T//2/f0++/9tzkyRf+ejXm3Pq5w09p35q3v6h5pz67M+ekysuHDynPuXY/OMX/6H7nelTZeRMsNttx6UcmeSrSaanuXLkW7XW925pm2OOPbpectnFkxLPVDetTO91CD1z75r+vpf0unuv7XUIPXP2b87vdQg9c8YzP31VrXVRL2PY1jx96WX/2o3wppzp0ybtbwNT3v1r7+l1CD11+V0/73UIPXPub/vz854kX3v2Z3qep5Otz9XHLjqm/vTyn3QrvClloA70OoSeeWDdfb0OoaeuuefqXofQM5++6ju9DqFnznvhGePm6Uk7a6u1Xp3k6MnaPwAPjzwNMPXJ1QBbzw2dAAAAQGsodAAAAACtodABAAAAtIZCBwAAANAaCh0AAABAayh0AAAAAK2h0AEAAAC0hkIHAAAA0BoKHQAAAEBrKHQAAAAAraHQAQAAALSGQgcAAADQGgodAAAAQGsodAAAAACtodABAAAAtIZCBwAAANAaCh0AAABAayh0AAAAAK2h0AEAAAC0hkIHAAAA0BoKHQAAAEBrKHQAAAAAraHQAQAAALSGQgcAAADQGgodAAAAQGsodAAAAACtMaPXAQxXUkp/1l6m9Wm/k2S3nfbodQg99cS9ju91CD3zrPe8s9chsNX6N0/3s7kzd+t1CD118r5P63UIPfPCd7+r1yFAx/r5fHruzF17HUJPHbfghF6H0DPnnfnWXocwJfVvNgAAAABaR6EDAAAAaA2FDgAAAKA1FDoAAACA1lDoAAAAAFpDoQMAAABoDYUOAAAAoDUUOgAAAIDWUOgAAAAAWkOhAwAAAGgNhQ4AAACgNRQ6AAAAgNZQ6AAAAABaQ6EDAAAAaA2FDgAAAKA1FDoAAACA1lDoAAAAAFpDoQMAAABoDYUOAAAAoDUUOgAAAIDWUOgAAAAAWkOhAwAAAGgNhQ4AAACgNRQ6AAAAgNZQ6AAAAABaQ6EDAAAAaI0ZvQ4AYLtYsyG59M7k9oeSWdOTRQuSP5k3drvLlya3PdS8fsxuyTHzN6+/anly64rkvrXJE/Ycvg6Ah2fNhuSyu5LbVyazpydH7ZkcNEauXrshWbwsWbKyeX3Yrk1OTpLV65MrlyVLVyXra7LbTsmxC5L5s7vXD4C2WrMhuWJZcufK5pz6yD2SA+eObrd2Q/KL5ckdg3n6kF2Tx+8xut3SVcmFtyeH75Ycuefkxj7EpBc6SinTkyxOsqTW+vzJPh7Qp352VzK9JK84JLl7dfL/liR7zEp2nzW83RVLk/UDyUsPTlZtSH74x2TOzOYkOknmzWyKJDfc3/0+9Ig8DXTNFUuTaSV5ycHJvWuSi25v8vRuI3L14mVNEeNFByarNyQ/WpLMmZH8ya7JuprsObspbsyenvz+geSiJcmfHZTMbO/FynI10BVXLW/u+/izA5P71iSX3Nnk6V13Gt7ul3cnG2ry7w5o8vRFtyePmJEcPKR4PVCbYsieI3J8F3Tjp8HbklzfheMA/WrdQPKHFc3VFzOnJfvskjxqTnLjA6Pb3vpQU22eMS2ZO1jg+N2Qosahuyb7z0lmlu7F33vyNDD51g8kf3ywuTJj5rRkr52T/R6R3LxidNslDyWH797k6jkzk0Pmbc7pc2c263aZ0RRNDt01GUjywNqudqcH5Gpgcq0fSG57sBCLLHoAAA+vSURBVDlXnjktWbBzsu8uyS3j5OnH7LY5Tx88L7lpRLvf3Necl8/dafT2k2xSCx2llP2SPC/JFyfzOECfe2BtUsrwSvMes5oq9ERqmr8q9il5Guiajbl63pBcvfus5lbBidQk94/T7p41zV8V587cLmFORXI10BUr1o2dp8fLvyMNbffQuqaQ/bjdt2+MHZrsKzo+meRv0tTZASbHuoFkpxHpbKdpzfKR9tslufqeZt0Da5urOdbX7sQ5NcnTQHesr6NvLZk5Tq5e+Ijk2sFcvWJtc3vKWLl67YbkZ3c295DvNH1y4p4a5Gpg8q0b2Io8vUty/X2DeXpdctMDyYYh7X6xfPOVIT0waUctpTw/ydJa61UTtDutlLK4lLJ4+fLlkxUO0GYzpyVrRyTgtWMk6iQ5fu9mLo9zbmru+T54bnM/YR+Sp4GumlFGnyyPdVKdJE9ckEyflvyvW5KL70gOnNPcqjLU+oFm3fzZyRFjTIDXEp3k6qF5etkyeRrYRmMVNcbL08fMb86pf3BrcukdyQFzkp0H8/SSh5rtHjVn8mMex2Se3Z+U5AWllOcmmZ1kXinlG7XWVw9tVGs9PcnpSXLMscf09Z9VgW00b6ek1uZyuY23r9yzZvTkdkkze/Qp+25+vXhZP8/UL08D3bMxVz+wdvNl0feuaZ6aMtKs6cmT9tn8+pcjJrPbMJD86x1N8eNP95rcuHtvwlw9NE8fu0ieBrbR3JlNnl6xdvO8GvetGT0RadLk6RP23vz6V3c3E0UnyV2rmnPx793SvF43kJQ05+pPXjiZPdhk0q7oqLX+Xa11v1rrgUlenuTCkSfPANvFzGnJAXObS+TWDSR3rUxufbCZvG6kB9Y2M0MP1GZSvBvubx5vuNFAbf5KWNMk+vUDzbIWkqeBrpoxrZns+Vd3N7l16armUd8HjfHYwhVrm0ccDtTmL4M33r/5sYUDtXkKwPSSnLh3cz95i8nVQNfMmJbsNyf59b1Nnl62qnnM91iPl12xbnOevv2h5hbDjfNxPH6P5HmPSp69X/Nv312ayUqP615huj+v1wba58S9k0vvTM68sakwn7h3M3nSnSuTC25LXnNY02756uTypc2tLbvulDxl4fBH0P7kzuFPa/nVPcmT92lm9Qfg4Tlur+TndyXfvqnJ1cctaK6+W7oquXBJ8vJDmnb3rGmuuFs70Dz2+6R9Nl+lt2xVU/yYXpJv/X7zvp/2yOZJLgBsu2PnN48C/+4tTZ4+dn5zzrx0VXLJHc3jwZPmiryNf2ScO7O5umPjlR8zpw2/3WX6tOb2xVndm0upK4WOWuvFSS7uxrGAPjVrevKMR45evs8um4scSVNNPniMKz02Onlh86/PyNNAV4y8fXCjvXbeXORImqv0DhjjL4hJsvcuyasPnZz4pji5Gph0s6aPfXvJXjtvLnIkzfwbnc7BcXz3bzHszRSoAAAAAJNAoQMAAABoDYUOAAAAoDUUOgAAAIDWUOgAAAAAWkOhAwAAAGgNhQ4AAACgNRQ6AAAAgNZQ6AAAAABaQ6EDAAAAaA2FDgAAAKA1FDoAAACA1lDoAAAAAFpDoQMAAABoDYUOAAAAoDUUOgAAAIDWUOgAAAAAWkOhAwAAAGgNhQ4AAACgNRQ6AAAAgNZQ6AAAAABaQ6EDAAAAaA2FDgAAAKA1FDoAAACA1lDoAAAAAFpDoQMAAABojVJr7XUMm5RSliX5Q48OPz/J8h4du9f6ue9Jf/df33vngFrrgh4ef5v0OE8nvR+3XtL3/qTvvSNPb5tej1sv6Xv/6uf+97Lv4+bpKVXo6KVSyuJa66Jex9EL/dz3pL/7r+/92fcdWT+Pm77re7/p577vyPp53PS9P/ue9Hf/p2rf3boCAAAAtIZCBwAAANAaCh2bnd7rAHqon/ue9Hf/9Z0dTT+Pm773J31nR9PP46bv/auf+z8l+26ODnYIpZQNSX6dZEaS65P8Ra115Tbu6ytJ/net9ZxSyheT/FOt9bpx2p6SZG2t9WdbeYxbkiyqtfbrpERAn5GnAaY+uZp+4YoOdhSraq1H1VqPSLI2yRuHriylzNiWndZaXz9eQh50SpITt2XfAH1GngaY+uRq+oJCBzuiS5McUko5pZRyaSnlvCTXlVKml1I+Wkq5spRydSnlPyRJafxzKeWGUsqPkuy1cUellItLKYsGvz61lPKLUsqvSik/LqUcmCb5v6OU8m+llCeXUhaUUs4dPMaVpZSTBrfds5RyQSnl2sGKdunuWwIwpcjTAFOfXE1rbVPFDnplsMr8nCQ/HFx0TJIjaq03l1JOS3J/rfWJpZRZSX5aSrkgydFJHp3k8CR7J7kuyZdG7HdBki8kOXlwX3vUWu8ppXwuyYO11o8Ntvtmkk/UWn9SSnlUkvOTPDbJe5L8pNb63lLK85K8blLfCIApSp4GmPrkatpOoYMdxc6llH8b/PrSJGekufztilrrzYPLn5XkyFLKSwZf75rk0CQnJzmz1rohye2llAvH2P/xSS7ZuK9a6z3jxPGMJIeXsqm4PK+UMmfwGC8e3PYHpZR7t7GfADsqeRpg6pOr6QsKHewoVtVajxq6YDAxPjR0UZK31FrPH9HuudsxjmlJjq+1rh4jFoB+Jk8DTH1yNX3BHB20yflJ/mMpZWaSlFIOK6U8IsklSV42eL/hwiRPHWPby5KcXEo5aHDbPQaXr0gyd0i7C5K8ZeOLUsrGHxSXJHnl4LLnJNl9u/UKoD3kaYCpT65mh6fQQZt8Mc29gr8opVyT5PNprlr6bpLfDa77WpKfj9yw1rosyWlJvlNK+VWSswdXfT/JizZOnJTkrUkWDU7MdF02z1T939Ik9WvTXG536yT1EWBHJk8DTH1yNTu8UmvtdQwAAAAA24UrOgAAAIDWUOgAAAAAWkOhAwAAAGgNhQ4AAACgNRQ6AAAAgNZQ6AAAAABaQ6EDAAAAaA2FDgAAAKA1FDoAAACA1lDoAAAAAFpDoQMAAABoDYUOAAAAoDUUOgAAAIDWUOgAAAAAWkOhAwAAAGgNhQ4AAACgNRQ6AAAAgNZQ6AAAAABaQ6EDAAAAaA2FDgAAAKA1FDoAAACA1lDoAAAAAFpjRq8DAEiSZz37mfXuu+9OHbli1IIJV4yzdsvtt9e2w7batNm2bd9s2dm247Yb5w3tdK9b2MU27LNu4dU4bYf/txX7Gmf9NsU+QZu6df2fMNYxd7TlY9SRX9UJ1ne4/Ta9L5u2nWjrcWLpIAmMf8yxW4z7vo3zjbGlyLd2m9HNOxzLDttvaeWo93iiNDH2N98WYpswso7Gs4NDb9pu4nZDt9jyN/L4oU0wtqPe1g6C6jCXTfzZ6+jD2Vmbid7MCbcfp+HDiW2in5tjLV6x7vxa66kdHBXoIwodwJRw991356eX/2TUCWPddCI28mS/Dls/avkE+9k++8qw9VsbeyfH3Lx+nNcdLh/dx9Exd7zvbYxh8y8dWxHD4OuBDHQUy8Cm11tuPzBq++Htx97nlrcZvc/xXg901H54my0fa3SMA2MuH68vAxv3P7LdmLFseWzGP8bw923kMcY79sbjjrWv0W0naLexLyO367DdNu1znJi2vt3w9YMvhi2rI9qM/n+i9Vu3n2HxbPdYtm79sHWjvleb9aO+57bT+zIwLIaJYunsmAMTtNv0jbe5c8OXD1s3MrgO14/c57jrx1ne0bYPM9Yk+dGS+QEYwa0rAAAAQGsodAAAAACtodABAAAAtIZCBwAAANAaCh0AAABAayh0AAAAAK2h0AEAAAC0hkIHAAAA0Bql1trrGABSSrkmyepex8GE5idZ3usg2CJjtGMwTlOfMdoxzK61HtHrIICpZUavAwAYtLrWuqjXQbBlpZTFxmlqM0Y7BuM09RmjHUMpZXGvYwCmHreuAAAAAK2h0AEAAAC0hkIHMFWc3usA6IhxmvqM0Y7BOE19xmjHYJyAUUxGCgAAALSGKzoAAACA1lDoALqqlHJqKeWGUsqNpZS/HWP9rFLK2YPrLy+lHNj9KPtbB2P0n0op15VSri6l/LiUckAv4ux3E43TkHZ/XkqppRRPj+iyTsaolPLSwc/TtaWUb3Y7RjrKeY8qpVxUSvnlYN57bi/i7GellC+VUpYOPop+rPWllPI/Bsfw6lLKMd2OEZhaFDqArimlTE/ymSTPSXJ4kleUUg4f0ex1Se6ttR6S5BNJPtzdKPtbh2P0yySLaq1HJjknyUe6GyUdjlNKKXOTvC3J5d2NkE7GqJRyaJK/S3JSrfVxSd7e9UD7XIefpX9I8q1a69FJXp7kX7obJUm+kuTULax/TpJDB/+dluSzXYgJmMIUOoBuOi7JjbXWm2qta5OcleSFI9q8MMlXB78+J8nTSymlizH2uwnHqNZ6Ua115eDLy5Ls1+UY6eyzlCTvS1MsXN3N4EjS2Ri9Iclnaq33JkmtdWmXY6SzcapJ5g1+vWuS27sYH0lqrZckuWcLTV6Y5Gu1cVmS3UopC7sTHTAVKXQA3fTIJH8c8vq2wWVjtqm1rk9yf5I9uxIdSWdjNNTrkvzfSY2IsUw4ToOXbu9fa/1BNwNjk04+S4clOayU8tNSymWllC39xZrJ0ck4/WOSV5dSbkvyf5K8pTuhsRW29mcX0HIzeh0AADumUsqrkyxK8pRex8JwpZRpSf4pyWt7HApbNiPNpfanpLky6pJSyuNrrff1NCpGekWSr9RaP15KOSHJ10spR9RaB3odGABjc0UH0E1Lkuw/5PV+g8vGbFNKmZHmMuG7uxIdSWdjlFLKM5K8K8kLaq1ruhQbm000TnOTHJHk4lLKLUmOT3KeCUm7qpPP0m1Jzqu1rqu13pzkt2kKH3RPJ+P0uiTfSpJa68+TzE4yvyvR0amOfnYB/UOhA+imK5McWko5qJSyU5pJ3c4b0ea8JH8x+PVLklxYa61djLHfTThGpZSjk3w+TZHDnAK9scVxqrXeX2udX2s9sNZ6YJq5VF5Qa13cm3D7Uif57ntpruZIKWV+mltZbupmkHQ0TrcmeXqSlFIem6bQsayrUTKR85K8ZvDpK8cnub/WekevgwJ6x60rQNfUWteXUt6c5Pwk05N8qdZ6bSnlvUkW11rPS3JGmsuCb0wz8djLexdx/+lwjD6aZE6Sbw/OE3trrfUFPQu6D3U4TvRQh2N0fpJnlVKuS7IhyX+utbqCrYs6HKd3JvlCKeUdaSYmfa0CfHeVUs5MUxScPzhXynuSzEySWuvn0syd8twkNyZZmeQvexMpMFUUeRoAAABoC7euAAAAAK2h0AEAAAC0hkIHAAAA0BoKHQAAAEBrKHQAAAAAraHQAQAAALSGQgcAAADQGgodAAAAQGv8fzmz13d/2pHZAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1080x792 with 7 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hGJYI2mGPDtZ"
      },
      "source": [
        "class BertEnsemble():\n",
        "\n",
        "  def __init__(self, predictors):\n",
        "    self.predictors = predictors\n",
        "\n",
        "  def predict(self, X):\n",
        "    predictions = self.predict_proba(X)\n",
        "    return np.argmax(predictions, axis=1)\n",
        "\n",
        "  def predict_proba(self, X):\n",
        "    predictions_all = [self.predictors[clf][0].predict(X, return_proba=True) * self.predictors[clf][1] for clf in self.predictors]\n",
        "    return np.sum(predictions_all, axis=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PNNc70B4Phsv",
        "outputId": "6efa7cc7-a3b2-4e27-f94d-8ac568f27d11",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "2f5a9d126b9a492bbae66ec02fc45f57",
            "abb0ecbd4b854ecba42e82141daa89e3",
            "cb9484d6b97041c4ac344c5bcf6d931c",
            "60549ba240af4e578298bfc7b401f9af",
            "f55467092ecc4a78ad7dfdc2dd807109",
            "98122e237e6e42b08ea4d95be5a354a3",
            "2a108e2173944e67bc9a08e133e6e74d",
            "168a9f4eb24349deb369a99fd0bde35b",
            "71e8447401164da0beb3c1a69aa70622",
            "f8feb2a56dcb476da9c4cf346df908b6",
            "46441cfd20af428d8d3c9066e4100956",
            "c0055b1829b44641b2dc4522c03bf4a5",
            "cac0e858532b4b47bfd4ddf048915a08",
            "2e8e6d96ab074f898e18641f6d9e932c",
            "05ee0270614b400f9bd9bb29cdab057f",
            "235510309d4b42248d5d66cf773daa26"
          ]
        }
      },
      "source": [
        "ensemble = BertEnsemble({\"base_line\": [base_line_predictor, 1.3], \"score_balanced\": [score_balanced_predictor, 1], \"only_not_helpful\": [not_helpful_predictor, 0.7]})\n",
        "predictions = ensemble.predict(test.TextClean.values)\n",
        "true_labels = test.Score.values - 1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2f5a9d126b9a492bbae66ec02fc45f57",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "71e8447401164da0beb3c1a69aa70622",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ANFhSunMLMJK"
      },
      "source": [
        "comparison = list(map(lambda x: x[0] == x[1], zip(predictions, true_labels)))\n",
        "score = comparison.count(True) / len(true_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "96LCu5S-sAc8",
        "outputId": "9141acd0-6742-4ffb-c378-861dc61a5b98",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "score"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7658106197237957"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZUxt9BqsKzZv"
      },
      "source": [
        "np.save(\"/content/drive/My Drive/Colab Notebooks/predictions2.npy\", predictions)\n",
        "np.save(\"/content/drive/My Drive/Colab Notebooks/true_labels2.npy\", true_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xLk5Y4aZQBFe",
        "outputId": "9f3f09f8-d85a-425d-e911-de1ae167e868",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        }
      },
      "source": [
        "confusion_matrix_custom(true_labels, predictions, \"/content/drive/My Drive/Colab Notebooks/confusion_all.pdf\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAE9CAYAAAD9HVKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xVdb3/8ddn7hdmABlAZIabYooXEBFNj/dLeEmz9HegtJtHstTMNNOTaVmdTtnFeqQVeYsstbwUGoVpmtZJBbwl4IW4IwrIxQFmGGbm8/tjb3Bmz2Zm9rjXXvPd837OYz/ca63v/q7PEvjs73zWd61l7o6IiISpIO4ARESk55TERUQCpiQuIhIwJXERkYApiYuIBExJXEQkYH0uiZvZFDN71cwWm9nVcceTLWZ2u5mtNbOX444lm8yszsweN7OFZrbAzC6LO6ZsMLMyM3vWzF5MHtfX444p28ys0MyeN7OH444ln/WpJG5mhcDNwKnAOGCamY2LN6qsuROYEncQEWgGrnD3ccARwMV58me2HTjB3ccDE4ApZnZEzDFl22XAoriDyHd9KokDk4HF7r7E3ZuAe4CzYo4pK9z9SWBD3HFkm7uvcffnku/rSSSF4fFG9d55wpbkYnHylTdX3plZLXA6cGvcseS7vpbEhwMr2yyvIg8SQl9hZqOAQ4Bn4o0kO5LlhheAtcBf3D0vjivpJuAqoDXuQPJdX0viEigz6wfcD3zB3d+JO55scPcWd58A1AKTzezAuGPKBjM7A1jr7vPjjqUv6GtJfDVQ12a5NrlOejEzKyaRwH/t7g/EHU+2ufsm4HHy55zGUcCZZraMRMnyBDO7K96Q8ldfS+JzgbFmNtrMSoCpwKyYY5JOmJkBtwGL3P0HcceTLWY22MwGJN+XAycDr8QbVXa4+zXuXuvuo0j8G/uru58Xc1h5q08lcXdvBi4B5pA4QfZbd18Qb1TZYWZ3A/8E3mdmq8zsgrhjypKjgPNJjOZeSL5OizuoLBgGPG5mL5EYXPzF3TUVTzJmuhWtiEi4+tRIXEQk3yiJi4gETElcRCRgSuIiIgFTEhcRyZGublRnCT9O3qDvJTOb2FWffTaJm9n0uGOIgo4rPPl6bPl6XO/RnXR+UdepwNjkazrw06467LNJnMT/oHyk4wpPvh5bvh5Xj3XjRnVnATOTN0h7GhhgZsM667MvJ3ERkd4m45v0FUUaToYKKoq9cEBZbvbVv5TivapycqXTAXuOycVuAKitG86EieNzdgVX4qr46NWNqOWQQyfk7rhyOL6pG1HHxEMn5uzYPEc3Fqyty92f2YrlK3l7/dtZ+8toNWVOU4b/n+p3LAAa26yZ4e4zshXT7vSqJF44oIxBn5kQdxhZ95er7o07hMiUFObmSzfXSgpK4w4hMo0tDXGHkHUnHHlSdjtsaoXDh2T2mUdXN7r7pPe454xv0qdyiohIOmaZvbJjFvDx5CyVI4DN7r6msw/0qpG4iEivYEQyxE3eqO44oMbMVgHXk3iqE+7+M2A2cBqwGNgGfKqrPpXERUTSieB8j7tP62K7Axdn0qeSuIhIOrk5Z/+eKYmLiHSQ1Tp3pJTERURSRVQTj4KSuIhIOhqJi4gELIwcriQuItKBAQVhZHElcRGRdMLI4UriIiJpqSYuIhKwMHK4kriISAeqiYuIBC6MHK4kLiLSka7YFBEJV0DllEAuLBURkXQ0EhcRSSeMgbiSuIhIWqqJi4gELIwcnj9JvHXbDt6Z9Trb/72Jgopi+p04kvKDOz7odONdC9ixfPOuZW9ximrKGfS5iQA0rXiH+j8voWV9A4UDSqk6fW9KRvbP2XGISC8Q0InNSJO4mU0BfgQUAre6+/9Gta93Zv8bCgsYfOXhNL+5hU2/WUjxnpUUDals127geQe0W95wx0uUjB4AJL4INt29kOoz9qF0/0E0/msdm+5eSM1lh1FQnjffdyLSHWHk8Ohmp5hZIXAzcCowDphmZuOi2Jc3tbB94dv0O34kBaWFlIzsT+n79qDhxXWdfq5lYyM7VrxD+fjEiH3HynoK+pVQdkANVmCUjx9CQUUx2xetjyJsEenN4nnafcainGI4GVjs7kvcvQm4Bzgrih01v90ABUZRTfmudUVDK2let7XTzzW8uJbiEdUUDix7d6V7x/7XbstarCISiIIMXzGJctfDgZVtllcl12WdN7VQUFrYbp2VFeHbWzr9XOOLaymfMHTXcnFdFa31TTT8ax3e0krDC2/RsqER39EaRdgi0ltlOgqPcSQee6HXzKYD0wEK+pf2rI+SQlpTErZvb8ZSEntbTcs307qlidJxNbvWFVQUM2DaOOofWUr9H/9NyT4DKBkzgILqkh7FJSIBC6QmHmUSXw3UtVmuTa5rx91nADMAiveq6ljL6IaiQeXQ6jS/3ZB4DzS/uZWiwZW7/Uzji2sp3X9QhxF8yaj+DJo+IRFbi7P+R/OoODKSXyBEpDcLZJ54lOWUucBYMxttZiXAVGBWFDuykkJK9x/ElseX400tNK14h+2vbqB8/OC07X1HC40L1rcrpey0Y80WvKWV1sZmtjyylML+JZTuMzCKsEWkNwukJh7ZSNzdm83sEmAOiSmGt7v7gqj2V3363rzzh9dZe+MzFJQXU3X63hQNqaRp+WY23bWAIV85clfbxlc2YGVFFI/uOP976z9W0fT6RgBK9hnIgP/cP6qQRaS3MoIZiUdaE3f32cDsKPex0856dqqSkf3bJXCA8oMGU35Q+lH6gHP2iyQ+EQlMGDk8/hObIiK9kq7YFBEJmMopIiKBMoIpp+ihECIiAdNIXESkA8MyLKf06CKXLFASFxFJQ0lcRCRggZzXVBIXEUmVeCZEZlm889vtRUdJXEQklWVeTomLkriISBpK4iIiwcp8dkpclMRFRNIIJIcriYuIpErcxDCMLK4kLiKSSic2RUTCZoHcPEVJXEQkDY3ERUQCFkgOVxIXEUllWMZXbMZFSVxEJI1Qyim6n7iISKrk7JRMXt3q1myKmb1qZovN7Oo020eY2eNm9ryZvWRmp3XVp5K4iEgOmFkhcDNwKjAOmGZmqU93vxb4rbsfAkwFbumqXyVxEZE0zDJ7dcNkYLG7L3H3JuAe4KyUNg5UJ9/3B97oqtNeVROv3WMQX5v68bjDyLp5656NO4TI1PWrizuESIyq2ifuECLT4s1xh5B1nuVHMvTwis0aM5vXZnmGu89oszwcWNlmeRVweEofXwMeMbNLgUrgpK522quSuIhIb9GDJL7e3Se9x91OA+509++b2fuBX5nZge7eursPKImLiHQQyV0MVwNtf3WtTa5r6wJgCoC7/9PMyoAaYO3uOlVNXEQkVTSzU+YCY81stJmVkDhxOSulzQrgRAAz2x8oA9Z11qlG4iIiaWR7IO7uzWZ2CTAHKARud/cFZnYDMM/dZwFXAL8ws8tJnOT8pLt3WvBXEhcRSRHVrWjdfTYwO2XddW3eLwSOyqRPJXERkTRCuWJTSVxEJA3dO0VEJFTdv4AndkriIiIpTA9KFhEJm57sIyISMI3ERUQCpiQuIhKwQHK4kriISCqzcEbiuneKiEjANBIXEelAUwxFRIKmJC4iErBAcriSuIhIOhqJi4gEKqTZKUriIiJpKIlL1tRvquemq27muSdfoHqPaj551Xkc/6FjOrR78NZZPHTnbDZvfIfyijKO+eB/cMF/f4LCokIAZn7vN/zzkWdYuXgVUy89l/Mun5rrQxEJRiA5XEk8BLd8dQZFxUX8Zv4dLFm4lOs/9S3GjBvFyH1HtGt3xMmHcfK5J9KvfyX1m+r51kXf5Q93PMyHLzwLgL1G7cmnr/k4f/r1nDgOQyQg4UwxjOxiHzO73czWmtnLUe2jL2jc1sg//vQ0518xjfLKcg44bByHn3QYf33giQ5th40cRr/+lQC4OwUFBaxZ9uau7SedcwKHHX8o5ZXluQpfJFgRPCg5ElFesXknMCXC/vuE1UveoLCwgNoxw3etG7P/KJa/tjJt+8d//yQfOeCjTJ3wCZYsWsapHzslV6GK5A2L5mn3kYisnOLuT5rZqKj67ysatjVSUVXRbl1ldQUNWxvStj/+Q8dw/IeOYfXSN3js/icYUDMgF2GK5J1Aqim6d0pvV15Rxrb6be3Wbatv6LIkMnz0Xozct45brv15lOGJ5K1QRuKxJ3Ezm25m88xsXv3G+rjD6XWGj9mLlpZWVi99Y9e6JYuWMXLfui4/29LcwpoVb0UZnkj+MsvsFZPYk7i7z3D3Se4+qWpgVdzh9DplFWUcOeUI7vrB3TRua2TB3EU8/ZdnOeHDx3Vo++e7/8Km9ZsAWPHaSn57ywOMP/KgXdubdzTT1NhEa6vT0txCU2MTLS0tuToUkYBkNgrPy5q4ZM/F35zOD7/0E6ZN/CTVA6u4+JufYeS+I3j52YVc94lv8MCiuwFYOP8VZn7v1zRsbaT/oGqOPu1Izr/io7v6+fHVt/DofY/vWr73J/dx+fcu5eRzT8j5MYn0agE97d7cPZqOze4GjgNqgLeA6939ts4+M/rAUf61+78SSTxxGlI+OO4QIlPXr+uyTohGVe0TdwiRaWxJf1I8ZCcd9QFemP9i1tJu5aiBvt+1x2f0mecufHC+u0/KVgzdFeXslGlR9S0iErU+f7GPiIhETzVxEZE0QhmJK4mLiKQRSA5XEhcR6SDmaYOZUBIXEUlhqJwiIhI0JXERkYApiYuIhCqgKzaVxEVE0tBIXEQkUBbQ49mUxEVE0lASFxEJWCA5XElcRKQD00hcRCRsSuIiIuHSSFxEJFAGFISRw5XERUQ6CmeKoR4KISISMCVxEZFUBgVmGb261a3ZFDN71cwWm9nVu2nz/8xsoZktMLPfdNWnyikiIimiuBWtmRUCNwMnA6uAuWY2y90XtmkzFrgGOMrdN5rZkK761UhcRCSNggxf3TAZWOzuS9y9CbgHOCulzYXAze6+EcDd13YnThERSRFBOWU4sLLN8qrkurb2BfY1s3+Y2dNmNqWrTntVOaW6pJpT6j4QdxhZ1+LNcYcQmZmv3B13CJGYOvYjcYcQmaKCXvXPPitavTWr/fWwnFJjZvPaLM9w9xkZ9lEEjAWOA2qBJ83sIHff1NkHRESkne6frGxjvbtP6mT7aqCuzXJtcl1bq4Bn3H0HsNTMXiOR1OfurlOVU0REUiXvnZLJqxvmAmPNbLSZlQBTgVkpbX5PYhSOmdWQKK8s6axTjcRFRFIY2R/hunuzmV0CzAEKgdvdfYGZ3QDMc/dZyW2nmNlCoAX4kru/3Vm/SuIiImn0oJzSJXefDcxOWXddm/cOfDH56hYlcRGRNEK57F5JXEQkReIGWEriIiLBCiOFK4mLiKTRoymGsVASFxFJYaZyiohI0HRiU0QkYKGMxHXFpohIwDQSFxFJYWh2iohI0EIppyiJi4h0oCmGIiLBMtPsFBGRoIUyEu9ydoolnGdm1yWXR5jZ5OhDExGJj2X4ikt3phjeArwfmJZcrifxxGYRkby08wZYWX7GZiS6U0453N0nmtnzAO6+MflUChGRvBVKOaU7SXyHmRUCDmBmg4HsPpVURKRX6fYj12LXnST+Y+BBYIiZfQs4B7g20qikT9i2eRsPfONBFj+9mMoBFZxyySmMnzJ+t+2bdzTzk2k3s33bdr48+6pd678y6VqKy4p3/aM76JSD+PBXz448fslfUTyeLSpdJnF3/7WZzQdOJHFsH3L3RV19zszqgJnAUBKj+Bnu/qP3GK/kkYe+8xBFxYVc88jVrHltDTMv+xV7jt2ToXsPTdv+qZl/p3JgBdu3be+w7dK7L2FQ3aCoQ5a+IqApht2ZnTIC2AY8ROLJzFuT67rSDFzh7uOAI4CLzWzcewlW8kdTQxML/rqQky46idKKUkZNGMX+x+zHC7NfSNt+w+oNvPinFznmk8fmOFLpq/LpxOYfSYykDSgDRgOvAgd09iF3XwOsSb6vN7NFwHBg4XsJWPLD+uXrKSgsoGZkza51e+47jKXPLU3b/uEb/8jJF59McWn6v7K/uPBW3J0RB4/gtMtPZeBeAyOJW/qGkB7P1uVI3N0PcveDk/8dC0wG/pnJTsxsFHAI8ExPgpT809TQRGm/0nbryvqV0rS1Y6lkweMLaW1t5YDj0/8i918zLuDKh67gC/ddRlVNFTO/8CtamlsiiVv6DjPL6BWXjGv37v4ccHh325tZP+B+4Avu/k6a7dPNbJ6ZzXt7/YZMw5FAlZSXsH1L+4S9fet2SirbJ/amhibm/HgOZ1x5+m77Gj1xNEXFRZRXlXPGlaez8Y2NrFu2LpK4pa8wCjJ8xaXLcoqZfbHNYgEwEXijO52bWTGJBP5rd38gXRt3nwHMABg/8WDvTr8SvpqRNbS2tLJ+xXpqRiRKKmtee5OhY4a0a7d+xdtsfGMjv7jwVgBadrTQuKWRb3/gf7nojs+kLZuYWXJCrEjPhXJiszs18ao275tJ1Mjv7+pDlvg/cBuwyN1/0LPwJF+VlJcw7vhxPPazxzj7q2ez5tU1LPrbIj5z+/R27YbuPYSr/vilXcsrXlrBQ999mIvv+hyVAyt5699v0dLcyp77DGXH9h08+tNHqR5czeDRg3N9SCKx6DSJJy/yqXL3K3vQ91HA+cC/zGznlIP/dvfZPehL8tCZV3+QB254kP85+dtU9K/grGvOZOjeQ1n2/DJ++fmZXP/UdRQWFVJV8+44ory6HCuwXeu2bNjCrG8/xOa1mykpL2HEwSM4/6bzKCwqjOuwJA/kxYOSzazI3ZvN7KiedOzufyech2NIDCr6V3De9z/WYf2oQ0Zx/VPXpf3MmElj2l3os/dhe3P5A1+ILEbpuyyQ9NXZSPxZEvXvF8xsFvA7YOvOjburcYuI5IN8qomXAW8DJ/DufHEHlMRFJC9ZnjzZZ0hyZsrLvJu8d9K5fxHJaxbI3VM6S+KFQD/S17WVxEUkr+XDSHyNu9+Qs0hERHqRfKiJh3EEIiJZZsmfEHSWxE/MWRQiIr1JPswTd3fdyERE+qx8KKeIiPRJiSf7hD87RUSkj8qvZ2yKiPQ5SuIiIgGL8x7hmVASFxFJYWgkLiISroCmGIZx+lVERNLSSFxEpIP8uGJTRKRPMqDAwihUKImLiKQRyonNML5qRERyzDL86VafZlPM7FUzW2xmV3fS7iNm5mY2qas+NRIXEekg+0/2ST54/mbgZGAVMNfMZrn7wpR2VcBlwDPd6VcjcRGRFEYkI/HJwGJ3X+LuTcA9wFlp2n0D+A7Q2J1OlcRFRNIoMMvo1Q3DgZVtllcl1+1iZhOBOnf/Y3fj7FXllEIrpLp4QNxhZJ0Fcpa7Jy4Y94m4Q4jE39c8FXcIkXl0+bNxh5B1b25dm90OrUf/bmvMbF6b5RnuPqPbu0zs8AfAJzPZaa9K4iIivUOP5omvd/fOTkSuBuraLNcm1+1UBRwIPJGcGbMnMMvMznT3tl8O7SiJi4ikSMwTz/oUw7nAWDMbTSJ5TwU+unOju28GanbFYPYEcGVnCRyUxEVE0sr2PHF3bzazS4A5QCFwu7svMLMbgHnuPqsn/SqJi4ikEcWtaN19NjA7Zd11u2l7XHf6VBIXEUmhW9GKiATNgplVpiQuIpKGnuwjIhIoM5VTRESCFsr9xMMo+oiISFoaiYuIdGAqp4iIhEwnNkVEApWYJx5GtVlJXESkAz0oWUQkaKqJi4gETCNxEZGAaSQuIhIoQ7NTRETCZZonLiISNAvkgnYlcRGRNEIZiYfxVSMSkPpNW/juRT/gYwd+iouO/jxPzfpH2nYP3T6bzx13GeePv4AL3/857vjmr2hpbunQbsEzizhn749y9/d/G3XokmTsnCne/Z+4RDYSN7My4EmgNLmf+9z9+qj2J9Jb3Hr9HRQVF3HrMz9l2aJlfPuCGxm130jq9q1t1+6wEw/lhHOOpbK6kvpNW/j+xTcx+5d/5oMXnL6rTfOOZu74xkzGTtgn14fRx1kUD0qORJQj8e3ACe4+HpgATDGzIyLcn0jsGrc18sycZ5n6xXMpryxj/0n7MemkQ/nb75/q0HbPkUOprK5MLLhjBcaby99q1+ah22Yz/j8OYviYvXIRvrQRykg8siTuCVuSi8XJl0e1P5He4I2lb1JQWMheo4ftWjdyvxGsfH1V2vZPzfoH54+/gE9N+gzLX1nBydNO3LVt3ep1/PV3T3DOpR+OPG7pyJIzVLr7ikukJzbNrBCYD+wD3Ozuz0S5P5G4NW5rpKJfebt1FVUVNGxtTNv+6DOP4ugzj2LN0jU88eBT9K/pv2vbbTfMZOrliRG95FaiJh7GKcNIo3T3FnefANQCk83swNQ2ZjbdzOaZ2bz169+OMhyRyJVVlLFtS0O7dQ1bGrpMxMNGD6NubC2/uO4OAOY9Np/GLQ0cdcb7I4tV8kNOphi6+yYzexyYArycsm0GMANg4qGHqNwiQdtr9J60trSwZukahiVLKssWLadubG0Xn4TWllbeWpGoif/r/xbw75eX8l+HfxaAbfXbKCgsYPlrK7n651dEdwCSpIt9MLPBwI5kAi8HTga+E9X+RHqDsooyJp9yGPfcdB+f/faFLFu4nHmPzudbv/t6h7aP3vs4h504kf41/Vn5+ioe+NkfmHD0wQBMvfxcPnTRmbva3nHDTAYOHcg5l5yds2Pp63TZPQwDfpmsixcAv3X3hyPcn0ivcOENn+aWL/+cCyZ/lqoB/bjwG5+mbt9aFs59hf/59He461+Jksmr81/l7u/fS+O27VTvUcX7Tz2cqV88F4DyfuWUt6mtl5SVUFZeStWAfrEcU5+jp92Du78EHBJV/yK9VdWAfnw5Tclj3GH77UrgABd/96Ju93nJjd1vK+/dzot9QqDL7kVE0ujzI3ERkXBZMFMMlcRFRNII5bJ7JXERkRSqiYuIBE41cRGRYMV7U6tMKImLiKShkbiISKASD0rW7BQRkTDpik0RkZCpJi4iEjSNxEVEAhbKSDyMyr2IiKSlkbiISApdsSkiEjrVxEVEQqXZKSIiQdPsFBGRgGkkLiISsFCSuKYYioikMBLllExe3erXbIqZvWpmi83s6jTbv2hmC83sJTN7zMxGdtWnkriISAeW8U+XPZoVAjcDpwLjgGlmNi6l2fPAJHc/GLgP+G5X/SqJi4ikke0kDkwGFrv7EndvAu4BzmrbwN0fd/dtycWngdquOu1lNXHDLP++Vwry8Jh2qioeEHcIkTip9gNxhxCZc669Lu4Qsm/95uz217O7GNaY2bw2yzPcfUab5eHAyjbLq4DDO+nvAuBPXe20lyVxEZHeoQcnNte7+6Ss7NvsPGAScGxXbZXERURS7DyxmWWrgbo2y7XJde33bXYS8BXgWHff3lWn+ft7vohIj2X/xCYwFxhrZqPNrASYCsxqt1ezQ4CfA2e6+9rudKqRuIhIGtmeJ+7uzWZ2CTAHKARud/cFZnYDMM/dZwE3Av2A3yV/E1jh7md21q+SuIhIGlFcdu/us4HZKeuua/P+pEz7VBIXEUlDV2yKiEjkNBIXEUmhh0KIiASt+/dDiZuSuIhIWkriIiJh6tll97FQEhcRSUM1cRGRgCmJi4gEynRiU0QkbBqJi4gETElcRCRgKqeIiARMI3ERkUDpxKaISOA0EhcRCZqSuIjkm+0t8OxaWLMNSgth/CAYVdWxXVMLzF+faAcwthoOGvTu9sdWw+bt0OLQrxgO2gNq++XmGLopjBSuJC4imZi3DgoMzh4Nm7bD39bAwBLoX9q+3XProaUVzhwJjS3w+GqoLIYx1YntE2ugf0mir/WNie1nlEF570lJodTEI38ohJkVmtnzZvZw1PsSkQg1t8KqLYlRc3EBDC6H4ZWwtL5j29VbYf+BUFSQGGmPqYYl77y7fWBpIoFDYsjbCmxrzsVR5J1cfO1dBiwCqnOwLxGJyjs7wAyqS95dN6AE1jZ07/Obmtov/+0NeLMBWh32rIA9StN/LjYaiWNmtcDpwK1R7kdEcqC5NTECb6u4ILE+1bAKWLgRdrRCfVNiFN6S0u7YveDcMXDsMBhWnviC6EUsw1dcoi6n3ARcReKXJREJWVFBIim3taM1sT7VoYOh0ODh5fDkmzCyCirS/OJfYLBXJaxpgFVbo4m7RzJN4fGl8cjKKWZ2BrDW3eeb2XGdtJsOTAeoG1EXVTgi8l5VF4N7YmRdlSypbGpKnKBMVVoIR+757vKLb8MeZbvv2x227MhuvO+BBfRQiChH4kcBZ5rZMuAe4AQzuyu1kbvPcPdJ7j6ppqYmwnBE5D0pKkhMA3xpQ6KEsq4hcQJzdJophvU7EtMRWx3e2AqLN8OBAxPb3mlKrGtuTWxfWp/oa0gnSV52K7KRuLtfA1wDkByJX+nu50W1PxHJgUmD4Zm18MDSxGh70uDE9MK1DYkTlefunWi3sTExzbCpFaqK4cih705DdODlDbC5KTHkrSpOjNo7G6nHQFdsikj+KS2EY4Z1XD+k/N0EDjCiKvFKp38JnNL7S6dK4m24+xPAE7nYl4hIX6KRuIhIGjqxKSIikdNIXESkA1NNXEQkbEriIiJBivtS+kwoiYuIpBHKiU0lcRGRtJTERUSCFUYKVxIXEdmNMNK45omLiARMI3ERkQ4smBObGomLiARMI3ERkRSJeeJhjMSVxEVE0lISFxEJVhgpXElcRCStUE5sKomLiHQQzt1TlMRFRNIII4VriqGIyG5Yhq9u9Gg2xcxeNbPFZnZ1mu2lZnZvcvszZjaqqz6VxEVEUlmiJp7Jq8suzQqBm4FTgXHANDMbl9LsAmCju+8D/BD4Tlf9KomLiOTGZGCxuy9x9ybgHuCslDZnAb9Mvr8PONG6+IZQEhcRSbHzYp9MfrphOLCyzfKq5Lq0bdy9GdgMDOqs0151YvP5555fX1lctTxHu6sB1udoX7mk4wpPvh5bLo9rZDY7e27+83PKiyprMvxYmZnNa7M8w91nZDOudHpVEnf3wbnal5nNc/dJudpfrui4wpOvxxbycbn7lAi6XQ3UtVmuTa5L12aVmRUB/YG3O+tU5RQRkdyYC4w1s9FmVgJMBWaltJkFfFw8Lo8AAALASURBVCL5/hzgr+7unXXaq0biIiL5yt2bzewSYA5QCNzu7gvM7AZgnrvPAm4DfmVmi4ENJBJ9p/pyEo+8VhWTWI/LzFqAf5H4u7UI+IS7b+thX3cCD7v7fUCDmY1z94W7aXsc0OTu/5fhPpYBk9w9zpq0/i72Ee4+G5idsu66Nu8bgXMz6bPPllNyccIhDr3guBrcfYK7Hwg0ARe13Zis82XM3Y/eXQJPOg44sid9x60X/JlFIl+Pq7fps0lccuIpYB8zO87MnjKzWcBCMys0sxvNbK6ZvWRmnwGwhJ8kr2h7FBiysyMze8LMJiXfTzGz58zsRTN7LHlV20XA5Wb2gpkdbWaDzez+5D7mmtlRyc8OMrNHzGyBmd1KOFdXi6TVl8spEqHkiPtU4M/JVROBA919qZlNBza7+2FmVgr8w8weAQ4B3kfiarahwELg9pR+BwO/AI5J9rWHu28ws58BW9z9e8l2vwF+6O5/N7MRJOqQ+wPXA3939xvM7HQSV8iJBEtJXLKt3MxeSL5/isSJmiOBZ919aXL9KcDBZnZOcrk/MBY4Brjb3VuAN8zsr2n6PwJ4cmdf7r5hN3GcBIxrc7FbtZn1S+7jw8nP/tHMNvbwOEV6BSVxybYGd5/QdkUykW5tuwq41N3npLQ7LYtxFABHJE8UpcYikjdUE5c4zAE+a2bFAGa2r5lVAk8C/5msmQ8Djk/z2aeBY8xsdPKzeyTX1wNVbdo9Aly6c8HMdn6xPAl8NLnuVGBg1o5KJAZK4hKHW0nUu58zs5eBn5P4rfBB4PXktpnAP1M/6O7rgOnAA2b2InBvctNDwNk7T2wCnwcmJU+cLuTdWTJfJ/ElsIBEWWVFRMcokhPWxcVAIiLSi2kkLiISMCVxEZGAKYmLiARMSVxEJGBK4iIiAVMSFxEJmJK4iEjAlMRFRAL2/wHZmcqvKauqQAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 504x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.79167048, 0.09299771, 0.04686499, 0.00778032, 0.0606865 ],\n",
              "       [0.37840909, 0.30633117, 0.20275974, 0.03068182, 0.08181818],\n",
              "       [0.11536718, 0.1362194 , 0.4493427 , 0.1569583 , 0.14211242],\n",
              "       [0.02268431, 0.00933365, 0.10284735, 0.33731096, 0.52782372],\n",
              "       [0.01175861, 0.00116922, 0.00892857, 0.04500159, 0.93314201]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    }
  ]
}