Updated openai example notebook to use the new generative text task type

notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb

@@ -101,18 +101,6 @@
     "dataset"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "46cd83d3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def replace_error_chars(message):\n",
-    "    message = message.replace('`', '')\n",
-    "    return message"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "f1bffd45",
@@ -128,13 +116,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "questions = []\n",
-    "context = []\n",
-    "answers = []\n",
+    "template = ('Answer the question given the context.\\n\\n'\n",
+    "            'context:\\n{context}\\n\\n'\n",
+    "            'question:\\n{question}')\n",
+    "prompts = []\n",
     "for row in dataset:\n",
-    "    context.append(row['context'])\n",
-    "    questions.append(row['question'])\n",
-    "    answers.append(replace_error_chars(row['answers']['text'][0]))"
+    "    templated_prompt = template.format(context=row['context'],\n",
+    "                                       question=row['question'])\n",
+    "    prompts.append(templated_prompt)"
    ]
   },
   {
@@ -144,17 +133,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "data = pd.DataFrame({'context': context, 'questions': questions, 'answers': answers})"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e6f87e9c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "data.iloc[:5]['answers'][0]"
+    "data = pd.DataFrame({'prompt': prompts})"
    ]
   },
   {
@@ -189,7 +168,13 @@
     "api_key = \"put_your_secret_key_here\"\n",
     "\n",
     "# Openai Wrapper that calls endpoint with given questions\n",
-    "openai_model = OpenaiWrapperModel(api_type, api_base, api_version, api_key, engine='gpt-4')"
+    "openai_model = OpenaiWrapperModel(api_type, api_base, api_version,\n",
+    "                                  api_key, engine='gpt-35-turbo-16k')\n",
+    "\n",
+    "# Openai Wrapper for the evaluation model that will judge the quality of the\n",
+    "# answers\n",
+    "openai_eval_model = OpenaiWrapperModel(api_type, api_base, api_version,\n",
+    "                                       api_key, engine='gpt-4')"
    ]
   },
   {
@@ -217,21 +202,35 @@
     "        self.model = model\n",
     "\n",
     "    def predict(self, dataset):\n",
-    "        template = 'Answer the question given the context.'\n",
-    "        for i, (context, question) in enumerate(zip(dataset['context'], dataset['questions'])):\n",
-    "            templated_question = template + '\\n\\ncontext: ' + context + '\\nquestion: ' + question\n",
-    "            if isinstance(dataset, pd.DataFrame):\n",
-    "                dataset.iloc[i]['questions'] = templated_question\n",
-    "            else:\n",
-    "                dataset['questions'] = templated_question\n",
     "        # NOTE: Remove this patch when calling your openai model\n",
     "        with patch('ml_wrappers.model.OpenaiWrapperModel.predict') as mock_predict:\n",
+    "            # dummy return value\n",
+    "            dummy_prediction = 'This is a dummy prediction'\n",
+    "\n",
     "            # wrap return value in mock class\n",
     "            if isinstance(dataset, pd.DataFrame):\n",
-    "                mock_predict.return_value = np.array(data['answers'][dataset.index])\n",
+    "                prediction = [dummy_prediction] * len(dataset)\n",
     "            else:\n",
-    "                mock_predict.return_value = np.array(data['answers'][0])\n",
-    "            context = {}\n",
+    "                prediction = [dummy_prediction]\n",
+    "            mock_predict.return_value = np.array(prediction)\n",
+    "            \n",
+    "            # Note: When calling a real openai model just return this line here\n",
+    "            return self.model.predict(dataset)\n",
+    "        \n",
+    "class template_eval(object):\n",
+    "    def __init__(self, model):\n",
+    "        self.model = model\n",
+    "\n",
+    "    def predict(self, dataset):\n",
+    "        # NOTE: Remove this patch when calling your openai model\n",
+    "        with patch('ml_wrappers.model.OpenaiWrapperModel.predict') as mock_predict:\n",
+    "            # dummy return value\n",
+    "            dummy_prediction = 4\n",
+    "\n",
+    "            # wrap return value in mock class\n",
+    "            prediction = [dummy_prediction] * len(dataset)\n",
+    "            mock_predict.return_value = np.array(prediction)\n",
+    "\n",
     "            # Note: When calling a real openai model just return this line here\n",
     "            return self.model.predict(dataset)"
    ]
@@ -258,6 +257,18 @@
     "pipeline.predict(test_data)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "010112e5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eval_pipeline = template_eval(openai_eval_model)\n",
+    "eval_data = pipeline.predict(test_data)\n",
+    "eval_pipeline.predict(eval_data)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "9fa4f8f2",
@@ -294,9 +305,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "rai_insights = RAITextInsights(pipeline, test_data,\n",
-    "                               \"answers\",\n",
-    "                               task_type=ModelTask.QUESTION_ANSWERING)"
+    "rai_insights = RAITextInsights(pipeline, test_data, None,\n",
+    "                               task_type=ModelTask.GENERATIVE_TEXT,\n",
+    "                               text_column='prompt', eval_model=eval_pipeline)"
    ]
   },
   {
@@ -372,7 +383,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.17"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,