diff --git a/notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb b/notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb index 4a6ac68186..61fefe1280 100644 --- a/notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb +++ b/notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb @@ -101,18 +101,6 @@ "dataset" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "46cd83d3", - "metadata": {}, - "outputs": [], - "source": [ - "def replace_error_chars(message):\n", - " message = message.replace('`', '')\n", - " return message" - ] - }, { "cell_type": "markdown", "id": "f1bffd45", @@ -128,13 +116,14 @@ "metadata": {}, "outputs": [], "source": [ - "questions = []\n", - "context = []\n", - "answers = []\n", + "template = ('Answer the question given the context.\\n\\n'\n", + " 'context:\\n{context}\\n\\n'\n", + " 'question:\\n{question}')\n", + "prompts = []\n", "for row in dataset:\n", - " context.append(row['context'])\n", - " questions.append(row['question'])\n", - " answers.append(replace_error_chars(row['answers']['text'][0]))" + " templated_prompt = template.format(context=row['context'],\n", + " question=row['question'])\n", + " prompts.append(templated_prompt)" ] }, { @@ -144,17 +133,7 @@ "metadata": {}, "outputs": [], "source": [ - "data = pd.DataFrame({'context': context, 'questions': questions, 'answers': answers})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e6f87e9c", - "metadata": {}, - "outputs": [], - "source": [ - "data.iloc[:5]['answers'][0]" + "data = pd.DataFrame({'prompt': prompts})" ] }, { @@ -189,7 +168,13 @@ "api_key = \"put_your_secret_key_here\"\n", "\n", "# Openai Wrapper that calls endpoint with given questions\n", - "openai_model = OpenaiWrapperModel(api_type, api_base, api_version, api_key, engine='gpt-4')" + "openai_model = OpenaiWrapperModel(api_type, api_base, api_version,\n", + " api_key, engine='gpt-35-turbo-16k')\n", + "\n", + "# Openai Wrapper for the evaluation model that will judge the quality of the\n", + "# answers\n", + "openai_eval_model = OpenaiWrapperModel(api_type, api_base, api_version,\n", + " api_key, engine='gpt-4')" ] }, { @@ -217,21 +202,35 @@ " self.model = model\n", "\n", " def predict(self, dataset):\n", - " template = 'Answer the question given the context.'\n", - " for i, (context, question) in enumerate(zip(dataset['context'], dataset['questions'])):\n", - " templated_question = template + '\\n\\ncontext: ' + context + '\\nquestion: ' + question\n", - " if isinstance(dataset, pd.DataFrame):\n", - " dataset.iloc[i]['questions'] = templated_question\n", - " else:\n", - " dataset['questions'] = templated_question\n", " # NOTE: Remove this patch when calling your openai model\n", " with patch('ml_wrappers.model.OpenaiWrapperModel.predict') as mock_predict:\n", + " # dummy return value\n", + " dummy_prediction = 'This is a dummy prediction'\n", + "\n", " # wrap return value in mock class\n", " if isinstance(dataset, pd.DataFrame):\n", - " mock_predict.return_value = np.array(data['answers'][dataset.index])\n", + " prediction = [dummy_prediction] * len(dataset)\n", " else:\n", - " mock_predict.return_value = np.array(data['answers'][0])\n", - " context = {}\n", + " prediction = [dummy_prediction]\n", + " mock_predict.return_value = np.array(prediction)\n", + " \n", + " # Note: When calling a real openai model just return this line here\n", + " return self.model.predict(dataset)\n", + " \n", + "class template_eval(object):\n", + " def __init__(self, model):\n", + " self.model = model\n", + "\n", + " def predict(self, dataset):\n", + " # NOTE: Remove this patch when calling your openai model\n", + " with patch('ml_wrappers.model.OpenaiWrapperModel.predict') as mock_predict:\n", + " # dummy return value\n", + " dummy_prediction = 4\n", + "\n", + " # wrap return value in mock class\n", + " prediction = [dummy_prediction] * len(dataset)\n", + " mock_predict.return_value = np.array(prediction)\n", + "\n", " # Note: When calling a real openai model just return this line here\n", " return self.model.predict(dataset)" ] @@ -258,6 +257,18 @@ "pipeline.predict(test_data)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "010112e5", + "metadata": {}, + "outputs": [], + "source": [ + "eval_pipeline = template_eval(openai_eval_model)\n", + "eval_data = pipeline.predict(test_data)\n", + "eval_pipeline.predict(eval_data)" + ] + }, { "cell_type": "markdown", "id": "9fa4f8f2", @@ -294,9 +305,9 @@ "metadata": {}, "outputs": [], "source": [ - "rai_insights = RAITextInsights(pipeline, test_data,\n", - " \"answers\",\n", - " task_type=ModelTask.QUESTION_ANSWERING)" + "rai_insights = RAITextInsights(pipeline, test_data, None,\n", + " task_type=ModelTask.GENERATIVE_TEXT,\n", + " text_column='prompt', eval_model=eval_pipeline)" ] }, { @@ -372,7 +383,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.17" + "version": "3.10.13" } }, "nbformat": 4,