From 2899deafeffb4a15098a1daa695124e7b6e4eb90 Mon Sep 17 00:00:00 2001 From: Kartik Choudhary Date: Sun, 15 Oct 2023 18:53:17 -0400 Subject: [PATCH 1/5] Added info about required packages --- ...d-question-answering-model-debugging.ipynb | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb b/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb index 3b663cfc61..4af484b9f1 100644 --- a/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb +++ b/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb @@ -42,6 +42,31 @@ "The following section examines the code necessary to create datasets and a model. It then generates insights using the `responsibleai` API that can be visually analyzed." ] }, +{ + "cell_type": "markdown", + "id": "6174bcad", + "metadata": {}, + "source": [ + "### Prepare\n", + "\n", + "To run this notebook, we need to install the following packages:\n", + "\n", + "```requirements.txt\n", + "raiutils\n", + "raiwidgets\n", + "datasets\n", + "transformers\n", + "responsibleai_text\n", + "torch\n", + "```\n", + "\n", + "Run the following command to load the spacy pipeline:\n", + "\n", + "```bash\n", + "python -m spacy download en_core_web_sm\n", + "```" + ] + }, { "cell_type": "markdown", "id": "40739025", From 80b0c3454bb8b34998da9e0bdb71af3c079bf829 Mon Sep 17 00:00:00 2001 From: Kartik Choudhary Date: Sun, 15 Oct 2023 18:53:55 -0400 Subject: [PATCH 2/5] Update responsibleaidashboard-question-answering-model-debugging.ipynb --- ...ponsibleaidashboard-question-answering-model-debugging.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb b/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb index 4af484b9f1..d804c9bed8 100644 --- a/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb +++ b/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb @@ -51,7 +51,7 @@ "\n", "To run this notebook, we need to install the following packages:\n", "\n", - "```requirements.txt\n", + "```\n", "raiutils\n", "raiwidgets\n", "datasets\n", From af0099398be0a75b2588a219a79db69057c63bf2 Mon Sep 17 00:00:00 2001 From: Kartik Choudhary Date: Sun, 15 Oct 2023 19:00:01 -0400 Subject: [PATCH 3/5] show example prediction --- ...d-question-answering-model-debugging.ipynb | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb b/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb index d804c9bed8..2d8de2ffcd 100644 --- a/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb +++ b/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb @@ -111,16 +111,7 @@ "metadata": {}, "outputs": [], "source": [ - "dataset = datasets.load_dataset(\"squad\", split=\"train\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a0eef443", - "metadata": {}, - "outputs": [], - "source": [ + "dataset = datasets.load_dataset(\"squad\", split=\"train\")\n", "dataset" ] }, @@ -155,17 +146,9 @@ "metadata": {}, "outputs": [], "source": [ - "data = pd.DataFrame({'context': context, 'questions': questions, 'answers': answers})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e6f87e9c", - "metadata": {}, - "outputs": [], - "source": [ - "data" + "data = pd.DataFrame({'context': context, 'questions': questions, 'answers': answers})\n", + "data = data.sample(frac=1.0, random_state=42).reset_index(drop=True)\n", + "data.head()" ] }, { @@ -184,18 +167,42 @@ "outputs": [], "source": [ "# load the question-answering model\n", - "pmodel = pipeline('question-answering')" + "pipeline_model = pipeline('question-answering')\n", + "test_size = 5\n", + "\n", + "train_data = data\n", + "test_data = data[:test_size]" + ] + }, + { + "cell_type": "markdown", + "id": "7cf8327b", + "metadata": {}, + "source": [ + "See an example of the model's predictions" ] }, { "cell_type": "code", "execution_count": null, - "id": "04801887", + "id": "ce087699", "metadata": {}, "outputs": [], "source": [ - "train_data = data\n", - "test_data = data[:5]" + "def get_answer(dataset, idx):\n", + " model_output = pipeline_model(question=dataset['questions'][idx], \n", + " context=dataset['context'][idx])\n", + " pred = model_output['answer']\n", + " return pred\n", + "\n", + "def check_answer(dataset, idx):\n", + " pred = get_answer(dataset, idx)\n", + " print('Question : ', dataset['questions'][idx])\n", + " print('Answer : ', dataset['answers'][idx])\n", + " print('Predicted : ', pred)\n", + " print('Correct : ', pred == dataset['answers'][idx])\n", + "\n", + "check_answer(test_data, 0)\n" ] }, { From 6c19f0fd3187d18631324a699d6ef9ecb24e9916 Mon Sep 17 00:00:00 2001 From: Kartik Choudhary Date: Sun, 15 Oct 2023 19:01:09 -0400 Subject: [PATCH 4/5] Update responsibleaidashboard-question-answering-model-debugging.ipynb --- ...onsibleaidashboard-question-answering-model-debugging.ipynb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb b/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb index 2d8de2ffcd..dbcb6b8dc9 100644 --- a/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb +++ b/notebooks/responsibleaidashboard/text/responsibleaidashboard-question-answering-model-debugging.ipynb @@ -241,8 +241,7 @@ "metadata": {}, "outputs": [], "source": [ - "rai_insights = RAITextInsights(pmodel, test_data,\n", - " \"answers\",\n", + "rai_insights = RAITextInsights(pipeline_model, test_data, \"answers\",\n", " task_type=ModelTask.QUESTION_ANSWERING)" ] }, From 6f59a06c234452436e53f4f515d8dc8cded5217b Mon Sep 17 00:00:00 2001 From: Kartik Choudhary Date: Fri, 2 Feb 2024 13:47:41 -0500 Subject: [PATCH 5/5] Updated openai example notebook to use the new generative text task type Signed-off-by: Kartik Choudhary --- ...leaidashboard-openai-model-debugging.ipynb | 99 ++++++++++--------- 1 file changed, 55 insertions(+), 44 deletions(-) diff --git a/notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb b/notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb index 4a6ac68186..61fefe1280 100644 --- a/notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb +++ b/notebooks/responsibleaidashboard/text/responsibleaidashboard-openai-model-debugging.ipynb @@ -101,18 +101,6 @@ "dataset" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "46cd83d3", - "metadata": {}, - "outputs": [], - "source": [ - "def replace_error_chars(message):\n", - " message = message.replace('`', '')\n", - " return message" - ] - }, { "cell_type": "markdown", "id": "f1bffd45", @@ -128,13 +116,14 @@ "metadata": {}, "outputs": [], "source": [ - "questions = []\n", - "context = []\n", - "answers = []\n", + "template = ('Answer the question given the context.\\n\\n'\n", + " 'context:\\n{context}\\n\\n'\n", + " 'question:\\n{question}')\n", + "prompts = []\n", "for row in dataset:\n", - " context.append(row['context'])\n", - " questions.append(row['question'])\n", - " answers.append(replace_error_chars(row['answers']['text'][0]))" + " templated_prompt = template.format(context=row['context'],\n", + " question=row['question'])\n", + " prompts.append(templated_prompt)" ] }, { @@ -144,17 +133,7 @@ "metadata": {}, "outputs": [], "source": [ - "data = pd.DataFrame({'context': context, 'questions': questions, 'answers': answers})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e6f87e9c", - "metadata": {}, - "outputs": [], - "source": [ - "data.iloc[:5]['answers'][0]" + "data = pd.DataFrame({'prompt': prompts})" ] }, { @@ -189,7 +168,13 @@ "api_key = \"put_your_secret_key_here\"\n", "\n", "# Openai Wrapper that calls endpoint with given questions\n", - "openai_model = OpenaiWrapperModel(api_type, api_base, api_version, api_key, engine='gpt-4')" + "openai_model = OpenaiWrapperModel(api_type, api_base, api_version,\n", + " api_key, engine='gpt-35-turbo-16k')\n", + "\n", + "# Openai Wrapper for the evaluation model that will judge the quality of the\n", + "# answers\n", + "openai_eval_model = OpenaiWrapperModel(api_type, api_base, api_version,\n", + " api_key, engine='gpt-4')" ] }, { @@ -217,21 +202,35 @@ " self.model = model\n", "\n", " def predict(self, dataset):\n", - " template = 'Answer the question given the context.'\n", - " for i, (context, question) in enumerate(zip(dataset['context'], dataset['questions'])):\n", - " templated_question = template + '\\n\\ncontext: ' + context + '\\nquestion: ' + question\n", - " if isinstance(dataset, pd.DataFrame):\n", - " dataset.iloc[i]['questions'] = templated_question\n", - " else:\n", - " dataset['questions'] = templated_question\n", " # NOTE: Remove this patch when calling your openai model\n", " with patch('ml_wrappers.model.OpenaiWrapperModel.predict') as mock_predict:\n", + " # dummy return value\n", + " dummy_prediction = 'This is a dummy prediction'\n", + "\n", " # wrap return value in mock class\n", " if isinstance(dataset, pd.DataFrame):\n", - " mock_predict.return_value = np.array(data['answers'][dataset.index])\n", + " prediction = [dummy_prediction] * len(dataset)\n", " else:\n", - " mock_predict.return_value = np.array(data['answers'][0])\n", - " context = {}\n", + " prediction = [dummy_prediction]\n", + " mock_predict.return_value = np.array(prediction)\n", + " \n", + " # Note: When calling a real openai model just return this line here\n", + " return self.model.predict(dataset)\n", + " \n", + "class template_eval(object):\n", + " def __init__(self, model):\n", + " self.model = model\n", + "\n", + " def predict(self, dataset):\n", + " # NOTE: Remove this patch when calling your openai model\n", + " with patch('ml_wrappers.model.OpenaiWrapperModel.predict') as mock_predict:\n", + " # dummy return value\n", + " dummy_prediction = 4\n", + "\n", + " # wrap return value in mock class\n", + " prediction = [dummy_prediction] * len(dataset)\n", + " mock_predict.return_value = np.array(prediction)\n", + "\n", " # Note: When calling a real openai model just return this line here\n", " return self.model.predict(dataset)" ] @@ -258,6 +257,18 @@ "pipeline.predict(test_data)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "010112e5", + "metadata": {}, + "outputs": [], + "source": [ + "eval_pipeline = template_eval(openai_eval_model)\n", + "eval_data = pipeline.predict(test_data)\n", + "eval_pipeline.predict(eval_data)" + ] + }, { "cell_type": "markdown", "id": "9fa4f8f2", @@ -294,9 +305,9 @@ "metadata": {}, "outputs": [], "source": [ - "rai_insights = RAITextInsights(pipeline, test_data,\n", - " \"answers\",\n", - " task_type=ModelTask.QUESTION_ANSWERING)" + "rai_insights = RAITextInsights(pipeline, test_data, None,\n", + " task_type=ModelTask.GENERATIVE_TEXT,\n", + " text_column='prompt', eval_model=eval_pipeline)" ] }, { @@ -372,7 +383,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.17" + "version": "3.10.13" } }, "nbformat": 4,