From 2a9d9734c9fc3be4441305705901a511f0e4aac2 Mon Sep 17 00:00:00 2001 From: Matt Gotteiner <[email protected]> Date: Thu, 22 Feb 2024 14:55:51 -0800 Subject: [PATCH] isolating dependencies --- ...r-python-langchain-sample-requirements.txt | 6 + ...earch-vector-python-langchain-sample.ipynb | 285 ++++++++---------- .../code/langchain_preview_patch/__init__.py | 0 .../langchain_preview_patch/azuresearch.py | 211 ------------- 4 files changed, 140 insertions(+), 362 deletions(-) create mode 100644 demo-python/code/azure-search-vector-python-langchain-sample-requirements.txt delete mode 100644 demo-python/code/langchain_preview_patch/__init__.py delete mode 100644 demo-python/code/langchain_preview_patch/azuresearch.py diff --git a/demo-python/code/azure-search-vector-python-langchain-sample-requirements.txt b/demo-python/code/azure-search-vector-python-langchain-sample-requirements.txt new file mode 100644 index 0000000..2d7b619 --- /dev/null +++ b/demo-python/code/azure-search-vector-python-langchain-sample-requirements.txt @@ -0,0 +1,6 @@ +python-dotenv +azure-search-documents==11.6.0b1 +azure-identity +langchain==0.1.8 +langchain-openai==0.0.6 +pypdf \ No newline at end of file diff --git a/demo-python/code/azure-search-vector-python-langchain-sample.ipynb b/demo-python/code/azure-search-vector-python-langchain-sample.ipynb index 69d43ec..b24733b 100644 --- a/demo-python/code/azure-search-vector-python-langchain-sample.ipynb +++ b/demo-python/code/azure-search-vector-python-langchain-sample.ipynb @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -46,7 +46,7 @@ } ], "source": [ - "! pip install -r requirements.txt --quiet" + "! pip install -r azure-search-vector-python-langchain-sample-requirements.txt --quiet" ] }, { @@ -58,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -118,85 +118,24 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 6, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "vector_search_configuration is not a known attribute of class and will be ignored\n" - ] - } - ], + "outputs": [], "source": [ "from langchain.vectorstores.azuresearch import AzureSearch\n", - "from azure.search.documents.indexes import SearchIndexClient\n", - "from azure.search.documents.indexes.models import (\n", - " SearchIndex,\n", - " SearchField,\n", - " VectorSearch,\n", - " HnswAlgorithmConfiguration,\n", - " VectorSearchProfile,\n", - " SemanticSearch,\n", - " SemanticConfiguration,\n", - " SemanticPrioritizedFields,\n", - " SemanticField\n", - ")\n", - "from langchain_preview_patch.azuresearch import fix_vectorstore\n", "\n", - "# LangChain is not yet compatible with the latest preview version of the Search SDK\n", - "# A workaround is to create the index prior to using the LangChain vector store\n", - "search_index = SearchIndex(\n", - " name=index_name,\n", - " fields=[\n", - " SearchField(name=\"id\", key=True, type=\"Edm.String\", searchable=True, filterable=True, facetable=False, sortable=True, hidden=False),\n", - " SearchField(name=\"content\", type=\"Edm.String\", searchable=True, filterable=False, facetable=False, sortable=False, hidden=False),\n", - " SearchField(name=\"content_vector\", type=\"Collection(Edm.Single)\", searchable=True, vector_search_dimensions=1536, vector_search_profile_name=\"myHnswProfile\", hidden=False),\n", - " SearchField(name=\"metadata\", type=\"Edm.String\", searchable=True, filterable=False, facetable=False, sortable=False, hidden=False)\n", - " ],\n", - " vector_search=VectorSearch(\n", - " algorithms=[\n", - " HnswAlgorithmConfiguration(name=\"hnsw\")\n", - " ],\n", - " profiles=[\n", - " VectorSearchProfile(name=\"myHnswProfile\", algorithm_configuration_name=\"hnsw\")\n", - " ]\n", - " ),\n", - " semantic_search=SemanticSearch(\n", - " default_configuration_name=\"semantic\",\n", - " configurations=[\n", - " SemanticConfiguration(\n", - " name=\"semantic\",\n", - " prioritized_fields=SemanticPrioritizedFields(\n", - " content_fields=[\n", - " SemanticField(field_name=\"content\")\n", - " ]\n", - " )\n", - " )\n", - " ]\n", - " )\n", - ")\n", - "search_index_result = SearchIndexClient(endpoint=endpoint, credential=credential).create_or_update_index(search_index)\n", - "\n", - "# This code will generate a warning that can safely be ignored.\n", "vector_store = AzureSearch(\n", " azure_search_endpoint=endpoint,\n", " azure_search_key=key_credential,\n", - " fields=search_index_result.fields,\n", - " vector_search=search_index_result.vector_search,\n", - " semantic_configuration_name=search_index_result.semantic_search.default_configuration_name,\n", " index_name=index_name,\n", - " embedding_function=embeddings.embed_query\n", - ")\n", - "# The LangChain vector search methods will not work with a newer SDK\n", - "# These methods need to be patched using the replacements found in the langchain_patch directory\n", - "fix_vectorstore(vector_store)" + " embedding_function=embeddings.embed_query,\n", + " semantic_configuration_name=\"default\"\n", + ")" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -238,7 +177,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -246,26 +185,43 @@ "output_type": "stream", "text": [ "--------------------------------------------------------------------------------\n", - "ID: MjNlYWE2OWMtNmIyYy00ZWUzLWE4ZjEtNTg1M2U1OTFiMGUy\n", - "Chunk Content: It is important to remember that the Northwind Health Plus plan covers only medically \n", - "necessary services. Non -essential services, such as elective or cosmetic procedures, are not \n", - "covered.\n", + "Source: ..\\data\\documents\\Northwind_Health_Plus_Benefits_Details.pdf\n", + "Chunk Content: Specialty Care Services: Northwind Health Plus covers specialty care services, such as \n", + "physical therapy, occupational therapy, and mental health services. The plan also covers \n", + "services provided by specialists, such as cardiologists, endocrinologists, and neurologists. \n", + "Exceptions: Northwin d Health Plus does not cover services that are not medically \n", + "necessary, such as cosmetic surgery, elective treatments, and experimental treatments. In \n", + "addition, the plan does not cover services for conditions that are not covered by the plan, \n", + "such as pre -existing conditions.\n", "--------------------------------------------------------------------------------\n", - "ID: OGVmNGI3MzItODU0Zi00ZDYwLTg5NGYtNTk0NjgyMDgwNzVh\n", - "Chunk Content: included in the plan documents or summary, then it does not apply to the plan. \n", - "You should also be aware that the Northwind Health Plus plan may contain certain \n", - "exceptions, exclusions, and limitations. It is important to familiarize yourself with the plan \n", - "documents to make sure that you u nderstand what services are covered and which are not \n", - "covered. If you have any questions, Northwind Health has customer service representatives \n", - "who are available to answer your questions.\n", + "Source: ..\\data\\documents\\Northwind_Health_Plus_Benefits_Details.pdf\n", + "Chunk Content: is important to understand these exclusions so that you can plan your care accordingly. \n", + "Services Not Covered: \n", + "Northwind Health Plus does not cover services that are not medically necessary, such as \n", + "cosmetic surgery or elective procedures. Additionally, services or treatments that are \n", + "experimental or investigational are not covered under this plan. \n", + "Prescriptions N ot Covered: The plan does not cover prescriptions that are not medically \n", + "necessary, certain over -the-counter medications, or prescription medications that are used \n", + "to enhance performance in athletics. \n", + "Mental Health and Substance Abuse Treatment: The plan does not cover mental health or \n", + "substance abuse treatment services provided by a non -network provider or any services \n", + "that are not medically necessary. \n", + "Preventive Care: Northwind Health Plus does not cover preventive care services provided \n", + "by a non -network provider. \n", + "Tips for Avoiding Exclusions\n", "--------------------------------------------------------------------------------\n", - "ID: NmUzYzYwNTMtNzAxZi00MjAxLWIzNTYtYzkwYWMxZTYyOGE0\n", - "Chunk Content: care servi ces. The plans also cover preventive care services such as mammograms, colonoscopies, and \n", - "other cancer screenings. \n", - "Northwind Health Plus offers more comprehensive coverage than Northwind Standard. This plan offers \n", - "coverage for emergency services, both in -network and out -of-network, as well as mental health and \n", - "substance abuse coverage. Northwind Standard does not offer coverage for emergency services, mental \n", - "health and substance abuse coverage, or out -of-network services.\n" + "Source: ..\\data\\documents\\Benefit_Options.pdf\n", + "Chunk Content: Both plans offer coverage for pre scription drugs. Northwind Health Plus offers a wider range of \n", + "prescription drug coverage than Northwind Standard. Northwind Health Plus covers generic, brand -\n", + "name, and specialty drugs, while Northwind Standard only covers generic and brand -name drugs. \n", + "Both plans offer coverage for vision and dental services. Northwind Health Plus offers coverage for vision \n", + "exams, glasses, and contact lenses, as well as dental exams, cleanings, and fillings. Northwind Standard \n", + "only offers coverage for vision exams and glas ses. \n", + "Both plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, \n", + "doctor visits, lab tests, and X -rays. Northwind Standard only offers coverage for doctor visits and lab \n", + "tests. \n", + "Northwind Health Plus is a compr ehensive plan that offers more coverage than Northwind Standard. \n", + "Northwind Health Plus offers coverage for emergency services, mental health and substance abuse\n" ] } ], @@ -278,7 +234,7 @@ ")\n", "for doc in docs:\n", " print(\"-\" * 80) \n", - " print(f\"ID: {doc.metadata['id']}\")\n", + " print(f\"Source: {doc.metadata['source']}\")\n", " print(f\"Chunk Content: {doc.page_content}\")" ] }, @@ -291,7 +247,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -299,26 +255,43 @@ "output_type": "stream", "text": [ "--------------------------------------------------------------------------------\n", - "ID: OGVmNGI3MzItODU0Zi00ZDYwLTg5NGYtNTk0NjgyMDgwNzVh\n", - "Chunk Content: included in the plan documents or summary, then it does not apply to the plan. \n", - "You should also be aware that the Northwind Health Plus plan may contain certain \n", - "exceptions, exclusions, and limitations. It is important to familiarize yourself with the plan \n", - "documents to make sure that you u nderstand what services are covered and which are not \n", - "covered. If you have any questions, Northwind Health has customer service representatives \n", - "who are available to answer your questions.\n", + "Source: ..\\data\\documents\\Northwind_Health_Plus_Benefits_Details.pdf\n", + "Chunk Content: is important to understand these exclusions so that you can plan your care accordingly. \n", + "Services Not Covered: \n", + "Northwind Health Plus does not cover services that are not medically necessary, such as \n", + "cosmetic surgery or elective procedures. Additionally, services or treatments that are \n", + "experimental or investigational are not covered under this plan. \n", + "Prescriptions N ot Covered: The plan does not cover prescriptions that are not medically \n", + "necessary, certain over -the-counter medications, or prescription medications that are used \n", + "to enhance performance in athletics. \n", + "Mental Health and Substance Abuse Treatment: The plan does not cover mental health or \n", + "substance abuse treatment services provided by a non -network provider or any services \n", + "that are not medically necessary. \n", + "Preventive Care: Northwind Health Plus does not cover preventive care services provided \n", + "by a non -network provider. \n", + "Tips for Avoiding Exclusions\n", "--------------------------------------------------------------------------------\n", - "ID: NmUzYzYwNTMtNzAxZi00MjAxLWIzNTYtYzkwYWMxZTYyOGE0\n", - "Chunk Content: care servi ces. The plans also cover preventive care services such as mammograms, colonoscopies, and \n", - "other cancer screenings. \n", - "Northwind Health Plus offers more comprehensive coverage than Northwind Standard. This plan offers \n", - "coverage for emergency services, both in -network and out -of-network, as well as mental health and \n", - "substance abuse coverage. Northwind Standard does not offer coverage for emergency services, mental \n", - "health and substance abuse coverage, or out -of-network services.\n", + "Source: ..\\data\\documents\\Benefit_Options.pdf\n", + "Chunk Content: Both plans offer coverage for pre scription drugs. Northwind Health Plus offers a wider range of \n", + "prescription drug coverage than Northwind Standard. Northwind Health Plus covers generic, brand -\n", + "name, and specialty drugs, while Northwind Standard only covers generic and brand -name drugs. \n", + "Both plans offer coverage for vision and dental services. Northwind Health Plus offers coverage for vision \n", + "exams, glasses, and contact lenses, as well as dental exams, cleanings, and fillings. Northwind Standard \n", + "only offers coverage for vision exams and glas ses. \n", + "Both plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, \n", + "doctor visits, lab tests, and X -rays. Northwind Standard only offers coverage for doctor visits and lab \n", + "tests. \n", + "Northwind Health Plus is a compr ehensive plan that offers more coverage than Northwind Standard. \n", + "Northwind Health Plus offers coverage for emergency services, mental health and substance abuse\n", "--------------------------------------------------------------------------------\n", - "ID: MjNlYWE2OWMtNmIyYy00ZWUzLWE4ZjEtNTg1M2U1OTFiMGUy\n", - "Chunk Content: It is important to remember that the Northwind Health Plus plan covers only medically \n", - "necessary services. Non -essential services, such as elective or cosmetic procedures, are not \n", - "covered.\n" + "Source: ..\\data\\documents\\Northwind_Health_Plus_Benefits_Details.pdf\n", + "Chunk Content: Specialty Care Services: Northwind Health Plus covers specialty care services, such as \n", + "physical therapy, occupational therapy, and mental health services. The plan also covers \n", + "services provided by specialists, such as cardiologists, endocrinologists, and neurologists. \n", + "Exceptions: Northwin d Health Plus does not cover services that are not medically \n", + "necessary, such as cosmetic surgery, elective treatments, and experimental treatments. In \n", + "addition, the plan does not cover services for conditions that are not covered by the plan, \n", + "such as pre -existing conditions.\n" ] } ], @@ -331,7 +304,7 @@ ")\n", "for doc in docs:\n", " print(\"-\" * 80) \n", - " print(f\"ID: {doc.metadata['id']}\")\n", + " print(f\"Source: {doc.metadata['source']}\")\n", " print(f\"Chunk Content: {doc.page_content}\")" ] }, @@ -344,7 +317,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -352,47 +325,57 @@ "output_type": "stream", "text": [ "--------------------------------------------------------------------------------\n", - "Content: Northwind Standard, you can choose from a variety of in -network providers, including primary care \n", - "physicians, specialists, hospitals, and pharmacies. This plan does not offer coverage for emergency \n", - "services, mental health and substance abuse coverage, or out -of-network services.\n", - "Comparison of Plans \n", - "Both plans offer coverage for routine physicals, well -child visits, immunizations, and other preventive \n", - "care servi ces. The plans also cover preventive care services such as mammograms, colonoscopies, and \n", - "other cancer screenings. \n", - "Northwind Health Plus offers more comprehensive coverage than Northwind Standard. This plan offers \n", - "coverage for emergency services, both in -network and out -of-network, as well as mental health and \n", - "substance abuse coverage. Northwind Standard does not offer coverage for emergency services, mental \n", - "health and substance abuse coverage, or out -of-network services.\n", - "Score: 0.014305992051959038\n", - "Caption: Northwind Standard does not offer coverage for emergency services, mental health and substance abuse coverage, or out -of-network services..\u0000\n", + "Content: is important to understand these exclusions so that you can plan your care accordingly. \n", + "Services Not Covered: \n", + "Northwind Health Plus does not cover services that are not medically necessary, such as \n", + "cosmetic surgery or elective procedures. Additionally, services or treatments that are \n", + "experimental or investigational are not covered under this plan. \n", + "Prescriptions N ot Covered: The plan does not cover prescriptions that are not medically \n", + "necessary, certain over -the-counter medications, or prescription medications that are used \n", + "to enhance performance in athletics. \n", + "Mental Health and Substance Abuse Treatment: The plan does not cover mental health or \n", + "substance abuse treatment services provided by a non -network provider or any services \n", + "that are not medically necessary. \n", + "Preventive Care: Northwind Health Plus does not cover preventive care services provided \n", + "by a non -network provider. \n", + "Tips for Avoiding Exclusions\n", + "Score: 0.021343937143683434\n", + "Caption: is important to understand these exclusions so that you can plan your care accordingly. Services Not Covered: Northwind Health Plus does not cover services that are not medically necessary, such as cosmetic surgery or elective procedures.\n", "--------------------------------------------------------------------------------\n", - "Content: Northwind Standard, you can choose from a variety of in -network providers, including primary care \n", - "physicians, specialists, hospitals, and pharmacies. This plan does not offer coverage for emergency \n", - "services, mental health and substance abuse coverage, or out -of-network services.\n", - "Comparison of Plans \n", - "Both plans offer coverage for routine physicals, well -child visits, immunizations, and other preventive \n", - "care servi ces. The plans also cover preventive care services such as mammograms, colonoscopies, and \n", - "other cancer screenings. \n", - "Northwind Health Plus offers more comprehensive coverage than Northwind Standard. This plan offers \n", - "coverage for emergency services, both in -network and out -of-network, as well as mental health and \n", - "substance abuse coverage. Northwind Standard does not offer coverage for emergency services, mental \n", - "health and substance abuse coverage, or out -of-network services.\n", - "Score: 0.014177345670759678\n", - "Caption: Northwind Standard does not offer coverage for emergency services, mental health and substance abuse coverage, or out -of-network services..\u0000\n", + "Content: Your coverage for skilled nursing facility care is limited to a maximum of 100 days. In order \n", + "to continue to receive benefits, you must make measurable progress toward a goal that is \n", + "established by your doctor. \n", + "What Are the Exceptions to Skilled Nursing Fa cility Care Coverage? \n", + "Not all skilled nursing facility services are covered under Northwind Standard. The plan \n", + "does not cover custodial care or services that are not medically necessary. Custodial care \n", + "includes services that are provided to help you with a ctivities of daily living such as bathing, \n", + "dressing, and other basic care. \n", + "Other exceptions to coverage include services provided in an assisted living facility, a \n", + "hospice, or in the home. \n", + "Tips for Getting the Most Out of Your Skilled Nursing Facility Care \n", + "• Make sure to communicate with your doctor and the skilled nursing facility staff about \n", + "your progress and treatment plan. \n", + "• Ask questions about any treatments or services that you are unfamiliar with.\n", + "Score: 0.012987012974917889\n", + "Caption: Not all skilled nursing facility services are covered under Northwind Standard. The plan does not cover custodial care or services that are not medically necessary. Custodial care includes services that are provided to help you with a ctivities of daily living such as bathing, dressing, and other basic care.\n", "--------------------------------------------------------------------------------\n", - "Content: Northwind Standard, you can choose from a variety of in -network providers, including primary care \n", - "physicians, specialists, hospitals, and pharmacies. This plan does not offer coverage for emergency \n", - "services, mental health and substance abuse coverage, or out -of-network services.\n", - "Comparison of Plans \n", - "Both plans offer coverage for routine physicals, well -child visits, immunizations, and other preventive \n", - "care servi ces. The plans also cover preventive care services such as mammograms, colonoscopies, and \n", - "other cancer screenings. \n", - "Northwind Health Plus offers more comprehensive coverage than Northwind Standard. This plan offers \n", - "coverage for emergency services, both in -network and out -of-network, as well as mental health and \n", - "substance abuse coverage. Northwind Standard does not offer coverage for emergency services, mental \n", - "health and substance abuse coverage, or out -of-network services.\n", - "Score: 0.013949556276202202\n", - "Caption: Northwind Standard does not offer coverage for emergency services, mental health and substance abuse coverage, or out -of-network services..\u0000\n" + "Content: physical, mental, and social well -being. \n", + "What is Covered under the Northwind Health Plus Plan? \n", + "The Northwind Health Plus plan covers a wide range of services related to the treatment of \n", + "SUD. These services include inpatient and outpatient treatment, counseling, and \n", + "medications to help with recovery. It also covers mental health services and support for \n", + "family members of those with SUD. \n", + "Exceptions \n", + "Not all services related to SUD are covered by the Northwind Health Plus plan. For example, \n", + "the plan does not cover experimental treatments or services that are not medically \n", + "necessary. It also does not cover services provided by non -network providers. \n", + "Tips for Employees \n", + "If you or someone you care about is struggling with SUD, there a re a few things you can do \n", + "to get the most out of your Northwind Health Plus plan: \n", + "• Talk to your doctor or a mental health professional about your symptoms and the \n", + "treatments that may be available.\n", + "Score: 0.011111111380159855\n", + "Caption: The Northwind Health Plus plan covers a wide range of services related to the treatment of SUD. These services include inpatient and outpatient treatment, counseling, and medications to help with recovery. It also covers mental health services and support for family members of those with SUD.\n" ] } ], diff --git a/demo-python/code/langchain_preview_patch/__init__.py b/demo-python/code/langchain_preview_patch/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/demo-python/code/langchain_preview_patch/azuresearch.py b/demo-python/code/langchain_preview_patch/azuresearch.py deleted file mode 100644 index 9ae267e..0000000 --- a/demo-python/code/langchain_preview_patch/azuresearch.py +++ /dev/null @@ -1,211 +0,0 @@ -from azure.search.documents import SearchClient -from azure.search.documents.models import ( - VectorizedQuery, - QueryType, - QueryCaptionType, - QueryAnswerType -) -from langchain_core.documents import Document -from langchain_community.vectorstores.azuresearch import ( - AzureSearch, - FIELDS_CONTENT_VECTOR, - FIELDS_CONTENT, - FIELDS_ID, - FIELDS_METADATA -) -from typing import Optional, List, Tuple, Any -import numpy as np -import json - -def vector_search_with_score( - self, query: str = "", k: int = 4, filters: Optional[str] = None -) -> List[Tuple[Document, float]]: - """Return docs most similar to query. - - Args: - query: Text to look up documents similar to. - k: Number of Documents to return. Defaults to 4. - - Returns: - List of Documents most similar to the query and score for each - """ - - results = self.client.search( - search_text="", - vector_queries=[ - VectorizedQuery( - vector=np.array(self.embed_query(query), dtype=np.float32).tolist(), - k_nearest_neighbors=k, - fields=FIELDS_CONTENT_VECTOR, - ) - ], - filter=filters, - ) - # Convert results to Document objects - docs = [ - ( - Document( - page_content=result.pop(FIELDS_CONTENT), - metadata={ - **( - {FIELDS_ID: result.pop(FIELDS_ID)} - if FIELDS_ID in result - else {} - ), - **( - json.loads(result[FIELDS_METADATA]) - if FIELDS_METADATA in result - else { - k: v - for k, v in result.items() - if k != FIELDS_CONTENT_VECTOR - } - ), - }, - ), - float(result["@search.score"]), - ) - for result in results - ] - return docs - - -def hybrid_search_with_score( - self, query: str = "", k: int = 4, filters: Optional[str] = None -) -> List[Tuple[Document, float]]: - """Return docs most similar to query with an hybrid query. - - Args: - query: Text to look up documents similar to. - k: Number of Documents to return. Defaults to 4. - - Returns: - List of Documents most similar to the query and score for each - """ - - results = self.client.search( - search_text=query, - vector_queries=[ - VectorizedQuery( - vector=np.array(self.embed_query(query), dtype=np.float32).tolist(), - k_nearest_neighbors=k, - fields=FIELDS_CONTENT_VECTOR, - ) - ], - filter=filters, - top=k, - ) - # Convert results to Document objects - docs = [ - ( - Document( - page_content=result.pop(FIELDS_CONTENT), - metadata={ - **( - {FIELDS_ID: result.pop(FIELDS_ID)} - if FIELDS_ID in result - else {} - ), - **( - json.loads(result[FIELDS_METADATA]) - if FIELDS_METADATA in result - else { - k: v - for k, v in result.items() - if k != FIELDS_CONTENT_VECTOR - } - ), - }, - ), - float(result["@search.score"]), - ) - for result in results - ] - return docs - -def semantic_hybrid_search_with_score_and_rerank( - self, query: str = "", k: int = 4, filters: Optional[str] = None -) -> List[Tuple[Document, float, float]]: - """Return docs most similar to query with an hybrid query. - - Args: - query: Text to look up documents similar to. - k: Number of Documents to return. Defaults to 4. - - Returns: - List of Documents most similar to the query and score for each - """ - - results = self.client.search( - search_text=query, - vector_queries=[ - VectorizedQuery( - vector=np.array(self.embed_query(query), dtype=np.float32).tolist(), - k_nearest_neighbors=50, - fields=FIELDS_CONTENT_VECTOR, - ) - ], - filter=filters, - query_type=QueryType.SEMANTIC, - semantic_configuration_name=self.semantic_configuration_name, - query_caption=QueryCaptionType.EXTRACTIVE, - query_answer=QueryAnswerType.EXTRACTIVE, - top=k, - ) - # Get Semantic Answers - semantic_answers = results.get_answers() or [] - semantic_answers_dict = {} - for semantic_answer in semantic_answers: - semantic_answers_dict[semantic_answer.key] = { - "text": semantic_answer.text, - "highlights": semantic_answer.highlights, - } - # Convert results to Document objects - docs = [ - ( - Document( - page_content=result.pop(FIELDS_CONTENT), - metadata={ - **( - {FIELDS_ID: result.pop(FIELDS_ID)} - if FIELDS_ID in result - else {} - ), - **( - json.loads(result[FIELDS_METADATA]) - if FIELDS_METADATA in result - else { - k: v - for k, v in result.items() - if k != FIELDS_CONTENT_VECTOR - } - ), - **{ - "captions": { - "text": result.get("@search.captions", [{}])[0].text, - "highlights": result.get("@search.captions", [{}])[ - 0 - ].highlights, - } - if result.get("@search.captions") - else {}, - "answers": semantic_answers_dict.get( - json.loads(result[FIELDS_METADATA]).get("key") - if FIELDS_METADATA in result - else "", - "", - ), - }, - }, - ), - float(result["@search.score"]), - float(result["@search.reranker_score"]), - ) - for result in results - ] - return docs - -def fix_vectorstore(store: AzureSearch): - store.vector_search_with_score = vector_search_with_score.__get__(store, store.__class__) - store.hybrid_search_with_score = hybrid_search_with_score.__get__(store, store.__class__) - store.semantic_hybrid_search_with_score_and_rerank = semantic_hybrid_search_with_score_and_rerank.__get__(store, store.__class__) \ No newline at end of file