From 3f357b93ff36af768de6ae2f6d46927e4e09f728 Mon Sep 17 00:00:00 2001
From: robin <850379744@qq.com>
Date: Sat, 28 Oct 2023 10:20:09 +0800
Subject: [PATCH] remove es
---
.../chat2db-server-web-api/pom.xml | 5 -
.../web/api/controller/ai/ChatController.java | 13 +-
.../api/controller/rdb/RdbDdlController.java | 2 +-
.../api/controller/rdb/TableController.java | 2 +-
.../server/web/api/util/SegmentUtils.java | 94 -
chat2db-server/pom.xml | 6 -
library/ambiguity.dic | 11 -
library/default.dic | 386260 ---------------
library/regex.dic | 0
library/stop.dic | 9 -
library/synonyms.dic | 9996 -
11 files changed, 8 insertions(+), 396390 deletions(-)
delete mode 100644 chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/util/SegmentUtils.java
delete mode 100644 library/ambiguity.dic
delete mode 100644 library/default.dic
delete mode 100644 library/regex.dic
delete mode 100644 library/stop.dic
delete mode 100644 library/synonyms.dic
diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/pom.xml b/chat2db-server/chat2db-server-web/chat2db-server-web-api/pom.xml
index 4b60a06e6..58e627461 100644
--- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/pom.xml
+++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/pom.xml
@@ -84,11 +84,6 @@
-
- org.ansj
- ansj_seg
-
-
diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/ChatController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/ChatController.java
index 73f4f83e2..7ac77ca40 100644
--- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/ChatController.java
+++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/ai/ChatController.java
@@ -45,7 +45,6 @@
import ai.chat2db.server.web.api.http.response.EsTableSchemaResponse;
import ai.chat2db.server.web.api.http.response.TableSchemaResponse;
import ai.chat2db.server.web.api.util.ApplicationContextUtil;
-import ai.chat2db.server.web.api.util.SegmentUtils;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson2.JSON;
@@ -556,10 +555,9 @@ public String mappingDatabaseSchema(ChatQueryRequest queryRequest) {
if (StringUtils.isNotBlank(apiKey)) {
boolean res = gatewayClientService.checkInWhite(new WhiteListRequest(apiKey, WhiteListTypeEnum.VECTOR.getCode())).getData();
if (res) {
- properties = queryDatabaseSchema(queryRequest) + querySchemaByEs(queryRequest);
+// properties = queryDatabaseSchema(queryRequest) + querySchemaByEs(queryRequest);
+ properties = queryDatabaseSchema(queryRequest);
}
- } else {
- properties = querySchemaByEs(queryRequest);
}
return properties;
}
@@ -573,10 +571,11 @@ public String mappingDatabaseSchema(ChatQueryRequest queryRequest) {
*/
public String queryDatabaseSchema(ChatQueryRequest queryRequest) {
// request embedding
- String input = SegmentUtils.baseAnalysis(queryRequest.getMessage());
- log.info("search message:{}", input);
- FastChatEmbeddingResponse response = distributeAIEmbedding(input);
+ FastChatEmbeddingResponse response = distributeAIEmbedding(queryRequest.getMessage());
List> contentVector = new ArrayList<>();
+ if (Objects.isNull(response) || CollectionUtils.isEmpty(response.getData())) {
+ return "";
+ }
contentVector.add(response.getData().get(0).getEmbedding());
// search embedding
diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDdlController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDdlController.java
index 0c77797ec..3871a71ce 100644
--- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDdlController.java
+++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/RdbDdlController.java
@@ -78,7 +78,7 @@ public WebPageResult list(@Valid TableBriefQueryRequest request) {
try {
Chat2DBContext.putContext(connectInfo);
syncTableVector(request);
- syncTableEs(request);
+// syncTableEs(request);
} catch (Exception e) {
log.error("sync table vector error", e);
} finally {
diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java
index d0e0b78a3..3e90d62d8 100644
--- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java
+++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/TableController.java
@@ -69,7 +69,7 @@ public WebPageResult list(@Valid TableBriefQueryRequest request) {
try {
Chat2DBContext.putContext(connectInfo);
syncTableVector(request);
- syncTableEs(request);
+// syncTableEs(request);
} catch (Exception e) {
log.error("sync table vector error", e);
} finally {
diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/util/SegmentUtils.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/util/SegmentUtils.java
deleted file mode 100644
index d3add687f..000000000
--- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/util/SegmentUtils.java
+++ /dev/null
@@ -1,94 +0,0 @@
-package ai.chat2db.server.web.api.util;
-
-import lombok.extern.slf4j.Slf4j;
-import org.ansj.domain.Result;
-import org.ansj.domain.Term;
-import org.ansj.splitWord.analysis.BaseAnalysis;
-import org.ansj.splitWord.analysis.NlpAnalysis;
-import org.ansj.splitWord.analysis.ToAnalysis;
-
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-@Slf4j
-public class SegmentUtils {
-
- /**
- * BaseAnalysis
- *
- * @param content
- */
- public static String baseAnalysis(String content) {
- Result result = BaseAnalysis.parse(delHTMLTag(content).replace("\n", "").replace(" ", "").replace("\t", ""));
- log.info("base analysis result:" + result);
- return convertResToString(result);
- }
-
- /**
- * ToAnalysis
- *
- * @param content
- */
- public static String toAnalysis(String content) {
- Result result = ToAnalysis.parse(content);
- log.info("to analysis result:" + result);
- return convertResToString(result);
- }
-
- /**
- * NlpAnalysis
- *
- * @param content
- */
- public static String nlpAnalysis(String content) {
- Result result = NlpAnalysis.parse(delHTMLTag(content).replace("\n", "").replace(" ", "").replace("\t", ""));
- log.info("nlp analysis result:" + result);
- return convertResToString(result);
- }
-
- /**
- * convert result to string
- *
- * @param result
- * @return
- */
- private static String convertResToString(Result result) {
- List terms = result.getTerms();
- StringBuilder sb = new StringBuilder();
- for (Term term : terms) {
- String name = term.getName();
- String nature = term.getNatureStr();
- if (nature.equals("nt") || nature.equals("nr") || nature.equals("n")) {
- sb.append(name).append(" ");
- }
- }
- return sb.toString();
- }
-
- /**
- * delete html tag
- *
- * @param htmlStr
- * @return
- */
- public static String delHTMLTag(String htmlStr) {
- String regEx_script = "