增加了任务管理的历史记录存储，index都采用 mds_demo1的方式

gtbook/src/main/java/org/cnnlp/data/splitter/BaseMdParser.java

@@ -380,6 +380,8 @@ public class BaseMdParser {
             Node nd = ls.get(i);
 
             GTNode2 gn = GTNode2.buildLeaf(i + baseNo);
+            // 2025.9.3 增加
+            gn.setMdNode(nd);
             ch1.addChild(gn);
             processNode(nd, gn, i);
             bns[i] = gn;

gtbook/src/main/java/org/cnnlp/data/splitter/SimpleMdSplitter.java

@@ -67,15 +67,15 @@ public class SimpleMdSplitter extends BaseMdParser implements ISplitter {
             if (beginIdx >= 1) {
                 GTNode2 nd = nodes[beginIdx - 1];
                 if (nd.getType() == GTBookConstants.MD_COMMENTS) {
-                    Map<String, Object> meta1 = processComment1(beginIdx-1, beginIdx);
-                    mergeMap(rets,meta1);
+                    Map<String, Object> meta1 = processComment1(beginIdx - 1, beginIdx);
+                    mergeMap(rets, meta1);
                 }
             }
             for (int i = beginIdx; i < endIdx; i++) {
                 GTNode2 nd = nodes[i];
                 if (nd.getType() == GTBookConstants.MD_COMMENTS) {
                     Map<String, Object> meta1 = processComment1(i, i + 1);
-                    mergeMap(rets,meta1);
+                    mergeMap(rets, meta1);
                 }
             }
         }
@@ -88,7 +88,7 @@ public class SimpleMdSplitter extends BaseMdParser implements ISplitter {
             List<String> hs2 = SplitUtils.mdStringToList(hs);
             Map<String, Object> meta1 = HtmlCommentParser.parseKv(hs2);
             return meta1;
-        }else{
+        } else {
             return Map.of();
         }
     }
@@ -150,7 +150,7 @@ public class SimpleMdSplitter extends BaseMdParser implements ISplitter {
                     //builder.metadata(GDocConstants.RAW_CONTENT, SplitUtils.listToString(htmls));
                     builder.metadata(GDocConstants.FIRST_HEADING, nd.getLabel());
 
-                    getCharOffset(builder,nowJ, nextJ);
+                    getCharOffset(builder, nowJ, nextJ);
                     Map<String, Object> metadata = getMetadata(nowJ, nextJ);
                     metadata.forEach((k, v) -> builder.metadata(k, v));
 
@@ -166,10 +166,26 @@ public class SimpleMdSplitter extends BaseMdParser implements ISplitter {
                 i++;
             }
         }
+        // 没有任何标题
+        if (docs.size() <= 0 && i > 0) {
+            List<String> texts = getTexts(0, docLen);
+            GDocument.Builder builder = GDocument.builder().id(idGen.generateId()).text(SplitUtils.listToString(texts));
+
+            builder.metadata(GDocConstants.CURRENT_SECTION_TOC, List.of(""));
+            //builder.metadata(GDocConstants.FIRST_HEADING, nd.getLabel());
+
+            getCharOffset(builder, 0, docLen);
+            Map<String, Object> metadata = getMetadata(0, docLen);
+            metadata.forEach((k, v) -> builder.metadata(k, v));
+
+            GDocument doc = builder.build();
+            docs.add(doc);
+        }
+
         return docs;
     }
 
-    protected void getCharOffset(GDocument.Builder builder,int startIndex, int endIndex){
+    protected void getCharOffset(GDocument.Builder builder, int startIndex, int endIndex) {
         String text = docTree.getText();
         Node mdNode = nodes[startIndex].getMdNode();
         int startOffset = mdNode.getStartOffset();
@@ -177,13 +193,13 @@ public class SimpleMdSplitter extends BaseMdParser implements ISplitter {
         if (endIndex < nodes.length) {
             mdNode = nodes[endIndex].getMdNode();
             endOffset = mdNode.getStartOffset();
-        }else {
+        } else {
             endOffset = text.length();
         }
 
         String s = text.substring(startOffset, endOffset);
         //System.out.println("s="+s);
-        builder.metadata(GDocConstants.RAW_CONTENT,s);
+        builder.metadata(GDocConstants.RAW_CONTENT, s);
         builder.metadata(GDocConstants.START_OFFSET, startOffset);
         builder.metadata(GDocConstants.END_OFFSET, endOffset);
     }

server/pom.xml

@@ -9,7 +9,7 @@
         <version>1.0.0</version>
     </parent>
 
-    <version>2.1.2</version>
+    <version>2.5.0</version>
     <artifactId>mdserver</artifactId>
 
     <properties>

server/src/main/java/com/giantan/data/index/HybridIndexer.java

@@ -0,0 +1,113 @@
+package com.giantan.data.index;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class HybridIndexer {
+    public static final String COLL_ID = "__cid";
+    public static final String DOC_ID = "__did";
+
+
+
+    protected String defaultIndexPrefix = "qas";
+    protected String defaultChunkMode = "single";
+
+    public HybridIndexer() {
+
+    }
+
+    public String getDefaultIndexPrefix() {
+        return defaultIndexPrefix;
+    }
+
+    public void setDefaultIndexPrefix(String defaultIndexPrefix) {
+        this.defaultIndexPrefix = defaultIndexPrefix;
+    }
+
+    public String getDefaultChunkMode() {
+        return defaultChunkMode;
+    }
+
+    public void setDefaultChunkMode(String defaultChunkMode) {
+        this.defaultChunkMode = defaultChunkMode;
+    }
+
+    public String indexName(String coll) {
+        return this.defaultIndexPrefix + "_" + coll;
+    }
+
+    public String indexName(String prefix, String coll) {
+        return prefix + coll;
+    }
+
+    public String getConfigIndexName(String collId, Map<String, Object> attibutes) {
+        //String mapped = indexName(collId);
+        String mapped = null;
+        if (attibutes != null && !attibutes.isEmpty()) {
+            Object o = attibutes.get(IndexConfig.INDEX_STRATEGY);
+            if (o != null && o instanceof Map strategy) {
+                Object o1 = strategy.get(IndexConfig.INDEX_MODE);
+                if (o1 != null && o1 instanceof String mode) {
+                    if (mode.equals(IndexConfig.INDEX_MODE_T2C)) {
+                        Object o2 = strategy.get(IndexConfig.INDEX_COLLECTION_PREFIX);
+                        if (o2 != null && o2 instanceof String prefix) {
+                            mapped = indexName(prefix, collId);
+                        }
+                    } else if (mode.equals(IndexConfig.INDEX_MODE_SINGLE)) {
+                        Object o2 = strategy.get(IndexConfig.INDEX_GLOBAL_COLLECTION);
+                        if (o2 != null && o2 instanceof String collection) {
+                            mapped = collection;
+                        }
+                    }
+                }
+            }
+        }
+        return mapped;
+    }
+
+    public Boolean getConfigOne2One(Map<String, Object> attibutes) {
+        Boolean isOne2One = null;
+        if (attibutes != null && !attibutes.isEmpty()) {
+            Object o = attibutes.get(IndexConfig.INDEX_STRATEGY);
+            if (o != null && o instanceof Map strategy) {
+                Object o1 = strategy.get(IndexConfig.INDEX_MODE);
+                if (o1 != null && o1 instanceof String mode) {
+                    if (mode.equals(IndexConfig.INDEX_MODE_SINGLE)) {
+                        isOne2One = false;
+                    } else if (mode.equals(IndexConfig.INDEX_MODE_T2C)) {
+                        isOne2One = true;
+                    }
+                }
+            }
+        }
+        return isOne2One;
+    }
+
+    public String getConfigChunkMode(Map<String, Object> attibutes) {
+        String chunkMode = null;
+
+        if (attibutes != null && !attibutes.isEmpty()) {
+            Object o = attibutes.get(IndexConfig.CHUNK_MODE);
+            if (o != null && o instanceof String mode) {
+                chunkMode = mode;
+            }
+        }
+        return chunkMode;
+    }
+
+    public List<String> getConfigChunkTemplates(Map<String, Object> attibutes) {
+        List<String> rets = null;
+        if (attibutes != null && !attibutes.isEmpty()) {
+            Object o = attibutes.get(IndexConfig.CHUNK_TEMPLATES);
+            if (o != null && o instanceof List<?> templates) {
+                rets = new ArrayList<>(templates.size());
+                for (Object o1 : templates) {
+                    rets.add(o1.toString());
+                }
+            }
+        }
+        return rets;
+    }
+
+}

server/src/main/java/com/giantan/data/index/HybridSearch.java

@@ -134,7 +134,7 @@ public class HybridSearch implements IHybridSearch {
     }
 
     @Override
-    public boolean deleteAll(String coll) throws IOException, InterruptedException {
+    public boolean clearCollection(String coll) throws IOException, InterruptedException {
         HttpRequest request = HttpRequest.newBuilder()
                 .uri(URI.create(url + coll + "/clear"))
                 .header("User-Agent", clientInfo)
@@ -148,7 +148,7 @@ public class HybridSearch implements IHybridSearch {
     }
 
     @Override
-    public boolean drop(String coll) throws IOException, InterruptedException {
+    public boolean dropCollection(String coll) throws IOException, InterruptedException {
         HttpRequest request = HttpRequest.newBuilder()
                 .uri(URI.create(url + coll))
                 .header("User-Agent", clientInfo)
@@ -299,4 +299,37 @@ public class HybridSearch implements IHybridSearch {
         }
         return rets;
     }
+
+    @Override
+    public int createCollection(String coll) throws IOException, InterruptedException {
+        HttpRequest request = HttpRequest.newBuilder()
+                .uri(URI.create(url + coll))
+                .header("User-Agent", "insomnia/11.0.2")
+                .method("POST", HttpRequest.BodyPublishers.noBody())
+                .build();
+        HttpResponse<String> response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
+        //System.out.println(response.body());
+        Map<String, Object> ret = JsonUtil.fromJsonString(response.body());
+        Object o = ret.get("data");
+        if (o != null && o instanceof Boolean) {
+            return 1;
+        }
+        return 0;
+    }
+
+    @Override
+    public List<String> listCollections() throws IOException, InterruptedException {
+        HttpRequest request = HttpRequest.newBuilder()
+                .uri(URI.create(url))
+                .header("User-Agent", "insomnia/11.0.2")
+                .method("GET", HttpRequest.BodyPublishers.noBody())
+                .build();
+        HttpResponse<String> response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());
+        Map<String, Object> ret = JsonUtil.fromJsonString(response.body());
+        Object o = ret.get("data");
+        if (o != null && o instanceof List) {
+            return (List<String>) o;
+        }
+        return new ArrayList<>();
+    }
 }

server/src/main/java/com/giantan/data/index/HybridSearch2Controller.java

@@ -17,33 +17,33 @@ import org.springframework.web.bind.annotation.*;
 import java.util.List;
 
 @RestController
-    @RequestMapping(IHybridSearch.API_PREFIX + "/collections/{coll}")
-    public class HybridSearch2Controller {
-        private static final org.slf4j.Logger log
-                = org.slf4j.LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-        @Autowired
-        IHybridSearch hybridSearch;
-
-        // /by-ids
-        @PostMapping("/documents/batch")
-        public ResponseEntity<R> getDocumentsByIds(@PathVariable String coll, @RequestBody List<String> ids) throws IOException, InterruptedException {
-            List<DocResp> docs = hybridSearch.getDocumentsByIds(coll, ids);
-            return ResponseEntity.ok(R.data(docs));
-        }
+@RequestMapping(IHybridSearch.API_PREFIX + "/collections/{coll}")
+public class HybridSearch2Controller {
+    private static final org.slf4j.Logger log
+            = org.slf4j.LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+    @Autowired
+    IHybridSearch hybridSearch;
+
+    // /by-ids
+    @PostMapping("/documents/batch")
+    public ResponseEntity<R> getDocumentsByIds(@PathVariable String coll, @RequestBody List<String> ids) throws IOException, InterruptedException {
+        List<DocResp> docs = hybridSearch.getDocumentsByIds(coll, ids);
+        return ResponseEntity.ok(R.data(docs));
+    }
 
-        @GetMapping("/{id}")
-        public ResponseEntity<R> getDocuments(@PathVariable String coll, @PathVariable String id) throws IOException, InterruptedException {
-            List<DocResp> docs = hybridSearch.getDocumentsByIds(coll, List.of(id));
-            if (docs.isEmpty()) {
-                return ResponseEntity.ok(R.data(null));
-            }
-            return ResponseEntity.ok(R.data(docs.get(0)));
+    @GetMapping("/{id}")
+    public ResponseEntity<R> getDocuments(@PathVariable String coll, @PathVariable String id) throws IOException, InterruptedException {
+        List<DocResp> docs = hybridSearch.getDocumentsByIds(coll, List.of(id));
+        if (docs.isEmpty()) {
+            return ResponseEntity.ok(R.data(null));
         }
+        return ResponseEntity.ok(R.data(docs.get(0)));
+    }
 
-        @GetMapping("/ids")
-        public ResponseEntity<R> getAllDocumentIds(@PathVariable String coll) throws IOException, InterruptedException {
-            List<String> ids = hybridSearch.getAllIds(coll);
-            return ResponseEntity.ok(R.data(ids));
+    @GetMapping("/ids")
+    public ResponseEntity<R> getAllDocumentIds(@PathVariable String coll) throws IOException, InterruptedException {
+        List<String> ids = hybridSearch.getAllIds(coll);
+        return ResponseEntity.ok(R.data(ids));
 
-        }
     }
+}

server/src/main/java/com/giantan/data/index/HybridSearchController.java

@@ -1,6 +1,5 @@
 package com.giantan.data.index;
 
-import com.giantan.data.qa.constant.QaConstants;
 import jakarta.servlet.http.HttpServletRequest;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.HttpEntity;

server/src/main/java/com/giantan/data/index/IHybridSearch.java

@@ -9,7 +9,7 @@ import java.util.List;
 import java.util.Map;
 
 public interface IHybridSearch {
-    public static final String API_PREFIX = "/v1/se";
+    String API_PREFIX = "/v1/se";
 
     List<DocResp> add(String coll, List<DocReq> docs) throws IOException, InterruptedException;
 
@@ -21,12 +21,10 @@ public interface IHybridSearch {
 
     List<String> getAllIds(String coll) throws IOException, InterruptedException;
 
-    boolean deleteAll(String coll) throws IOException, InterruptedException;
-
-    boolean drop(String coll) throws IOException, InterruptedException;
-
+    // 可以是  doc_id 也可以是 "doc_id%"，例如 01k2zv5r5f75cz9k305wb8d489%
     List<DocResp> getDocumentsByIdFilter(String coll, String filter) throws IOException, InterruptedException;
 
+    // 可以是  doc_id 也可以是 "doc_id%"，例如 01k2zv5r5f75cz9k305wb8d489%
     int deleteDocumentsByIdFilter(String coll, String filter) throws IOException, InterruptedException;
 
     int deleteDocumentsByFilter(String coll, Map<String, Object> query) throws IOException, InterruptedException;
@@ -36,4 +34,13 @@ public interface IHybridSearch {
     List<DocSearchResp> similaritySearch(String coll, Map<String, Object> query) throws IOException, InterruptedException;
 
     List<DocSearchResp> hybridSearch(String coll, Map<String, Object> query) throws IOException, InterruptedException;
+
+    /////////
+    int createCollection(String coll) throws IOException, InterruptedException;
+
+    boolean clearCollection(String coll) throws IOException, InterruptedException;
+
+    boolean dropCollection(String coll) throws IOException, InterruptedException;
+
+    List<String> listCollections() throws IOException, InterruptedException;
 }

server/src/main/java/com/giantan/data/mds/service/IVectorization.java → server/src/main/java/com/giantan/data/index/IVectorization.java

@@ -1,6 +1,4 @@
-package com.giantan.data.mds.service;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
+package com.giantan.data.index;
 
 import java.io.IOException;
 import java.util.List;

server/src/main/java/com/giantan/data/index/IndexConfig.java

@@ -1,5 +1,23 @@
 package com.giantan.data.index;
 
 public class IndexConfig {
+    //可配置项（用户可控）
+    //字段选择：["name", "description", "tags", "attributes.color"]
+    //模式：single | per-field
+    //连接符：=, :, -> 等（输出格式）
+    //空值处理：跳过 | 输出 "null"
+    //public static final String CHUNK_TEMPLATE = "chunkTemplate";
+    public static final String CHUNK_MODE = "chunkMode";
+    public static final String CHUNK_MODE_SINGLE = "single";
+    public static final String CHUNK_MODE_MULTIPLE = "multiple";
+    public static final String CHUNK_MODE_CUSTOM = "custom";
+    public static final String CHUNK_TEMPLATES = "chunkTemplates";
 
+    public static final String INDEX_STRATEGY = "indexStrategy";
+    public static final String INDEX_MODE = "indexMode";
+    public static final String INDEX_MODE_T2C = "table2collection";  // 一个table 对应 一个milvus的collection
+    public static final String INDEX_MODE_SINGLE = "singleCollection";  // 所有table 都索引到一个collection
+
+    public static final String INDEX_COLLECTION_PREFIX = "collectionPrefix";
+    public static final String INDEX_GLOBAL_COLLECTION = "globalCollection";
 }

server/src/main/java/com/giantan/data/index/IndexUtils.java

@@ -81,4 +81,30 @@ public class IndexUtils {
         }
         return false;
     }
+
+
+    public static double cosineSimilarity(float[] a, float[] b) {
+        if (a == null || b == null) {
+            throw new IllegalArgumentException("Input vectors must not be null");
+        }
+        if (a.length != b.length) {
+            throw new IllegalArgumentException("Vectors must be of same length");
+        }
+
+        double dotProduct = 0.0;
+        double normA = 0.0;
+        double normB = 0.0;
+
+        for (int i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+
+        if (normA == 0.0 || normB == 0.0) {
+            return 0.0; // 避免除以0，表示至少一个向量是零向量
+        }
+
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
 }

server/src/main/java/com/giantan/data/mds/service/impl/Vectorization.java → server/src/main/java/com/giantan/data/index/Vectorization.java

@@ -1,7 +1,6 @@
-package com.giantan.data.mds.service.impl;
+package com.giantan.data.index;
 
 import com.giantan.ai.util.JsonUtil;
-import com.giantan.data.mds.service.IVectorization;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 

server/src/main/java/com/giantan/data/mds/MdsApplication.java

@@ -16,7 +16,7 @@ public class MdsApplication {
 
     public static void main(String[] args) {
         SpringApplication.run(MdsApplication.class, args);
-        log.info("Mds server started. Version 2.1.2");
+        log.info("Mds server started. Version 2.5.0");
     }
 
 }

server/src/main/java/com/giantan/data/mds/chunk/DynamicChunkRepository.java

@@ -445,4 +445,27 @@ public class DynamicChunkRepository {
         }
     }
 
+
+    public List<Map<String, Object>> getSectionPathByMdId(String collId, Integer mdId) {
+        String sql1 = "SELECT id,section_path FROM %s WHERE md_id = ?";
+        String sql = String.format(sql1, tableName(collId));
+        List<Map<String, Object>> paths = jdbc.query(
+                sql,
+                new Object[]{mdId},
+                (rs, rowNum) -> {
+                    Map<String, Object> map = new HashMap<>();
+                    long id = rs.getLong("id");
+                    map.put("id", id);
+                    String p = rs.getString("section_path");
+                    map.put("section_path", p);
+                    return map;
+                }
+        );
+        return paths;
+    }
+
+    public List<MdChunk> getChunksByMdId(String collId, Integer mdId) {
+        String sql = String.format("SELECT * FROM %s WHERE md_id = ? ORDER BY chunk_index", tableName(collId));
+        return jdbc.query(sql, new Object[]{mdId}, this::mapRow);
+    }
 }

server/src/main/java/com/giantan/data/mds/config/TaskConfiguration.java

@@ -4,41 +4,35 @@ import com.giantan.data.mds.bot.GChatClient;
 import com.giantan.data.index.IHybridSearch;
 import com.giantan.data.mds.service.IMdChunksService;
 import com.giantan.data.mds.service.IMdFilesService;
-import com.giantan.data.mds.service.IVectorization;
+import com.giantan.data.index.IVectorization;
 import com.giantan.data.qa.service.IQaDocsService;
-import com.giantan.data.qa.service.task.QasTaskHandler;
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
-
-import java.util.List;
-import java.util.concurrent.Executor;
 
 @Configuration
 class TaskConfiguration {
 
-    @Autowired
-    IMdFilesService mdFilesService;
-
-    @Autowired
-    IMdChunksService mdChunksService;
-
-    @Autowired
-    GChatClient gChatClient;
-
-    @Autowired
-    IVectorization vectorizationService;
-
-    @Autowired
-    IHybridSearch hybridSearch;
+//    @Autowired
+//    IMdFilesService mdFilesService;
+//
+//    @Autowired
+//    IMdChunksService mdChunksService;
+//
+//    @Autowired
+//    GChatClient gChatClient;
+//
+//    @Autowired
+//    IVectorization vectorizationService;
+//
+//    @Autowired
+//    IHybridSearch hybridSearch;
 
 //    @Autowired
 //    IPersistentTaskService persistentTaskService;
 
-    @Autowired
-    IQaDocsService qaDocsService;
+//    @Autowired
+//    IQaDocsService qaDocsService;
 
 //    @Bean
 //    public Executor taskExecutor() {
@@ -85,8 +79,8 @@ class TaskConfiguration {
 //        return new ChunksTaskHandler(mdChunksService,vectorizationService,hybridSearch,gChatClient);
 //    }
 
-    @Bean
-    public QasTaskHandler qasTaskHandler() {
-        return new QasTaskHandler(qaDocsService);
-    }
+//    @Bean
+//    public QasTaskHandler qasTaskHandler() {
+//        return new QasTaskHandler(qaDocsService);
+//    }
 }

server/src/main/java/com/giantan/data/mds/controller/ChunkController.java

@@ -7,6 +7,7 @@ import com.giantan.data.mds.service.impl.MdChunksService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -19,65 +20,45 @@ public class ChunkController {
 
     @GetMapping("/all")
     public R<List<MdChunk>> getAll(@PathVariable String coll) {
-        List<MdChunk> ret = null;
-        try {
-            ret = mdChunksService.findAll(coll);
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
+        List<MdChunk> ret = mdChunksService.findAll(coll);
         return R.data(ret);
     }
 
     @GetMapping("/by-md/{mdId}")
     public R<List<Map<String, Object>>> getKeywordsByMdId(@PathVariable String coll, @PathVariable Integer mdId) {
-        List<Map<String, Object>> ret = null;
-        try {
-            ret = mdChunksService.getKeywordsByMdId(coll, mdId);
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
+        List<Map<String, Object>> ret = mdChunksService.getKeywordsByMdId(coll, mdId);;
+        return R.data(ret);
+    }
+
+
+    @GetMapping("/by-md/{mdId}/all")
+    public R<List<MdChunk>> getChunksByMdId(@PathVariable String coll, @PathVariable Integer mdId) {
+        List<MdChunk> ret = mdChunksService.getChunksByMdId(coll, mdId);;
         return R.data(ret);
     }
 
     @GetMapping("/{id}")
     public R<MdChunk> getById(@PathVariable String coll, @PathVariable Long id) {
-        MdChunk ret = null;
-        try {
-            ret = mdChunksService.findById(coll, id);
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
+        MdChunk ret = mdChunksService.findById(coll, id);;
         return R.data(ret);
     }
 
     @DeleteMapping("/all")
-    public R<Map<String, Object>> deleteAll( @PathVariable String coll) throws Exception {
+    public R<Map<String, Object>> deleteAll(@PathVariable String coll) throws Exception {
         mdChunksService.deleteAll(coll);
-        return R.data(Map.of("deleted",true));
+        return R.data(Map.of("deleted", true));
     }
 
     @DeleteMapping("/{id}")
-    public R deleteByIds(@PathVariable String coll, @PathVariable List<String> ids) {
-        long ret = 0;
-        try {
-            ret = mdChunksService.deleteByIds(coll, ids);
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
+    public R deleteByIds(@PathVariable String coll, @PathVariable List<String> ids) throws IOException, InterruptedException {
+        long ret = mdChunksService.deleteByIds(coll, ids);;
         return R.data(ret);
     }
 
     @DeleteMapping("/by-md/{mdId}")
-    public R deleteByMdId(@PathVariable String coll, @PathVariable Integer id) {
-        long ret = 0;
-        try {
-            ret = mdChunksService.deleteByMdId(coll, id);
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
+    public R deleteByMdId(@PathVariable String coll, @PathVariable Integer mdId) throws IOException, InterruptedException {
+        long ret = mdChunksService.deleteByMdId(coll, mdId);
         return R.data(ret);
     }
 
-
-
 }

server/src/main/java/com/giantan/data/mds/controller/MdCollectionsController.java

@@ -20,9 +20,6 @@ public class MdCollectionsController {
     @Autowired
     MdCollectionsService mdCollectionsService;
 
-//    @Autowired
-//    MdDocsService mdDocsService;
-
     @PostMapping
     public ResponseEntity<R> createCollection(@RequestParam String name) throws Throwable {
         return ResponseEntity.ok(R.data(mdCollectionsService.createCollection(name)));

server/src/main/java/com/giantan/data/mds/controller/MdDocsController.java

@@ -67,21 +67,22 @@ public class MdDocsController {
     @Autowired
     IMdDocsService mdDocsService;
 
-    @PostMapping("/mds/batch")
-    public ResponseEntity<Map<String, String>> uploadZip(@PathVariable String collId,
-                                                         @RequestParam("file") MultipartFile file,
-                                                         @RequestParam Map<String, String> params
-    ) throws IOException {
-        String taskId = UUID.randomUUID().toString();
-        //System.out.println("taskId = " + taskId);
-        //System.out.println("file = " + file.getOriginalFilename());
-        log.info("上传 ZIP 批量文件: {}, taskId: {}", file.getOriginalFilename(), taskId);
-        taskStatusManager.putProcessing(taskId, new TaskStatus(collId, taskId, "处理中", "", System.currentTimeMillis(), 0));
-        fileProcessingService.uploadZip(collId, taskId, file, params);
-
-        log.info("taskId = " + taskId);
-        return ResponseEntity.ok(Map.of("taskId", taskId));
-    }
+    // 2025.9.8 这个功能暂时关掉
+//    @PostMapping("/mds/batch")
+//    public ResponseEntity<Map<String, String>> uploadZip(@PathVariable String collId,
+//                                                         @RequestParam("file") MultipartFile file,
+//                                                         @RequestParam Map<String, String> params
+//    ) throws IOException {
+//        String taskId = UUID.randomUUID().toString();
+//        //System.out.println("taskId = " + taskId);
+//        //System.out.println("file = " + file.getOriginalFilename());
+//        log.info("上传 ZIP 批量文件: {}, taskId: {}", file.getOriginalFilename(), taskId);
+//        taskStatusManager.putProcessing(taskId, new TaskStatus(collId, taskId, "处理中", "", System.currentTimeMillis(), 0));
+//        fileProcessingService.uploadZip(collId, taskId, file, params);
+//
+//        log.info("taskId = " + taskId);
+//        return ResponseEntity.ok(Map.of("taskId", taskId));
+//    }
 
     //@PostMapping("/upload")
     @PostMapping("/mds")
@@ -141,24 +142,38 @@ public class MdDocsController {
 
 
     @GetMapping("/upload-tasks/{taskId}")
-    public ResponseEntity<TaskStatus> getStatus(@PathVariable String collId, @PathVariable String taskId) {
+//    public ResponseEntity<TaskStatus> getStatus(@PathVariable String collId, @PathVariable String taskId) {
+//        TaskStatus status = taskStatusManager.get(taskId);
+//        if (status == null) {
+//            return ResponseEntity.status(HttpStatus.NOT_FOUND)
+//                    .body(new TaskStatus(collId, taskId, "未知", "任务不存在或已过期", 0, 0));
+//        }
+//        return ResponseEntity.ok(status);
+//    }
+
+    public R<TaskStatus> getStatus(@PathVariable String collId, @PathVariable String taskId) {
         TaskStatus status = taskStatusManager.get(taskId);
-        if (status == null) {
-            return ResponseEntity.status(HttpStatus.NOT_FOUND)
-                    .body(new TaskStatus(collId, taskId, "未知", "任务不存在或已过期", 0, 0));
-        }
-        return ResponseEntity.ok(status);
+        return R.data(status);
     }
 
     @DeleteMapping("/upload-tasks/{taskId}")
-    public ResponseEntity<Map<String, Object>> deleteStatus(@PathVariable String taskId) {
+//    public ResponseEntity<Map<String, Object>> deleteStatus(@PathVariable String taskId) {
+//        boolean removed = taskStatusManager.delete(taskId);
+//        return ResponseEntity.ok(Map.of(
+//                "taskId", taskId,
+//                "removed", removed
+//        ));
+//    }
+
+    public R<Map<String, Object>> deleteStatus(@PathVariable String taskId) {
         boolean removed = taskStatusManager.delete(taskId);
-        return ResponseEntity.ok(Map.of(
+        return R.data(Map.of(
                 "taskId", taskId,
                 "removed", removed
         ));
     }
 
+
     /////////////////////
 
     //    @PostMapping("/mds/locate")
@@ -273,4 +288,37 @@ public class MdDocsController {
         Map<String, Object> r = mdDocsService.rename(collId, mdId, req);
         return R.data(r);
     }
+
+    @DeleteMapping("/mds/{gid}/cascade")
+    public R<?> deleteCascade(@PathVariable String collId, @PathVariable String gid
+    ) throws Throwable {
+        //String taskId = UUID.randomUUID().toString();
+        //System.out.println("taskId = " + taskId);
+        //System.out.println("file = " + file.getOriginalFilename());
+
+        //taskStatusManager.putProcessing(taskId, new TaskStatus(collId, taskId, "处理中", "", System.currentTimeMillis(), 0));
+        //fileProcessingService.processAsyncDirect(collId, taskId, file, params);
+        //System.out.println("params = " + params);
+        int deleted = mdDocsService.deleteCascadeByMdid(collId, gid);
+        log.info("删除文件,mdid: {}", gid);
+        //log.info("taskId = " + taskId);
+        return R.data(Map.of("deleted", deleted));
+    }
+
+    @DeleteMapping("/mds/{gid}/indexes")
+    public R<?> deleteIndexes(@PathVariable String collId, @PathVariable String gid
+    ) throws Throwable {
+        //String taskId = UUID.randomUUID().toString();
+        //System.out.println("taskId = " + taskId);
+        //System.out.println("file = " + file.getOriginalFilename());
+
+        //taskStatusManager.putProcessing(taskId, new TaskStatus(collId, taskId, "处理中", "", System.currentTimeMillis(), 0));
+        //fileProcessingService.processAsyncDirect(collId, taskId, file, params);
+        //System.out.println("params = " + params);
+        int deleted = mdDocsService.deleteIndexesByMdid(collId, gid);
+        log.info("删除索引文件,mdid: {}", gid);
+        //log.info("taskId = " + taskId);
+        return R.data(Map.of("deleted", deleted));
+    }
+
 }

server/src/main/java/com/giantan/data/mds/controller/MdSearchContoller.java

@@ -1,10 +1,8 @@
 package com.giantan.data.mds.controller;
 
 import com.giantan.data.mds.constant.MdConstants;
+import com.giantan.data.mds.repository.MdIndexer;
 import com.giantan.data.mds.service.IMdDocsService;
-import com.giantan.data.mds.service.impl.MdDocsService;
-import com.giantan.data.qa.constant.QaConstants;
-import com.giantan.data.qa.service.QaDocsService;
 import jakarta.servlet.http.HttpServletRequest;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -30,8 +28,11 @@ public class MdSearchContoller {
     @Value("${qas.url}")
     String url = "http://120.78.4.46:7387/v1/collections/";
 
+    //@Autowired
+    //IMdDocsService mdDocsService;
+
     @Autowired
-    IMdDocsService mdDocsService;
+    MdIndexer indexer;
 
     private final RestTemplate restTemplate = new RestTemplate();
 
@@ -50,7 +51,8 @@ public class MdSearchContoller {
         targetPath = targetPath.replaceFirst("/"+coll,"");
         targetPath = targetPath.replaceFirst("/indexes", "/documents");
 
-        String collName = mdDocsService.getIndexName(coll);
+        //String collName = mdDocsService.getIndexName(coll);
+        String collName = indexer.getMappedIndexNameByName(coll);
 
         String targetUrl = url + collName + targetPath + (query != null ? "?" + query : "");
 

server/src/main/java/com/giantan/data/mds/controller/StatusController.java

@@ -27,12 +27,12 @@ public class StatusController {
      */
     @PostConstruct
     public void init() {
-        try (var conn = dataSource.getConnection()) {
-            // 触发 HikariDataSource 初始化
-            log.info("HikariDataSource initialized at startup");
-        } catch (Exception e) {
-            throw new RuntimeException("Failed to initialize HikariDataSource", e);
-        }
+//        try (var conn = dataSource.getConnection()) {
+//            // 触发 HikariDataSource 初始化
+//            log.info("HikariDataSource initialized at startup");
+//        } catch (Exception e) {
+//            throw new RuntimeException("Failed to initialize HikariDataSource", e);
+//        }
     }
 
     @GetMapping("/hikari/status")

server/src/main/java/com/giantan/data/mds/controller/TaskController.java

@@ -1,5 +1,6 @@
 package com.giantan.data.mds.controller;
 
+import com.giantan.ai.common.reponse.R;
 import com.giantan.data.mds.constant.MdConstants;
 import com.giantan.data.mds.service.impl.MdTaskManager;
 import com.giantan.data.tasks.*;
@@ -34,7 +35,7 @@ public class TaskController {
      */
 
     @PostMapping("/submit")
-    public Map submit(@PathVariable String collId, @RequestBody Map<String, Object> payload) {
+    public R<Map> submit(@PathVariable String collId, @RequestBody Map<String, Object> payload) {
         String t = (String) payload.remove("type");
 
         TaskType type = TaskType.valueOf(t);
@@ -58,52 +59,59 @@ public class TaskController {
         Map<String, Object> params = new HashMap<>(payload);
 
         String ret = manager.submit(collId, type, params);
-        return Map.of("taskId", ret);
+        return R.data(Map.of("taskId", ret));
     }
 
     @PostMapping("/{id}/cancel")
-    public Map cancel(@PathVariable String collId, @PathVariable String id) {
+    public R<Map> cancel(@PathVariable String collId, @PathVariable String id) {
         boolean ok = manager.cancel(collId, id);
-        return Map.of("canceled", ok);
+        return R.data(Map.of("canceled", ok));
     }
 
     @DeleteMapping("/{id}")
-    public Map delete(@PathVariable String collId, @PathVariable String id) {
+    public R<Map> delete(@PathVariable String collId, @PathVariable String id) {
         boolean ok = manager.delete(collId, id);
-        return Map.of("deleted", ok);
+        return R.data(Map.of("deleted", ok));
     }
 
     @GetMapping("/{id}/status")
-    public Map<String, TaskOperationsStatus> status(@PathVariable String collId, @PathVariable String id) {
+    public R<Map> status(@PathVariable String collId, @PathVariable String id) {
         TaskContext ctx = manager.getTask(collId, id);
-        return ctx != null ? ctx.getObjectStatus() : Collections.emptyMap();
+        Map ret = ctx != null ? ctx.getObjectStatus() : Collections.emptyMap();
+        return R.data(ret);
     }
 
     @GetMapping("/{id}")
-    public TaskContext getTask(@PathVariable String collId, @PathVariable String id) {
-        return manager.getTask(collId, id);
+    public R<TaskContext> getTask(@PathVariable String collId, @PathVariable String id) {
+        TaskContext task = manager.getTask(collId, id);
+        return R.data(task);
     }
 
     @DeleteMapping("/cleanup")
-    public Map cleanup(@PathVariable String collId) {
+    public R<Map> cleanup(@PathVariable String collId) {
         int r = manager.cleanupNow(collId);
         //return ResponseEntity.ok("Task cleanup triggered.");
-        return Map.of("deleted", r);
+        return R.data(Map.of("deleted", r));
     }
 
     @GetMapping
-    public Collection<TaskContext> listAllTasks(@PathVariable String collId) {
-        return manager.allTasks(collId);
+    public R<Collection<TaskContext>> listAllTasks(@PathVariable String collId) {
+        return R.data(manager.allTasks(collId));
     }
 
     @GetMapping("/status/{status}")
-    public Collection<TaskContext> listTasksByStatus(@PathVariable String collId, @PathVariable String status) {
+    public R<Collection<TaskContext>> listTasksByStatus(@PathVariable String collId, @PathVariable String status) {
         //TaskStatus statusEnum = TaskStatus.valueOf(status.toUpperCase());
-        return manager.findByStatus(collId, status);
+        return R.data(manager.findByStatus(collId, status));
+    }
+
+    @GetMapping("/{id}/history")
+    public R<TaskStatusHistory> getHistoryTask(@PathVariable String collId, @PathVariable String id){
+        return R.data(manager.getHistoryTask(collId, id));
     }
 
     @GetMapping("/history")
-    public List<TaskStatusHistory> getTasks(@PathVariable String collId,
+    public R<List<TaskStatusHistory>> getTasks(@PathVariable String collId,
                                             @RequestParam(value = "createdAtStart", required = false) String createdAtStart,
                                             @RequestParam(value = "createdAtEnd", required = false) String createdAtEnd,
                                             @RequestParam(value = "status", required = false) String status
@@ -117,11 +125,11 @@ public class TaskController {
         if (createdAtEnd != null) {
             endTime = LocalDateTime.parse(createdAtEnd);
         }
-        return manager.getHistoryTasks(collId, startTime, endTime, status);
+        return R.data(manager.getHistoryTasks(collId, startTime, endTime, status));
     }
 
     @DeleteMapping("/history/cleanup")
-    public int deleteHistory(@PathVariable String collId,
+    public R deleteHistory(@PathVariable String collId,
                              @RequestParam(value = "createdAtStart", required = false) String createdAtStart,
                              @RequestParam(value = "createdAtEnd", required = false) String createdAtEnd,
                              @RequestParam(value = "status", required = false) String status
@@ -135,7 +143,7 @@ public class TaskController {
         if (createdAtEnd != null) {
             endTime = LocalDateTime.parse(createdAtEnd);
         }
-        return manager.deleteHistoryTasks(collId, startTime, endTime, status);
+        return R.data(manager.deleteHistoryTasks(collId, startTime, endTime, status));
     }
 
 }

server/src/main/java/com/giantan/data/mds/controller/TaxonomyController.java

@@ -6,7 +6,6 @@ import com.giantan.data.mds.constant.MdConstants;
 import com.giantan.data.mds.service.impl.MdTaxonomyService;
 import com.giantan.data.taxonomy.model.TaxonomyNode;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
@@ -25,35 +24,59 @@ public class TaxonomyController {
     private MdTaxonomyService dynamicTaxonomyService;
 
     @GetMapping("/all")
-    public ResponseEntity<?> listAll(@PathVariable String collName) {
+//    public ResponseEntity<?> listAll(@PathVariable String collName) {
+//        List<TaxonomyNode> r = dynamicTaxonomyService.listAll(collName);
+//        return ResponseEntity.ok(r);
+//    }
+    public R listAll(@PathVariable String collName) {
         List<TaxonomyNode> r = dynamicTaxonomyService.listAll(collName);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
     @GetMapping("/tree")
-    public ResponseEntity<?> listTree(@PathVariable String collName) {
+//    public ResponseEntity<?> listTree(@PathVariable String collName) {
+//        List<TaxonomyNode> r = dynamicTaxonomyService.listTree(collName);
+//        return ResponseEntity.ok(r);
+//    }
+    public R listTree(@PathVariable String collName) {
         List<TaxonomyNode> r = dynamicTaxonomyService.listTree(collName);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
     @GetMapping("/{nodeId}/children")
-    public ResponseEntity<?> findChildren(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
+//    public ResponseEntity<?> findChildren(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
+//        List<TaxonomyNode> r = dynamicTaxonomyService.findChildren(collName, nodeId);
+//        return ResponseEntity.ok(r);
+//    }
+    public R findChildren(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
         List<TaxonomyNode> r = dynamicTaxonomyService.findChildren(collName, nodeId);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
     @GetMapping("/{nodeId}/mds")
-    public ResponseEntity<?> findMds(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
+//    public ResponseEntity<?> findMds(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
+//        List<GBaseKeyValue> r = dynamicTaxonomyService.findMds(collName, nodeId);
+//        return ResponseEntity.ok(r);
+//    }
+    public R findMds(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
         List<GBaseKeyValue> r = dynamicTaxonomyService.findMds(collName, nodeId);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
     @PostMapping()
-    public ResponseEntity<?> createNode(@PathVariable String collName, @RequestBody Map<String, Object> data) throws Exception {
+//    public ResponseEntity<?> createNode(@PathVariable String collName, @RequestBody Map<String, Object> data) throws Exception {
+//        TaxonomyNode r = dynamicTaxonomyService.createNode(collName, data);
+//        return ResponseEntity.ok(r);
+//    }
+    public R createNode(@PathVariable String collName, @RequestBody Map<String, Object> data) throws Exception {
         TaxonomyNode r = dynamicTaxonomyService.createNode(collName, data);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
     @PostMapping("/{nodeId}/mds")
     public R<Map<String, Object>> upload(@PathVariable String collName,
                                          @PathVariable String nodeId,

server/src/main/java/com/giantan/data/mds/repository/MdDynamicRepository.java

@@ -14,6 +14,9 @@ public class MdDynamicRepository extends GDynamicRepository {
     @Autowired
     private JdbcTemplate jdbc;
 
+    @Autowired
+    MdIndexer indexer;
+
     private String indexPrefix;
 
     public MdDynamicRepository(){
@@ -31,6 +34,7 @@ public class MdDynamicRepository extends GDynamicRepository {
     public String indexName(String coll) {
         //return schema + "_" + indexPrefix + "_" + collId;
         //return this.indexPrefix + "_" + coll;
-        return coll;
+        //return coll;
+        return indexer.getMappedIndexNameByName(coll);
     }
 }

server/src/main/java/com/giantan/data/mds/repository/MdIndexer.java

@@ -0,0 +1,146 @@
+package com.giantan.data.mds.repository;
+
+import com.giantan.data.index.HybridIndexer;
+import com.giantan.data.index.IHybridSearch;
+import com.giantan.data.index.dto.DocReq;
+import com.giantan.data.index.dto.DocResp;
+import com.giantan.data.mds.service.impl.MdCollectionsService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Repository
+public class MdIndexer extends HybridIndexer {
+    @Autowired
+    IHybridSearch hybridSearch;
+
+    MdCollectionsService collectionService;
+
+    //String collectionPrefix = "";
+
+    public MdIndexer() {
+
+    }
+
+    public void init(MdCollectionsService collectionService) {
+        defaultIndexPrefix = "mds";
+        this.collectionService = collectionService;
+    }
+
+//    public String indexName(String coll) {
+//        //return schema + "_" + indexPrefix + "_" + collId;
+//        //return this.indexPrefix + "_" + coll;
+//        return coll;
+//    }
+
+    public String indexName(String coll) {
+        return defaultIndexPrefix + "_" + coll;
+    }
+
+//    public String indexName(String prefix, String coll) {
+//        return prefix + coll;
+//    }
+
+    public String getMappedIndexNameById(String collId) {
+        String collName = collectionService.getCollectionName(Integer.parseInt(collId));
+        String mapped = indexName(collName);
+        Map<String, Object> attibutes = collectionService.getAttibutes(collId);
+        String configIndexName = getConfigIndexName(collId, attibutes);
+        if (configIndexName != null) {
+            mapped = configIndexName;
+        }
+        return mapped;
+    }
+
+    public String getMappedIndexNameByName(String coll) {
+        String mapped = indexName(coll);
+        int collId1 = collectionService.getCollectionId(coll);
+        String collId = Integer.toString(collId1);
+        Map<String, Object> attibutes = collectionService.getAttibutes(collId);
+        String configIndexName = getConfigIndexName(collId, attibutes);
+        if (configIndexName != null) {
+            mapped = configIndexName;
+        }
+
+        return mapped;
+    }
+
+    public boolean isOne2One(String collId) {
+        boolean isOne2One = true;
+        Map<String, Object> attibutes = collectionService.getAttibutes(collId);
+
+        Boolean configOne2One = getConfigOne2One(attibutes);
+        if (configOne2One != null) {
+            isOne2One = configOne2One;
+        }
+
+        return isOne2One;
+    }
+
+
+    public Map<String, Object> appendMetadata(String collection, int docId) {
+        int collId = collectionService.getCollectionId(collection);
+        Map<String, Object> metadata = new HashMap<>();
+        metadata.put(DOC_ID, docId);
+        metadata.put(COLL_ID, collId);
+        return metadata;
+    }
+
+    public int deleteCollection(String collId) {
+        boolean b = false;
+        try {
+            if (isOne2One(collId)) {
+                b = hybridSearch.dropCollection(getMappedIndexNameById(collId));
+            } else {
+                //TODO 特殊处理
+                //  "filterExpression": "metadata[\"__cid\"] == \"qas_2\""
+                Map<String, Object> filterMap = Map.of("filterExpression", "metadata[\"__cid\"] == \"" + collId + "\"");
+                int r = hybridSearch.deleteDocumentsByFilter(getMappedIndexNameById(collId), filterMap);
+                return r;
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return b ? 0 : 1;
+    }
+
+    public int deleteAllDocs(String collId) {
+        boolean b = false;
+        try {
+            if (isOne2One(collId)) {
+                b = hybridSearch.clearCollection(getMappedIndexNameById(collId));
+            } else {
+                //TODO 特殊处理
+                //  "filterExpression": "metadata[\"__cid\"] == \"qas_2\""
+                Map<String, Object> filterMap = Map.of("filterExpression", "metadata[\"__cid\"] == \"" + collId + "\"");
+                int r = hybridSearch.deleteDocumentsByFilter(getMappedIndexNameById(collId), filterMap);
+                return r;
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return 0;
+    }
+
+
+    public int createCollection(String coll) throws IOException, InterruptedException {
+        String indexName = getMappedIndexNameByName(coll);
+        int r = hybridSearch.createCollection(indexName);
+        return r;
+    }
+
+    public void delete(String coll, List<String> uids) throws IOException, InterruptedException {
+        String indexName = getMappedIndexNameByName(coll);
+        hybridSearch.delete(indexName, uids);
+    }
+
+    public List<DocResp> addDirect(String coll, List<DocReq> docs) throws IOException, InterruptedException {
+        String indexName = getMappedIndexNameByName(coll);
+        List<DocResp> ret = hybridSearch.addDirect(indexName, docs);
+        return ret;
+    }
+}

server/src/main/java/com/giantan/data/mds/service/IMdChunksService.java

@@ -18,6 +18,8 @@ public interface IMdChunksService {
 
     List<Map<String, Object>> getKeywordsByMdId(String coll, Integer mdId);
 
+    List<Map<String, Object>> getSectionPathByMdId(String coll, Integer mdId);
+
     int updateKeywordsOrMetadata(String coll, Long id, List<String> keywords, Map<String, Object> metadata);
 
     List<MdChunk> findAll(String coll);
@@ -30,7 +32,11 @@ public interface IMdChunksService {
 
     long deleteByIds(String coll, List<String> ids) throws IOException, InterruptedException;
 
-    String getIndexName(String coll);
+    //String getIndexName(String coll);
+
+    String findSectionPathById(String coll, Long chunkId);
+
+    List<MdChunk> getChunksByMdId(String coll, Integer mdId);
 
     //String getEmbedding(String coll, Long id);
 

server/src/main/java/com/giantan/data/mds/service/IMdDocsService.java

@@ -3,6 +3,7 @@ package com.giantan.data.mds.service;
 import com.giantan.data.kvs.kvstore.GBaseKeyValue;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -42,10 +43,13 @@ public interface IMdDocsService {
 
     Map<String, Object> rename(String coll, String mdId, Map<String, Object> req) throws Throwable;
 
-    String getIndexName(String coll);
-
+    //String getIndexName(String coll);
 
     Map<String, Object> locateText(String coll, Map<String, Object> req) throws Throwable;
 
-    Map<String, Object> locateByHeading(String collId, Map<String, Object> req) throws Throwable;
+    Map<String, Object> locateByHeading(String coll, Map<String, Object> req) throws Throwable;
+
+    int deleteCascadeByMdid(String coll, String gid) throws Throwable;
+
+    int deleteIndexesByMdid(String coll, String gid) throws Throwable;
 }

server/src/main/java/com/giantan/data/mds/service/impl/MdChunksService.java

@@ -1,9 +1,8 @@
 package com.giantan.data.mds.service.impl;
 
-import com.giantan.data.index.HybridSearch;
-import com.giantan.data.index.IHybridSearch;
 import com.giantan.data.mds.chunk.MdChunk;
 import com.giantan.data.mds.repository.MdDynamicChunkRepository;
+import com.giantan.data.mds.repository.MdIndexer;
 import com.giantan.data.mds.service.IMdChunksService;
 import jakarta.annotation.PostConstruct;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -23,8 +22,11 @@ public class MdChunksService implements IMdChunksService {
     @Autowired
     MdDynamicChunkRepository mdDynamicChunkRepository;
 
+//    @Autowired
+//    IHybridSearch hybridSearch;
+
     @Autowired
-    IHybridSearch hybridSearch;
+    MdIndexer mdIndexer;
 
     boolean isIndexDeleted = true;
 
@@ -42,7 +44,8 @@ public class MdChunksService implements IMdChunksService {
         if (isIndexDeleted) {
             List<String> uids = mdDynamicChunkRepository.findUidsByMdId(Integer.toString(collId), mdId);
             if (uids != null && !uids.isEmpty()) {
-                hybridSearch.delete(coll, uids);
+                //hybridSearch.delete(coll, uids);
+                mdIndexer.delete(coll,uids);
             }
         }
         long r = mdDynamicChunkRepository.deleteByMdId(Integer.toString(collId), mdId);
@@ -69,6 +72,12 @@ public class MdChunksService implements IMdChunksService {
         return mdDynamicChunkRepository.getKeywordsByMdId(Integer.toString(collId), mdId);
     }
 
+    @Override
+    public List<Map<String, Object>> getSectionPathByMdId(String coll, Integer mdId) {
+        int collId = mdCollectionsService.getCollectionId(coll);
+        return mdDynamicChunkRepository.getSectionPathByMdId(Integer.toString(collId), mdId);
+    }
+
     @Override
     public int updateKeywordsOrMetadata(String coll, Long id, List<String> keywords, Map<String, Object> metadata) {
         int collId = mdCollectionsService.getCollectionId(coll);
@@ -97,7 +106,8 @@ public class MdChunksService implements IMdChunksService {
     public void deleteAll(String coll) throws IOException, InterruptedException {
         int collId = mdCollectionsService.getCollectionId(coll);
         if (isIndexDeleted) {
-            hybridSearch.deleteAll(coll);
+            //hybridSearch.deleteAll(coll);
+            mdIndexer.deleteAllDocs(coll);
         }
         mdDynamicChunkRepository.deleteAll(Integer.toString(collId));
     }
@@ -109,7 +119,8 @@ public class MdChunksService implements IMdChunksService {
         if (isIndexDeleted) {
             uids = mdDynamicChunkRepository.findUidsByIds(Integer.toString(collId), toListLong(ids));
             if (uids != null && !uids.isEmpty()) {
-                hybridSearch.delete(coll, uids);
+                //hybridSearch.delete(coll, uids);
+                mdIndexer.delete(coll, uids);
             }
         }
         int r = 0;
@@ -119,9 +130,21 @@ public class MdChunksService implements IMdChunksService {
         return r;
     }
 
+//    @Override
+//    public String getIndexName(String coll) {
+//        return mdCollectionsService.getIndexName(coll);
+//    }
+
+    @Override
+    public String findSectionPathById(String coll, Long chunkId) {
+        int collId = mdCollectionsService.getCollectionId(coll);
+        return mdDynamicChunkRepository.findUidById(Integer.toString(collId),chunkId);
+    }
+
     @Override
-    public String getIndexName(String coll) {
-        return mdCollectionsService.getIndexName(coll);
+    public List<MdChunk> getChunksByMdId(String coll, Integer mdId) {
+        int collId = mdCollectionsService.getCollectionId(coll);
+        return mdDynamicChunkRepository.getChunksByMdId(Integer.toString(collId),mdId);
     }
 
     private List<Long> toListLong(List<String> ss) {

server/src/main/java/com/giantan/data/mds/service/impl/MdCollectionsService.java

@@ -3,9 +3,7 @@ package com.giantan.data.mds.service.impl;
 import com.giantan.data.kvs.kvstore.GBaseKeyValue;
 import com.giantan.data.kvs.repository.GEntityConfig;
 import com.giantan.data.kvs.repository.GRepository;
-//import com.giantan.data.kvs.service.ICollection;
 import com.giantan.data.mds.repository.*;
-//import com.giantan.data.taxonomy.repository.DynamicTaxonomyRepository;
 import com.giantan.data.mds.service.CollectionInstance;
 import com.giantan.gfs.service.impl.S3GkbService;
 import jakarta.annotation.PostConstruct;
@@ -13,6 +11,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
 import org.springframework.web.server.ResponseStatusException;
 
 import java.util.List;
@@ -54,6 +53,8 @@ public class MdCollectionsService {   //extends KvCollectionService
     @Autowired
     MdDynamicTaskRepository taskRepository;
 
+    @Autowired
+    MdIndexer indexer;
 
     public MdCollectionsService() {
 
@@ -62,6 +63,7 @@ public class MdCollectionsService {   //extends KvCollectionService
     @PostConstruct
     public void init() {
         collections = new GRepository(GKG_CATALOG, GKG_ENTRY_COLLECTIONS, jdbcTemplate);
+        indexer.init(this);
     }
 
 
@@ -86,20 +88,36 @@ public class MdCollectionsService {   //extends KvCollectionService
         } else {
             try {
                 List<GBaseKeyValue> r = collections.findByName(name);
-                if (r != null && r.size() > 0) {
+                if (!CollectionUtils.isEmpty(r)) {
                     GBaseKeyValue kv = r.get(0);
-                    instance = CollectionInstance.build(kv.getIntId(), kv.getGid(), kv.getName(),kv.get(GEntityConfig.ATTRIBUTES));
+                    instance = CollectionInstance.build(kv.getIntId(), kv.getGid(), kv.getName(), kv.get(GEntityConfig.ATTRIBUTES));
                     mdCores.put(name, instance);
                     return instance.getId();
-                }else{
+                } else {
                     log.warn("Collection with name '" + name + "' not exist.");
+                    throw new ResponseStatusException(HttpStatus.NOT_FOUND,
+                            "Collection with name '" + name + "' does not exist.");
                 }
+            } catch (ResponseStatusException e) {
+                // 直接透传，不要覆盖
+                throw e;
             } catch (Throwable e) {
+                log.error("Failed to get collection by name={}", name, e);
+                throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
+                        "Error occurred while retrieving collection with name '" + name + "'", e);
 
             }
-            throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
-                    "Collection with name '" + name + "' not exist.");
+//            throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
+//                    "Collection with name '" + name + "' not exist.");
+        }
+    }
+
+    public String getCollectionName(int id) {
+        CollectionInstance instance = mdCores.getById(id);
+        if (instance != null) {
+            return instance.getName();
         }
+        return null;
     }
 
     public int getCollectionOrNew(String name) {
@@ -114,7 +132,7 @@ public class MdCollectionsService {   //extends KvCollectionService
                 } else {
                     r = keys.get(0);
                 }
-                CollectionInstance instance = CollectionInstance.build(r.getIntId(), r.getGid(), r.getName(),r.get(GEntityConfig.ATTRIBUTES));
+                CollectionInstance instance = CollectionInstance.build(r.getIntId(), r.getGid(), r.getName(), r.get(GEntityConfig.ATTRIBUTES));
                 mdCores.put(name, instance);
                 return instance.getId();
             } catch (Throwable e) {
@@ -174,50 +192,94 @@ public class MdCollectionsService {   //extends KvCollectionService
         return false;
     }
 
+    private void onCollectionUpdate(String name, GBaseKeyValue entity) {
+        if (entity != null) {
+            CollectionInstance info = mdCores.getByName(name);
+            info.updateAttributes(entity.get(GEntityConfig.ATTRIBUTES));
+        }
+    }
+
+
     //@Override
     public GBaseKeyValue updateCollection(String name, Map<String, Object> entity) throws Throwable {
-        GBaseKeyValue kv = getKvByName(name);
-        if (kv == null) {
-            return null;
-        }
+//        GBaseKeyValue kv = getKvByName(name);
+//        if (kv == null) {
+//            return null;
+//        }
+        int id = getCollectionId(name);
         GBaseKeyValue gkv = new GBaseKeyValue(entity);
-        gkv.put("id", kv.getIntId());
+        gkv.put("id", id);
         GBaseKeyValue updated = collections.update(gkv);
+        onCollectionUpdate(name, updated);
         return updated;
     }
 
-    public Map<String, Object> getCollectionAttributes(String name) throws Throwable {
-        GBaseKeyValue ret = getKvByName(name);
-        if (ret == null) {
+    //@Override
+    public Map<String, Object> getAttibutes(String collId) {
+        CollectionInstance instance = mdCores.getById(Integer.parseInt(collId));
+        if (instance == null) {
             return null;
         }
-        return (Map<String, Object>) ret.get(GEntityConfig.ATTRIBUTES);
+        return instance.getAttributes();
     }
 
-    public Map<String, Object> updateCollectionAttributes(String name, Map<String, Object> attributes) throws Throwable {
-        GBaseKeyValue ret = getKvByName(name);
-        if (ret == null) {
+//    @Override
+    public Map<String, Object> getAttributesByName(String name) {
+        int collId = getCollectionId(name);
+        CollectionInstance instance = mdCores.getById(collId);
+        //CollectionInstance instance = mdCores.getByName(name);
+        if (instance == null) {
             return null;
         }
-        int id = ret.getIntId();
-        attributes.forEach((k, v) -> {
-            try {
-                collections.updateAttribute(id, k, v);
-            } catch (Throwable e) {
-                throw new RuntimeException(e);
-            }
-        });
-        //return getCollectionAttributes(id);
-        return collections.find(id);
+        return instance.getAttributes();
     }
 
-    public GBaseKeyValue removeCollectionAttribute(String name, List<String> keys) throws Throwable {
-        GBaseKeyValue kv = getKvByName(name);
-        if (kv == null) {
+
+    public Map<String, Object> getCollectionAttributes(String name) throws Throwable {
+        GBaseKeyValue ret = getKvByName(name);
+        if (ret == null) {
             return null;
         }
-        GBaseKeyValue ret = collections.removeAttribute(kv.getIntId(), keys);
-        return ret;
+       Object o = ret.get(GEntityConfig.ATTRIBUTES);
+        if (o == null) {
+            return null;
+        }
+        return (Map<String, Object>) o;
+    }
+
+    public Map<String, Object> updateCollectionAttributes(String name, Map<String, Object> attributes) throws Throwable {
+//        GBaseKeyValue ret = getKvByName(name);
+//        if (ret == null) {
+//            return null;
+//        }
+//        int id = ret.getIntId();
+//        attributes.forEach((k, v) -> {
+//            try {
+//                collections.updateAttribute(id, k, v);
+//            } catch (Throwable e) {
+//                throw new RuntimeException(e);
+//            }
+//        });
+//
+//        GBaseKeyValue ret2 = collections.find(id);
+//        onCollectionUpdate(name, ret2);
+//        return ret2;
+
+        int id = getCollectionId(name);
+        GBaseKeyValue ret = collections.updateAttribute(id, attributes);
+        onCollectionUpdate(name, ret);
+        return (Map<String, Object>) ret.get(GEntityConfig.ATTRIBUTES);
+    }
+
+    public Map<String, Object> removeCollectionAttribute(String name, List<String> keys) throws Throwable {
+//        GBaseKeyValue kv = getKvByName(name);
+//        if (kv == null) {
+//            return null;
+//        }
+        int id = getCollectionId(name);
+        GBaseKeyValue ret = collections.removeAttribute(id, keys);
+        onCollectionUpdate(name, ret);
+        return (Map<String, Object>) ret.get(GEntityConfig.ATTRIBUTES);
     }
 
     private void removeFromStore(String name) {
@@ -259,6 +321,11 @@ public class MdCollectionsService {   //extends KvCollectionService
 
         removeFromStore(name);
 
+        try {
+            indexer.deleteCollection(id);
+        } catch (Exception e) {
+            log.error(e.getMessage());
+        }
         //deleteEntryTable2(KvConfig.ENTRY_SCHEMA, getTableName(intId.toString()));
         try {
             docsRepository.deleteTable(id);
@@ -298,6 +365,13 @@ public class MdCollectionsService {   //extends KvCollectionService
         GBaseKeyValue kv = getKvByName(name);
         //Integer intId = kv.getIntId();
         String id = kv.getId();
+
+        try {
+            indexer.deleteAllDocs(id);
+        } catch (Exception e) {
+            log.error(e.getMessage());
+        }
+
         try {
             docsRepository.deleteAll(id);
         } catch (Exception e) {
@@ -353,17 +427,18 @@ public class MdCollectionsService {   //extends KvCollectionService
 
         //createEntryTable(KvConfig.ENTRY_SCHEMA, getTableName(id));
         docsRepository.createTable(id);
-
         taxonomyRepository.createTable(id);
         chunksRepository.createTable(id);
         extraRepository.createTable(id);
         taskRepository.createTable(id);
+        indexer.createCollection(id);
         return ret;
     }
 
-    public String getIndexName(String coll) {
-        return docsRepository.indexName(coll);
-    }
+//    public String getIndexName(String coll) {
+//        //return docsRepository.indexName(coll);
+//        return indexer.getMappedIndexNameByName(coll);
+//    }
 
 //    public int deleteEntryTable2(String schema, String tableName) {
 //        List<String> sqls = GEntityConfig.deleteGEntityTable(schema, tableName);

server/src/main/java/com/giantan/data/mds/service/impl/MdDocsService.java

@@ -1,7 +1,9 @@
 package com.giantan.data.mds.service.impl;
 
 import com.giantan.data.kvs.kvstore.GBaseKeyValue;
+import com.giantan.data.mds.repository.MdDynamicChunkRepository;
 import com.giantan.data.mds.repository.MdDynamicRepository;
+import com.giantan.data.mds.repository.MdIndexer;
 import com.giantan.data.mds.service.IMdDocsService;
 import com.giantan.gfs.service.impl.S3GkbService;
 import com.giantan.gfs.storer.util.FileUtil;
@@ -14,7 +16,9 @@ import org.springframework.web.multipart.MultipartFile;
 
 import org.apache.commons.io.IOUtils;
 
+import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -52,6 +56,15 @@ public class MdDocsService implements IMdDocsService {
     @Autowired
     MdDynamicRepository mdDynamicRepository;
 
+    @Autowired
+    MdDynamicChunkRepository mdDynamicChunkRepository;
+
+    //@Autowired
+    //IHybridSearch hybridSearch;
+
+    @Autowired
+    MdIndexer mdIndexer;
+
     @Autowired
     MdCache mdCache;
 
@@ -71,15 +84,15 @@ public class MdDocsService implements IMdDocsService {
         return Integer.toString(id);
     }
 
-    protected String normalizePath(String path){
-        if (path == null || path.length() == 0){
+    protected String normalizePath(String path) {
+        if (path == null || path.length() == 0) {
             return "/";
         }
-        path = path.replaceAll("\\\\","/");
-        if (path.startsWith("/")){
+        path = path.replaceAll("\\\\", "/");
+        if (path.startsWith("/")) {
             return path;
-        }else {
-            return "/"+path;
+        } else {
+            return "/" + path;
         }
     }
 
@@ -98,12 +111,12 @@ public class MdDocsService implements IMdDocsService {
             } else {
                 objectPath = p;
                 int i1 = p.lastIndexOf("/");
-                p = p.substring(0,i1+1);
+                p = p.substring(0, i1 + 1);
             }
             int collId = mdCollectionsService.getCollectionOrNew(coll);
 
-            params.put("path",p);
-            params.put("name",objectPath);
+            params.put("path", p);
+            params.put("name", objectPath);
             //TaskStatus status = new TaskStatus(coll, taskId, "成功", "处理完成", System.currentTimeMillis(), System.currentTimeMillis());
             //taskStatusManager.markSuccess(taskId, status);
             ret.put("taskId", taskId);
@@ -176,7 +189,7 @@ public class MdDocsService implements IMdDocsService {
     public int deleteByMdid(String coll, String gid) throws Throwable {
         GBaseKeyValue kv = findByMdid(coll, gid);
         if (kv == null) {
-            log.error("该 {} 不存在", gid);
+            log.error("该 id: {} 不存在", gid);
             return -1;
         }
         return doDelete(coll, kv);
@@ -198,7 +211,7 @@ public class MdDocsService implements IMdDocsService {
     public int deleteByPathPrefix(String coll, String prefix) throws Throwable {
         String collId = getStrOfCollId(coll);
         List<GBaseKeyValue> kvs = mdDynamicRepository.findByPath(collId, prefix);
-        if (kvs == null || kvs.size()<=0){
+        if (kvs == null || kvs.size() <= 0) {
             return 0;
         }
         int count = mdDynamicRepository.deleteByPrefix(collId, prefix);
@@ -229,7 +242,6 @@ public class MdDocsService implements IMdDocsService {
     }
 
 
-
     private String getKvObjectName(String coll, Map<String, Object> req) throws Throwable {
         String collId = getStrOfCollId(coll);
         Object o = req.get(FIELD_NAME);
@@ -281,7 +293,7 @@ public class MdDocsService implements IMdDocsService {
         String collId = getStrOfCollId(coll);
 
         String mdid = getKvObjectId(collId, req);
-        MdSearcher searcher = getSearcher2(coll,collId, mdid);
+        MdSearcher searcher = getSearcher2(coll, collId, mdid);
 
         Object o1 = req.get(QUERY_TEXT);
         Map<String, Object> map = searcher.searchAndHeadings((String) o1);
@@ -291,7 +303,7 @@ public class MdDocsService implements IMdDocsService {
         return map;
     }
 
-    private MdSearcher getSearcher2(String coll, String collId,String id) throws Throwable {
+    private MdSearcher getSearcher2(String coll, String collId, String id) throws Throwable {
         String key = coll + ":" + id;
         MdSearcher searcher = mdCache.get(key);
         if (searcher == null) {
@@ -332,9 +344,9 @@ public class MdDocsService implements IMdDocsService {
             id = (String) o;
         } else {
             o = req.get(FIELD_MDID2);
-            if (o!=null){
+            if (o != null) {
                 id = (String) o;
-            }else {
+            } else {
                 o = req.get(FIELD_GID);
                 String gid = null;
                 if (o != null) {
@@ -369,7 +381,6 @@ public class MdDocsService implements IMdDocsService {
 //    }
 
 
-
     private String getMdFileContentById(String coll, String fn) throws Exception {
         String repository = coll;
         String fromObject = getObjectPath("", fn);
@@ -541,24 +552,25 @@ public class MdDocsService implements IMdDocsService {
             String r = gkbStorer.renameFile(coll, oldName, newName);
             GBaseKeyValue value = GBaseKeyValue.build();
             int i1 = newName.lastIndexOf('/');
-            String path = newName.substring(0,i1+1);
-            value.put("path",path);
+            String path = newName.substring(0, i1 + 1);
+            value.put("path", path);
             value.setName(newName);
             value.setId(intId);
 
             GBaseKeyValue updated = mdDynamicRepository.update(collId, value);
 
             return updated;
-        }else{
+        } else {
 
         }
         return Map.of("error", "No supported fields to update");
     }
 
-    @Override
-    public String getIndexName(String coll) {
-        return mdDynamicRepository.indexName(coll);
-    }
+//    @Override
+//    public String getIndexName(String coll) {
+//        //return mdDynamicRepository.indexName(coll);
+//        return mdCollectionsService.getIndexName(coll);
+//    }
 
     @Override
     public Map<String, Object> locateByHeading(String coll, Map<String, Object> req) throws Throwable {
@@ -567,13 +579,74 @@ public class MdDocsService implements IMdDocsService {
         //MdSearcher searcher = getSearcher(coll, name);
         String collId = getStrOfCollId(coll);
         String mdid = getKvObjectId(collId, req);
-        MdSearcher searcher = getSearcher2(coll,collId, mdid);
+        MdSearcher searcher = getSearcher2(coll, collId, mdid);
 
         Map<String, Object> ret = searcher.searchByHeadings(headings);
         ret.put(QUERY_HEADINGS, headings);
         return ret;
     }
 
+
+    @Override
+    //删除 md，同时删 chunk 和 index。
+    public int deleteCascadeByMdid(String coll, String gid) throws Throwable {
+        String collId = getStrOfCollId(coll);
+        GBaseKeyValue kv = findByMdid(coll, gid);
+        if (kv == null) {
+            log.error("该 id: {} 不存在", gid);
+            return -1;
+        }
+        Integer mdId = kv.getIntId();
+//        List<Map<String, Object>> rets =  mdDynamicChunkRepository.getKeywordsByMdId(collId, mdId);
+//        List<String> objects = new ArrayList<>();
+//        if (rets != null) {
+//            for (Map<String, Object> ret : rets) {
+//                Object o = ret.get("id");
+//                objects.add(o.toString());
+//            }
+//        }
+//        if (objects.size() >0){
+//            int deleted = hybridSearch.delete(coll, objects);
+//            log.info("删除索引文件: {}", gid);
+//            long deleted2 = mdDynamicChunkRepository.deleteByMdId(collId,mdId);
+//            log.info("删除chunk文件: {}", gid);
+//        }
+
+        List<String> uids = mdDynamicChunkRepository.findUidsByMdId(collId, mdId);
+        if (uids != null && !uids.isEmpty()) {
+            //hybridSearch.delete(coll, uids);
+            mdIndexer.delete(coll,uids);
+            log.info("删除索引文件,mdid: {}", gid);
+            long r = mdDynamicChunkRepository.deleteByMdId(collId, mdId);
+            log.info("删除chunk文件,mdid: {}", gid);
+        }
+
+        return doDelete(coll, kv);
+    }
+
+    @Override
+    public int deleteIndexesByMdid(String coll, String gid) throws Throwable {
+        String collId = getStrOfCollId(coll);
+        GBaseKeyValue kv = findByMdid(coll, gid);
+        if (kv == null) {
+            log.error("该 id: {} 不存在", gid);
+            return -1;
+        }
+        Integer mdId = kv.getIntId();
+        List<String> uids = mdDynamicChunkRepository.findUidsByMdId(collId, mdId);
+        if (uids != null && !uids.isEmpty()) {
+            //hybridSearch.delete(coll, uids);
+            mdIndexer.delete(coll,uids);
+            log.info("删除索引文件,mdid: {}", gid);
+            //long r = mdDynamicChunkRepository.deleteByMdId(collId, mdId);
+            //log.info("删除chunk文件: {}", gid);
+        }
+        return uids.size();
+    }
+
+    //mdDynamicChunkRepository
+
+
     ///////////////////////
     // renameByPath / deleteByPath
 

server/src/main/java/com/giantan/data/mds/service/impl/MdTaskManager.java

@@ -1,11 +1,12 @@
 package com.giantan.data.mds.service.impl;
 
-import com.giantan.data.index.IHybridSearch;
+//import com.giantan.data.index.IHybridSearch;
 import com.giantan.data.mds.bot.GChatClient;
 import com.giantan.data.mds.repository.MdDynamicTaskRepository;
+import com.giantan.data.mds.repository.MdIndexer;
 import com.giantan.data.mds.service.IMdChunksService;
 import com.giantan.data.mds.service.IMdFilesService;
-import com.giantan.data.mds.service.IVectorization;
+import com.giantan.data.index.IVectorization;
 import com.giantan.data.mds.task.MdPersistentTaskService;
 import com.giantan.data.mds.task.impl.ChunksTaskHandler;
 import com.giantan.data.mds.task.impl.MdsTaskHandler;
@@ -54,8 +55,11 @@ public class MdTaskManager extends TaskManager {
     @Autowired
     IVectorization vectorizationService;
 
+//    @Autowired
+//    IHybridSearch hybridSearch;
+
     @Autowired
-    IHybridSearch hybridSearch;
+    MdIndexer indexer;
 
     @Autowired
     MdCollectionsService mdCollectionsService;
@@ -128,7 +132,7 @@ public class MdTaskManager extends TaskManager {
         List<ITaskHandler> handlers = new ArrayList<>();
         MdsTaskHandler mdsTaskHandler = new MdsTaskHandler(mdFilesService, mdChunksService);
         handlers.add(mdsTaskHandler);
-        ChunksTaskHandler chunksTaskHandler = new ChunksTaskHandler(mdChunksService, vectorizationService, hybridSearch, gChatClient);
+        ChunksTaskHandler chunksTaskHandler = new ChunksTaskHandler(mdChunksService, vectorizationService, indexer, gChatClient);
         handlers.add(chunksTaskHandler);
         TaskHandlerRegistry registry = new TaskHandlerRegistry(handlers);
         return registry;

server/src/main/java/com/giantan/data/mds/task/MdPersistentTaskService.java

@@ -58,4 +58,11 @@ public class MdPersistentTaskService implements IPersistentTaskService {
         return rs;
     }
 
+    @Override
+    public TaskStatusHistory find(String coll, String id) {
+        int collId = mdCollectionsService.getCollectionId(coll);
+        TaskStatusHistory r = mdDynamicTaskRepository.find(Integer.toString(collId), id);
+        return r;
+    }
+
 }

server/src/main/java/com/giantan/data/mds/task/impl/ChunksTaskHandler.java

@@ -4,9 +4,9 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.giantan.ai.util.JsonUtil;
 import com.giantan.data.mds.bot.GChatClient;
 import com.giantan.data.mds.chunk.MdChunk;
-import com.giantan.data.index.IHybridSearch;
+import com.giantan.data.mds.repository.MdIndexer;
 import com.giantan.data.mds.service.IMdChunksService;
-import com.giantan.data.mds.service.IVectorization;
+import com.giantan.data.index.IVectorization;
 import com.giantan.data.index.dto.DocReq;
 import com.giantan.data.index.dto.DocResp;
 import com.giantan.data.tasks.TaskContext;
@@ -25,17 +25,20 @@ public class ChunksTaskHandler extends BaseTaskHandler {
 
     IMdChunksService mdChunksService;
     IVectorization vectorizationService;
-    IHybridSearch hybridSearch;
+    //IHybridSearch hybridSearch;
+    MdIndexer indexer;
     GChatClient gChatClient;
 
     public ChunksTaskHandler(IMdChunksService mdChunksService,
                              IVectorization vectorizationService,
-                             IHybridSearch hybridSearch,
+                             //IHybridSearch hybridSearch,
+                             MdIndexer indexer,
                              GChatClient gChatClient
     ) {
         this.mdChunksService = mdChunksService;
         this.vectorizationService = vectorizationService;
-        this.hybridSearch = hybridSearch;
+        //this.hybridSearch = hybridSearch;
+        this.indexer = indexer;
         this.gChatClient = gChatClient;
     }
 
@@ -69,7 +72,7 @@ public class ChunksTaskHandler extends BaseTaskHandler {
         } else if (payload.containsKey(MD_IDS)) {
             List<Object> mdIds = (List<Object>) payload.get(MD_IDS);
             for (Object mdId : mdIds) {
-                List<Map<String, Object>> rets = mdChunksService.getKeywordsByMdId(coll, toInt(mdId));
+                List<Map<String, Object>> rets = mdChunksService.getSectionPathByMdId(coll, toInt(mdId));
                 if (rets != null) {
                     for (Map<String, Object> ret : rets) {
                         Object o = ret.get("id");
@@ -81,7 +84,7 @@ public class ChunksTaskHandler extends BaseTaskHandler {
             int from = (int) payload.get(MD_START_ID);
             int to = (int) payload.get(MD_END_ID);
             for (int i = from; i < to; i++) {
-                List<Map<String, Object>> rets = mdChunksService.getKeywordsByMdId(coll, i);
+                List<Map<String, Object>> rets = mdChunksService.getSectionPathByMdId(coll, i);
                 if (rets != null) {
                     for (Map<String, Object> ret : rets) {
                         Object o = ret.get("id");
@@ -117,9 +120,11 @@ public class ChunksTaskHandler extends BaseTaskHandler {
         try {
             String coll = context.getCollection();
             Long chunkId = toLong(objectId);
-            MdChunk chunk = mdChunksService.findById(coll, chunkId);
-            String uid = chunk.getSectionPath();
-            hybridSearch.delete(coll, List.of(uid));
+//            MdChunk chunk = mdChunksService.findById(coll, chunkId);
+//            String uid = chunk.getSectionPath();
+            String uid = mdChunksService.findSectionPathById(coll, chunkId);
+            //hybridSearch.delete(coll, List.of(uid));
+            indexer.delete(coll, List.of(uid));
             context.logSuccess(objectId.toString(), operation);
         } catch (Exception e) {
             context.logFailure(objectId.toString(), operation, e.getMessage());
@@ -135,7 +140,7 @@ public class ChunksTaskHandler extends BaseTaskHandler {
             Long chunkId = toLong(objectId);
             MdChunk chunk = mdChunksService.findById(coll, chunkId);
 
-            String plainText = chunk.getPlainText();
+            //String plainText = chunk.getPlainText();
             Object o = params.get("embeddingMetadata");
 
             Map<String, Object> extra = null;
@@ -144,9 +149,7 @@ public class ChunksTaskHandler extends BaseTaskHandler {
                 //Map<String, String> kvs = getChunkMetadata(mapping, chunkMetadata);
                 extra = buildExtra(mapping, chunk);
             }
-
-            Boolean skipChatGptIfExists = (Boolean) params.getOrDefault("skipChatGptIfExists", true);
-
+            //Boolean skipChatGptIfExists = (Boolean) params.getOrDefault("skipChatGptIfExists", true);
             callIndex(coll, chunkId, chunk, extra);
             context.logSuccess(objectId.toString(), operation);
         } catch (Exception e) {
@@ -308,15 +311,20 @@ public class ChunksTaskHandler extends BaseTaskHandler {
         metadata.put("iid", chunk.getId());
         metadata.put("offsetStart", chunk.getOffsetStart());
         metadata.put("offsetEnd", chunk.getOffsetEnd());
-        metadata.put("content", chunk.getContent());
+        // 原始的content可能会超过json的限制,所以去掉
+        //metadata.put("content", chunk.getContent());
+
+        Map<String, Object> metadata1 = indexer.appendMetadata(coll, chunk.getMdId());
+        metadata.putAll(metadata1);
 
         DocReq doc = new DocReq();
         doc.setId(chunk.getSectionPath());
         doc.setText(chunk.getPlainText());
         doc.setEmbedding(chunk.getEmbedding());
         doc.setMetadata(metadata);
-        String indexName = mdChunksService.getIndexName(coll);
-        List<DocResp> ret = hybridSearch.addDirect(indexName, List.of(doc));
+        //String indexName = mdChunksService.getIndexName(coll);
+        //List<DocResp> ret = hybridSearch.addDirect(indexName, List.of(doc));
+        List<DocResp> ret = indexer.addDirect(coll, List.of(doc));
         //System.out.println(ret);
     }
 

server/src/main/java/com/giantan/data/mds/task/impl/EmbeddingTaskHandler.java

@@ -1,140 +0,0 @@
-package com.giantan.data.mds.task.impl;
-
-import com.giantan.ai.util.JsonUtil;
-import com.giantan.data.mds.chunk.MdChunk;
-import com.giantan.data.mds.service.IMdChunksService;
-import com.giantan.data.mds.service.IVectorization;
-import com.giantan.data.tasks.TaskContext;
-import com.giantan.data.tasks.TaskType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class EmbeddingTaskHandler extends BaseTaskHandler {
-    private static final org.slf4j.Logger log
-            = org.slf4j.LoggerFactory.getLogger(EmbeddingTaskHandler.class);
-
-
-    IMdChunksService mdChunksService;
-
-    IVectorization vectorizationService;
-
-    public EmbeddingTaskHandler(IMdChunksService mdChunksService, IVectorization vectorizationService) {
-        this.mdChunksService = mdChunksService;
-        this.vectorizationService = vectorizationService;
-    }
-
-    @Override
-    public TaskType getType() {
-        return TaskType.EMBEDDING;
-    }
-
-    protected void preProcess(final TaskContext context) {
-        List<Object> objects = new ArrayList<>();
-        String coll = context.getCollection();
-        Map<String, Object> payload = context.getParams();
-        if (payload.containsKey(MD_IDS)) {
-
-            List<Object> mdIds = (List<Object>) payload.get(MD_IDS);
-            for (Object mdId : mdIds) {
-                List<Map<String, Object>> rets = mdChunksService.getKeywordsByMdId(coll, toInt(mdId));
-                if (rets != null) {
-                    for (Map<String, Object> ret : rets) {
-                        Object o = ret.get("id");
-                        objects.add(o);
-                    }
-                }
-            }
-        }
-        context.setObjectIds(objects);
-    }
-
-    @Override
-    public void doing(TaskContext context, Object objectId) {
-        try {
-            String coll = context.getCollection();
-            Map<String, Object> params = context.getParams();
-            Long chunkId = toLong(objectId);
-            MdChunk chunk = mdChunksService.findById(coll, chunkId);
-            //System.out.println("chunk=" + chunk);
-            // Map<String, Object> chunkMetadata = chunk.getMetadata();
-            //System.out.println("chunkMetadata=" + chunkMetadata);
-            String plainText = chunk.getPlainText();
-            //System.out.println("plainText=" + plainText);
-            Object o = params.get("embeddingMetadata");
-
-            Map<String, Object> extra = null;
-            if (o != null && o instanceof Map) {
-                Map<String, String> mapping = (Map<String, String>) o;
-                //Map<String, String> kvs = getChunkMetadata(mapping, chunkMetadata);
-                extra = buildExtra(mapping, chunk);
-            }
-
-            //System.out.println("kvs=" + kvs);
-
-            Boolean skipChatGptIfExists = (Boolean) params.getOrDefault("skipChatGptIfExists", true);
-            //System.out.println("skipChatGptIfExists=" + skipChatGptIfExists);
-
-            String embedding = chunk.getEmbedding();
-            if (skipChatGptIfExists) {
-                if (embedding != null) {
-
-                } else {
-                    // 调用LLM
-                    callEmbedding(coll, chunkId, plainText, extra);
-                }
-            } else {
-                // 调用LLM
-                callEmbedding(coll, chunkId, plainText, extra);
-            }
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    private int callEmbedding(String coll, Long chunkId, String plainText, Map<String, Object> extra) throws IOException, InterruptedException {
-        StringBuilder sb = new StringBuilder();
-        extra.forEach((k, v) -> {
-            sb.append(k).append(": ").append(v).append("\n");
-        });
-
-        sb.append(plainText);
-        float[] fs = vectorizationService.singleVectorization(sb.toString());
-        if (fs != null) {
-            String js = JsonUtil.toJsonString(fs);
-            int r = mdChunksService.setEmbedding(coll, chunkId, js);
-            return r;
-        }
-        return 0;
-    }
-
-    private Map<String, Object> buildExtra(Map<String, String> mapping, MdChunk chunk) {
-        Map<String, Object> extra = new HashMap<String, Object>();
-        Map<String, Object> metadata = chunk.getMetadata();
-        mapping.forEach((k, v) -> {
-            if (k.equals("keywords")) {
-                List<String> keywords = chunk.getKeywords();
-                if (keywords != null && keywords.size() > 0) {
-                    extra.put(v, String.join(",", keywords));
-                }
-            } else {
-                Object o = metadata.get(k);
-                if (o != null) {
-                    if (o instanceof String) {
-                        extra.put(v, (String) o);
-                    } else if (o instanceof List) {
-                        List l = (List) o;
-                        if (l.size() > 0) {
-                            extra.put(v, String.join("\n", l));
-                        }
-                    }
-                }
-            }
-        });
-        return extra;
-    }
-
-}

server/src/main/java/com/giantan/data/mds/task/impl/KeywordsTaskHandler.java

@@ -1,140 +0,0 @@
-package com.giantan.data.mds.task.impl;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.giantan.data.mds.bot.GChatClient;
-import com.giantan.data.mds.chunk.MdChunk;
-import com.giantan.data.mds.service.IMdChunksService;
-import com.giantan.data.tasks.TaskContext;
-import com.giantan.data.tasks.TaskType;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class KeywordsTaskHandler extends BaseTaskHandler {
-    private static final org.slf4j.Logger log
-            = org.slf4j.LoggerFactory.getLogger(KeywordsTaskHandler.class);
-
-
-    IMdChunksService mdChunksService;
-    GChatClient gChatClient;
-
-    public KeywordsTaskHandler(IMdChunksService mdChunksService, GChatClient gChatClient) {
-        this.mdChunksService = mdChunksService;
-        this.gChatClient = gChatClient;
-    }
-
-    // mdIds  chunkIds
-    protected void preProcess(final TaskContext context) {
-        List<Object> objects = new ArrayList<>();
-        String coll = context.getCollection();
-        Map<String, Object> payload = context.getParams();
-
-
-        if (payload.containsKey(MD_IDS)) {
-            List<Object> mdIds = (List<Object>) payload.get(MD_IDS);
-            for (Object mdId : mdIds) {
-                List<Map<String, Object>> rets = mdChunksService.getKeywordsByMdId(coll, toInt(mdId));
-                if (rets != null){
-                    for (Map<String, Object> ret : rets) {
-                        Object o = ret.get("id");
-                        objects.add(o);
-                    }
-                }
-            }
-        }
-        context.setObjectIds(objects);
-    }
-
-    @Override
-    public TaskType getType() {
-        return TaskType.EXTRACT_KEYWORDS;
-    }
-
-    @Override
-    public void doing(TaskContext context, Object objectId) {
-        try {
-            String coll = context.getCollection();
-            Map<String, Object> params = context.getParams();
-            Long chunkId = toLong(objectId);
-            MdChunk chunk = mdChunksService.findById(coll, chunkId);
-            //System.out.println("chunk=" + chunk);
-            Map<String, Object> metadata = chunk.getMetadata();
-            //System.out.println("metadata=" + metadata);
-            String plainText = chunk.getPlainText();
-            //System.out.println("plainText=" + plainText);
-
-            Map<String, String> kvs = new HashMap<>();
-
-            Object o = params.get("chunkMetadata");
-            if (o != null && o instanceof Map) {
-                Map<String, String> mapping = (Map<String, String>) o;
-                kvs = getChunkMetadata(mapping, metadata);
-            }
-
-            Boolean skipChatGptIfExists = (Boolean) params.getOrDefault("skipChatGptIfExists", true);
-            //System.out.println("skipChatGptIfExists=" + skipChatGptIfExists);
-            List<String> keywords = chunk.getKeywords();
-            if (skipChatGptIfExists) {
-                if (keywords != null && keywords.size() > 0) {
-
-                } else {
-                    // 调用LLM
-                    callLLM(coll,chunkId,plainText, kvs);
-                }
-            } else {
-                // 调用LLM
-                callLLM(coll,chunkId,plainText, kvs);
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-            throw new RuntimeException(e);
-        }
-
-    }
-
-    private void callLLM(String coll, Long chunkId, String plainText, Map<String, String> kvs) throws JsonProcessingException {
-        //long t = System.currentTimeMillis();
-        Map<String, Object> kws = gChatClient.getKeywordsAndQuestions(plainText, kvs);
-        //t = System.currentTimeMillis() - t;
-        //System.out.println("kws=" + kws);
-        //System.out.println("used time = " + t);
-        if (kws != null && kws.size() > 0) {
-            List<String> keywords = null;
-            List<String> questions = null;
-            Object o = kws.get("keywords");
-            if (o != null && o instanceof List) {
-                keywords = (List<String>) o;
-            }
-            o = kws.get("questions");
-            if (o != null && o instanceof List) {
-                questions = (List<String>) o;
-            }
-
-            if (keywords != null && questions != null) {
-                int i = mdChunksService.updateKeywordsOrMetadata(coll, chunkId, keywords, Map.of("llm_questions", questions));
-                //System.out.println("i=" + i);
-            }
-        }
-    }
-
-
-    private Map<String, String> getChunkMetadata(Map<String, String> mapping, Map<String, Object> metadata) {
-        Map<String, String> m2 = new HashMap<String, String>();
-        mapping.forEach((k, v) -> {
-            Object o = metadata.get(k);
-            if (o != null) {
-                if (o instanceof String) {
-                    m2.put(v, (String) o);
-                } else if (o instanceof List) {
-                    List l = (List) o;
-                    if (l.size() > 0) {
-                        m2.put(v, String.join(",", l));
-                    }
-                }
-            }
-        });
-        return m2;
-    }
-}

server/src/main/java/com/giantan/data/mds/task/impl/SliceTaskHandler.java

@@ -1,213 +0,0 @@
-package com.giantan.data.mds.task.impl;
-
-import com.giantan.data.kvs.kvstore.GBaseKeyValue;
-import com.giantan.data.mds.chunk.MdChunk;
-import com.giantan.data.mds.service.IMdChunksService;
-import com.giantan.data.mds.service.IMdFilesService;
-import com.giantan.data.tasks.TaskContext;
-import com.giantan.data.tasks.TaskType;
-import org.cnnlp.data.document.GDocConstants;
-import org.cnnlp.data.document.GDocument;
-import org.cnnlp.data.splitter.*;
-import org.cnnlp.data.util.BaseParameters;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.*;
-
-
-//@Component
-public class SliceTaskHandler extends BaseTaskHandler {
-
-    private static final org.slf4j.Logger log
-            = org.slf4j.LoggerFactory.getLogger(SliceTaskHandler.class);
-
-    private static final String MD_TYPE = "mdType";
-    private static final String CHUNK_METADATA = "chunkMetadata";
-    private static final String FILE_NAME = "_fileName";
-    //private static final String CHUNK_ULID = "_ulid";
-
-    IMdFilesService mdFilesService;
-    IMdChunksService mdChunksService;
-
-    //IdGenerator idGen = new UlidGenerator();
-
-    public SliceTaskHandler(IMdFilesService mdFilesService, IMdChunksService mdChunksService) {
-        this.mdFilesService = mdFilesService;
-        this.mdChunksService = mdChunksService;
-    }
-
-//    @Override
-//    public void handle(TaskContext context) {
-//        for (Object objectId : context.getObjectIds()) {
-//            if (context.isCancelled()) {
-//                context.setStatus(TaskStatus.CANCELLED);
-//                log.info(getType()+" task: {}(objectId: {}) cancelled", context.getTaskId(),objectId);
-//                break;
-//            }
-//            try {
-//                System.out.println("Slicing object: " + objectId);
-//                Thread.sleep(15000); // simulate
-//                context.logSuccess(objectId.toString());
-//            } catch (Exception e) {
-//                context.logFailure(objectId.toString(), e.getMessage());
-//                log.error(getType()+" task: {}(objectId: {}) error: ", context.getTaskId(),objectId,e.getMessage());
-//            }
-//        }
-//    }
-
-
-    @Override
-    public TaskType getType() {
-        return TaskType.SLICE;
-    }
-
-    @Override
-    public void doing(TaskContext context, Object objectId) {
-        List<String> operations = context.getOperations();
-        for (String operation : operations) {
-            doing(context, objectId, operation);
-        }
-    }
-
-    public void doing(TaskContext context, Object objectId, String operation) {
-        if (operation.equals("slice")) {
-            slicing(context, objectId, "slice");
-        }
-    }
-
-    public void slicing(TaskContext context, Object objectId, String operation) {
-
-        //System.out.println("Slicing object: " + objectId);
-        try {
-            String coll = context.getCollection();
-            Map<String, Object> params = context.getParams();
-
-            String mdId = objectId.toString();
-            GBaseKeyValue mdMeta = mdFilesService.findByMdid(coll, mdId);
-            if (mdMeta != null) {
-                String text = mdFilesService.getMdFileContent(coll, mdId);
-                String mdType = getMdType(params);
-                String name = mdMeta.getName();
-                String baseName = SplitUtils.getBaseName(name);
-
-                List<GDocument> chunks = splitToChunks(text, mdType, params);
-
-                if (chunks != null) {
-                    // 删除 gid 的chunks
-                    // 存 chunks
-                    //System.out.println(chunks.get(0));
-                    Map<String, Object> chunkMetadata = new HashMap<>();
-                    Object mo = params.get(CHUNK_METADATA);
-                    if (mo != null && mo instanceof Map) {
-                        chunkMetadata.putAll((Map) mo);
-                    }
-                    mdChunksService.deleteByMdId(coll, toInt(mdId));
-                    List<MdChunk> mdChunks = new ArrayList<MdChunk>();
-                    for (int i = 0; i < chunks.size(); i++) {
-                        MdChunk chunk = toChunk(toInt(mdId), chunks.get(i), i, baseName, chunkMetadata);
-                        mdChunks.add(chunk);
-                    }
-
-                    List<Integer> rets = mdChunksService.saveAll(coll, mdChunks);
-                    //System.out.println("saved="+rets.size());
-                }
-            }
-            context.logSuccess(mdId,operation);
-        } catch (Throwable e) {
-            context.logFailure(objectId.toString(),operation,e.getMessage());
-            throw new RuntimeException(e);
-        }
-    }
-
-    private String getMdType(Map<String, Object> params) {
-        String type = MdChunking.MD_TYPE_SIMPLE;
-        if (params != null) {
-            Object o = params.get(MD_TYPE);
-            if (o != null) {
-                type = o.toString();
-            }
-        }
-        return type;
-    }
-
-
-//    public List<GDocument> splitSimple(String text) throws IOException {
-//        SimpleMdSplitter splitter = new SimpleMdSplitter();
-//        splitter.setCommentProcessor(KvCommentProcessor.build());
-//        BaseParameters params = BaseParameters.defaultParams();
-//        //Path path = Paths.get(md);
-//        //String baseName = SplitUtils.getFileBaseName(path);
-//        //System.out.println("baseName=" + baseName);
-//        List<GDocument> docs = splitter.split(text, params);
-//        //SplitUtils.toJsonFile(new File(json), docs);
-//        return docs;
-//    }
-//
-//    public List<GDocument> splitFaq(String text) throws IOException {
-//        FaqMdSplitter splitter = new FaqMdSplitter();
-//        BaseParameters params = BaseParameters.defaultParams();
-//        List<GDocument> docs = splitter.split(text, params);
-//        //SplitUtils.toJsonFile(new File(json), docs);
-//        return docs;
-//    }
-
-
-    protected MdChunk toChunk(Integer mdId, GDocument doc, int idx, String baseName, Map<String, Object> userMetadata) {
-        MdChunk chunk = new MdChunk();
-        chunk.setMdId(mdId);
-        chunk.setChunkIndex(idx);
-        chunk.setPlainText(doc.getText());
-        chunk.setSectionPath(doc.getId());
-        chunk.setCreatedAt(new Date().toInstant());
-
-        Map<String, Object> metadata = doc.getMetadata();
-        Object o = metadata.remove(GDocConstants.RAW_CONTENT);
-        if (o != null) {
-            chunk.setContent(o.toString());
-        }
-
-        o = metadata.remove(GDocConstants.START_OFFSET);
-        if (o != null) {
-            chunk.setOffsetStart(toInt(o));
-        }
-        o = metadata.remove(GDocConstants.END_OFFSET);
-        if (o != null) {
-            chunk.setOffsetEnd(toInt(o));
-        }
-
-        o = metadata.remove(GDocConstants.FROM_IDX);
-        if (o != null) {
-            chunk.setParagraphStart(toInt(o));
-        }
-        o = metadata.remove(GDocConstants.TO_IDX);
-        if (o != null) {
-            chunk.setParagraphEnd(toInt(o));
-        }
-
-        Map<String, Object> metadata1 = chunk.getMetadata();
-        if (metadata1 == null) {
-            metadata1 = new HashMap<String, Object>();
-            chunk.setMetadata(metadata1);
-        }
-
-        if (userMetadata.size() > 0) {
-            metadata1.putAll(userMetadata);
-        }
-        metadata1.put(FILE_NAME, baseName);
-        metadata1.putAll(metadata);
-
-        //metadata1.put(CHUNK_ULID, idGen.generateId());
-        return chunk;
-    }
-
-    public List<GDocument> splitToChunks(String text, String type, Map<String, Object> params) throws IOException {
-        BaseParameters params2 = new BaseParameters(params);
-        params2.put(MD_TYPE, type);
-        IMdChunking chunker = new MdChunking();
-        List<GDocument> chunks = chunker.chunking(text, params2);
-        return chunks;
-    }
-
-}

server/src/main/java/com/giantan/data/qa/constant/QaConstants.java

@@ -14,23 +14,24 @@ public class QaConstants {
     public static final String FIELD_RETURN = "returnFields";
 
 
-    //可配置项（用户可控）
-    //字段选择：["name", "description", "tags", "attributes.color"]
-    //模式：single | per-field
-    //连接符：=, :, -> 等（输出格式）
-    //空值处理：跳过 | 输出 "null"
-    //public static final String CHUNK_TEMPLATE = "chunkTemplate";
-    public static final String CHUNK_MODE = "chunkMode";
-    public static final String CHUNK_MODE_SINGLE = "single";
-    public static final String CHUNK_MODE_MULTIPLE = "multiple";
-    public static final String CHUNK_MODE_CUSTOM = "custom";
-    public static final String CHUNK_TEMPLATES = "chunkTemplates";
-    
-    public static final String INDEX_STRATEGY = "indexStrategy";
-    public static final String INDEX_MODE = "indexMode";  //  、singleCollection
-    public static final String INDEX_MODE_T2C = "table2collection";
-    public static final String INDEX_MODE_SINGLE = "singleCollection";
-    public static final String INDEX_COLLECTION_PREFIX = "collectionPrefix";
-    public static final String INDEX_GLOBAL_COLLECTION = "globalCollection";
+//    //可配置项（用户可控）
+//    //字段选择：["name", "description", "tags", "attributes.color"]
+//    //模式：single | per-field
+//    //连接符：=, :, -> 等（输出格式）
+//    //空值处理：跳过 | 输出 "null"
+//    //public static final String CHUNK_TEMPLATE = "chunkTemplate";
+//    public static final String CHUNK_MODE = "chunkMode";
+//    public static final String CHUNK_MODE_SINGLE = "single";
+//    public static final String CHUNK_MODE_MULTIPLE = "multiple";
+//    public static final String CHUNK_MODE_CUSTOM = "custom";
+//    public static final String CHUNK_TEMPLATES = "chunkTemplates";
+//
+//    public static final String INDEX_STRATEGY = "indexStrategy";
+//    public static final String INDEX_MODE = "indexMode";
+//    public static final String INDEX_MODE_T2C = "table2collection";  // 一个table 对应 一个milvus的collection
+//    public static final String INDEX_MODE_SINGLE = "singleCollection";  // 所有table 都索引到一个collection
+//
+//    public static final String INDEX_COLLECTION_PREFIX = "collectionPrefix";
+//    public static final String INDEX_GLOBAL_COLLECTION = "globalCollection";
 
 }

server/src/main/java/com/giantan/data/qa/controller/QaSearchContoller.java

@@ -5,6 +5,7 @@ import com.giantan.data.index.IHybridSearch;
 import com.giantan.data.kvs.kvstore.GBaseKeyValue;
 import com.giantan.data.qa.constant.QaConstants;
 import com.giantan.data.qa.repository.QaDocRepository;
+import com.giantan.data.qa.repository.QaIndexer;
 import com.giantan.data.qa.service.QaDocsService;
 import jakarta.servlet.http.HttpServletRequest;
 import org.springframework.beans.factory.annotation.Autowired;

server/src/main/java/com/giantan/data/qa/controller/QaTaskController.java

@@ -1,9 +1,9 @@
 package com.giantan.data.qa.controller;
 
+import com.giantan.ai.common.reponse.R;
 import com.giantan.data.qa.constant.QaConstants;
+import com.giantan.data.qa.service.task.QaTaskManager;
 import com.giantan.data.tasks.TaskContext;
-import com.giantan.data.tasks.TaskManager;
-import com.giantan.data.tasks.TaskOperationsStatus;
 import com.giantan.data.tasks.TaskType;
 import com.giantan.data.tasks.repository.TaskStatusHistory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,7 +17,7 @@ import java.util.*;
 public class QaTaskController {
 
     @Autowired
-    private TaskManager manager;
+    private QaTaskManager manager;
 
     /*
     {
@@ -33,7 +33,7 @@ public class QaTaskController {
      */
 
     @PostMapping("/submit")
-    public Map submit(@PathVariable String collId, @RequestBody Map<String, Object> payload) {
+    public R<Map> submit(@PathVariable String collId, @RequestBody Map<String, Object> payload) {
         String t = (String) payload.remove("type");
 
         TaskType type = TaskType.valueOf(t);
@@ -57,55 +57,62 @@ public class QaTaskController {
         Map<String, Object> params = new HashMap<>(payload);
 
         String ret = manager.submit(collId, type, params);
-        return Map.of("taskId", ret);
+        return R.data(Map.of("taskId", ret));
     }
 
     @PostMapping("/{id}/cancel")
-    public Map cancel(@PathVariable String collId, @PathVariable String id) {
+    public R<Map> cancel(@PathVariable String collId, @PathVariable String id) {
         boolean ok = manager.cancel(collId, id);
-        return Map.of("canceled", ok);
+        return R.data(Map.of("canceled", ok));
     }
 
     @DeleteMapping("/{id}")
-    public Map delete(@PathVariable String collId, @PathVariable String id) {
+    public R<Map> delete(@PathVariable String collId, @PathVariable String id) {
         boolean ok = manager.delete(collId, id);
-        return Map.of("deleted", ok);
+        return R.data(Map.of("deleted", ok));
     }
 
     @GetMapping("/{id}/status")
-    public Map<String, TaskOperationsStatus> status(@PathVariable String collId, @PathVariable String id) {
+    public R<Map> status(@PathVariable String collId, @PathVariable String id) {
         TaskContext ctx = manager.getTask(collId, id);
-        return ctx != null ? ctx.getObjectStatus() : Collections.emptyMap();
+        Map ret = ctx != null ? ctx.getObjectStatus() : Collections.emptyMap();
+        return R.data(ret);
     }
 
     @GetMapping("/{id}")
-    public TaskContext getTask(@PathVariable String collId, @PathVariable String id) {
-        return manager.getTask(collId, id);
+    public R<TaskContext> getTask(@PathVariable String collId, @PathVariable String id) {
+        TaskContext task = manager.getTask(collId, id);
+        return R.data(task);
     }
 
     @DeleteMapping("/cleanup")
-    public Map cleanup(@PathVariable String collId) {
+    public R<Map> cleanup(@PathVariable String collId) {
         int r = manager.cleanupNow(collId);
         //return ResponseEntity.ok("Task cleanup triggered.");
-        return Map.of("deleted", r);
+        return R.data(Map.of("deleted", r));
     }
 
     @GetMapping
-    public Collection<TaskContext> listAllTasks(@PathVariable String collId) {
-        return manager.allTasks(collId);
+    public R<Collection<TaskContext>> listAllTasks(@PathVariable String collId) {
+        return R.data(manager.allTasks(collId));
     }
 
     @GetMapping("/status/{status}")
-    public Collection<TaskContext> listTasksByStatus(@PathVariable String collId, @PathVariable String status) {
+    public R<Collection<TaskContext>> listTasksByStatus(@PathVariable String collId, @PathVariable String status) {
         //TaskStatus statusEnum = TaskStatus.valueOf(status.toUpperCase());
-        return manager.findByStatus(collId, status);
+        return R.data(manager.findByStatus(collId, status));
+    }
+
+    @GetMapping("/{id}/history")
+    public R<TaskStatusHistory> getHistoryTask(@PathVariable String collId, @PathVariable String id){
+        return R.data(manager.getHistoryTask(collId, id));
     }
 
     @GetMapping("/history")
-    public List<TaskStatusHistory> getTasks(@PathVariable String collId,
-                                            @RequestParam(value = "createdAtStart", required = false) String createdAtStart,
-                                            @RequestParam(value = "createdAtEnd", required = false) String createdAtEnd,
-                                            @RequestParam(value = "status", required = false) String status
+    public R<List<TaskStatusHistory>> getTasks(@PathVariable String collId,
+                                               @RequestParam(value = "createdAtStart", required = false) String createdAtStart,
+                                               @RequestParam(value = "createdAtEnd", required = false) String createdAtEnd,
+                                               @RequestParam(value = "status", required = false) String status
     ) {
         LocalDateTime startTime = null;
         if (createdAtStart != null) {
@@ -116,14 +123,14 @@ public class QaTaskController {
         if (createdAtEnd != null) {
             endTime = LocalDateTime.parse(createdAtEnd);
         }
-        return manager.getHistoryTasks(collId,startTime, endTime, status);
+        return R.data(manager.getHistoryTasks(collId, startTime, endTime, status));
     }
 
     @DeleteMapping("/history/cleanup")
-    public int deleteHistory(@PathVariable String collId,
-                             @RequestParam(value = "createdAtStart", required = false) String createdAtStart,
-                             @RequestParam(value = "createdAtEnd", required = false) String createdAtEnd,
-                             @RequestParam(value = "status", required = false) String status
+    public R deleteHistory(@PathVariable String collId,
+                           @RequestParam(value = "createdAtStart", required = false) String createdAtStart,
+                           @RequestParam(value = "createdAtEnd", required = false) String createdAtEnd,
+                           @RequestParam(value = "status", required = false) String status
     ) {
         LocalDateTime startTime = null;
         if (createdAtStart != null) {
@@ -134,7 +141,7 @@ public class QaTaskController {
         if (createdAtEnd != null) {
             endTime = LocalDateTime.parse(createdAtEnd);
         }
-        return manager.deleteHistoryTasks(collId,startTime, endTime, status);
+        return R.data(manager.deleteHistoryTasks(collId, startTime, endTime, status));
     }
 
 }

server/src/main/java/com/giantan/data/qa/controller/QaTaxonomyController.java

@@ -25,35 +25,60 @@ public class QaTaxonomyController {
     private QaTaxonomyService dynamicTaxonomyService;
 
     @GetMapping("/taxonomy/all")
-    public ResponseEntity<?> listAll(@PathVariable String collName) {
+//    public ResponseEntity<?> listAll(@PathVariable String collName) {
+//        List<TaxonomyNode> r = dynamicTaxonomyService.listAll(collName);
+//        return ResponseEntity.ok(r);
+//    }
+    public R listAll(@PathVariable String collName) {
         List<TaxonomyNode> r = dynamicTaxonomyService.listAll(collName);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
     @GetMapping("/taxonomy/tree")
-    public ResponseEntity<?> listTree(@PathVariable String collName) {
+//    public ResponseEntity<?> listTree(@PathVariable String collName) {
+//        List<TaxonomyNode> r = dynamicTaxonomyService.listTree(collName);
+//        return ResponseEntity.ok(r);
+//    }
+    public R listTree(@PathVariable String collName) {
         List<TaxonomyNode> r = dynamicTaxonomyService.listTree(collName);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
     @GetMapping("/taxonomy/{nodeId}/children")
-    public ResponseEntity<?> findChildren(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
+//    public ResponseEntity<?> findChildren(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
+//        List<TaxonomyNode> r = dynamicTaxonomyService.findChildren(collName, nodeId);
+//        return ResponseEntity.ok(r);
+//    }
+    public R findChildren(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
         List<TaxonomyNode> r = dynamicTaxonomyService.findChildren(collName, nodeId);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
     @GetMapping("/taxonomy/{nodeId}/docs")
-    public ResponseEntity<?> findMds(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
+//    public ResponseEntity<?> findMds(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
+//        List<GBaseKeyValue> r = dynamicTaxonomyService.findMds(collName, nodeId);
+//        return ResponseEntity.ok(r);
+//    }
+    public R findMds(@PathVariable String collName, @PathVariable String nodeId) throws Exception {
         List<GBaseKeyValue> r = dynamicTaxonomyService.findMds(collName, nodeId);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
     @PostMapping("/taxonomy")
-    public ResponseEntity<?> createNode(@PathVariable String collName, @RequestBody Map<String, Object> data) throws Exception {
+//    public ResponseEntity<?> createNode(@PathVariable String collName, @RequestBody Map<String, Object> data) throws Exception {
+//        TaxonomyNode r = dynamicTaxonomyService.createNode(collName, data);
+//        return ResponseEntity.ok(r);
+//    }
+    public R createNode(@PathVariable String collName, @RequestBody Map<String, Object> data) throws Exception {
         TaxonomyNode r = dynamicTaxonomyService.createNode(collName, data);
-        return ResponseEntity.ok(r);
+        return R.data(r);
     }
 
+
 //    @PostMapping("/{nodeId}/mds")
 //    public R<Map<String, Object>> upload(@PathVariable String collName,
 //                                         @PathVariable String nodeId,

server/src/main/java/com/giantan/data/qa/repository/QaDocRepository.java

@@ -1,7 +1,5 @@
 package com.giantan.data.qa.repository;
 
-import com.giantan.data.kvs.kvstore.GBaseKeyValue;
-import com.giantan.data.kvs.repository.GConverter;
 import com.giantan.data.kvs.repository.GEntity;
 import com.giantan.data.kvs.repository.index.GIndexedRepository;
 

server/src/main/java/com/giantan/data/qa/repository/QaExtraRepository.java

@@ -0,0 +1,29 @@
+package com.giantan.data.qa.repository;
+
+import com.giantan.data.kvs.repository.GDynamicRepository;
+import jakarta.annotation.PostConstruct;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Repository;
+
+// 用来以后 扩展的存储
+@Repository
+public class QaExtraRepository extends GDynamicRepository {
+    private static final org.slf4j.Logger log
+            = org.slf4j.LoggerFactory.getLogger(QaExtraRepository.class);
+
+
+    @Autowired
+    private JdbcTemplate jdbc;
+
+    public QaExtraRepository() {
+        this.schema = "qadb";
+        this.tablePrefix = "extra";
+    }
+
+    @PostConstruct
+    public void init() {
+        //collections = new GRepository(KvConfig.GKG_CATALOG, KvConfig.GKG_ENTRY_COLLECTIONS, jdbcTemplate);
+        setJdbcTemplate(this.jdbc);
+    }
+}

server/src/main/java/com/giantan/data/qa/repository/QaIndexer.java

@@ -1,12 +1,13 @@
 package com.giantan.data.qa.repository;
 
+import com.giantan.data.index.HybridIndexer;
 import com.giantan.data.index.IHybridSearch;
+import com.giantan.data.index.IndexConfig;
 import com.giantan.data.index.dto.DocReq;
 import com.giantan.data.index.dto.DocResp;
 import com.giantan.data.kvs.repository.GEntity;
 import com.giantan.data.kvs.repository.GEntityConfig;
 import com.giantan.data.kvs.repository.index.IIndexer;
-import com.giantan.data.qa.constant.QaConstants;
 import com.giantan.data.qa.service.IQaCollectionService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.expression.Expression;
@@ -23,15 +24,10 @@ import java.util.List;
 import java.util.Map;
 
 @Repository
-public class QaIndexer implements IIndexer {
+public class QaIndexer extends HybridIndexer implements IIndexer {
     private static final org.slf4j.Logger log
             = org.slf4j.LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-    public static final String COLL_ID = "__cid";
-    public static final String DOC_ID = "__did";
-
-    String defaultIndexPrefix = "qas";
-    String defaultChunkMode = "single";
 
     String cellectionPrefix = "qas_";
 
@@ -46,6 +42,7 @@ public class QaIndexer implements IIndexer {
 
     public QaIndexer() {
         //collMapper = new DefaultCollectionMapper();
+        defaultIndexPrefix = "qas";
     }
 
     @Override
@@ -81,7 +78,7 @@ public class QaIndexer implements IIndexer {
     }
 
     public String indexName(String coll) {
-        return this.defaultIndexPrefix + "_" + coll;
+        return defaultIndexPrefix + "_" + coll;
     }
 
     public String indexName(String prefix, String coll) {
@@ -89,26 +86,12 @@ public class QaIndexer implements IIndexer {
     }
 
     public String getMappedIndexName(String collId) {
-        String mapped = indexName(collId);
+        String collName = collectionService.getCollectionName(Integer.parseInt(collId));
+        String mapped = indexName(collName);
         Map<String, Object> attibutes = collectionService.getAttibutes(collId);
-        if (attibutes != null && !attibutes.isEmpty()) {
-            Object o = attibutes.get(QaConstants.INDEX_STRATEGY);
-            if (o != null && o instanceof Map strategy) {
-                Object o1 = strategy.get(QaConstants.INDEX_MODE);
-                if (o1 != null && o1 instanceof String mode) {
-                    if (mode.equals(QaConstants.INDEX_MODE_T2C)) {
-                        Object o2 = strategy.get(QaConstants.INDEX_COLLECTION_PREFIX);
-                        if (o2 != null && o2 instanceof String prefix) {
-                            mapped = indexName(prefix, collId);
-                        }
-                    } else if (mode.equals(QaConstants.INDEX_MODE_SINGLE)) {
-                        Object o2 = strategy.get(QaConstants.INDEX_GLOBAL_COLLECTION);
-                        if (o2 != null && o2 instanceof String collection) {
-                            mapped = collection;
-                        }
-                    }
-                }
-            }
+        String configIndexName = getConfigIndexName(collId, attibutes);
+        if (configIndexName != null) {
+            mapped = configIndexName;
         }
         return mapped;
     }
@@ -116,21 +99,12 @@ public class QaIndexer implements IIndexer {
     public boolean isOne2One(String collId) {
         boolean isOne2One = true;
         Map<String, Object> attibutes = collectionService.getAttibutes(collId);
-        if (attibutes != null && !attibutes.isEmpty()) {
-            Object o = attibutes.get(QaConstants.INDEX_STRATEGY);
-            if (o != null && o instanceof Map strategy) {
-                Object o1 = strategy.get(QaConstants.INDEX_MODE);
-                if (o1 != null && o1 instanceof String mode) {
-                    if (mode.equals(QaConstants.INDEX_MODE_SINGLE)) {
-//                        Object o2 = strategy.get(QaConstants.INDEX_GLOBAL_COLLECTION);
-//                        if (o2 != null && o2 instanceof String collection) {
-//                            mapped = collection;
-//                        }
-                        isOne2One = true;
-                    }
-                }
-            }
+
+        Boolean configOne2One = getConfigOne2One(attibutes);
+        if (configOne2One != null) {
+            isOne2One = configOne2One;
         }
+
         return isOne2One;
     }
 
@@ -138,11 +112,10 @@ public class QaIndexer implements IIndexer {
         String chunkMode = defaultChunkMode;
 
         Map<String, Object> attibutes = collectionService.getAttibutes(coll);
-        if (attibutes != null && !attibutes.isEmpty()) {
-            Object o = attibutes.get(QaConstants.CHUNK_MODE);
-            if (o != null && o instanceof String mode) {
-                chunkMode = mode;
-            }
+
+        String configChunkMode = getConfigChunkMode(attibutes);
+        if (configChunkMode != null) {
+            chunkMode = configChunkMode;
         }
         return chunkMode;
     }
@@ -150,14 +123,10 @@ public class QaIndexer implements IIndexer {
     private List<String> getChunkTemplates(String coll) {
         List<String> rets = null;
         Map<String, Object> attibutes = collectionService.getAttibutes(coll);
-        if (attibutes != null && !attibutes.isEmpty()) {
-            Object o = attibutes.get(QaConstants.CHUNK_TEMPLATES);
-            if (o != null && o instanceof List<?> templates) {
-                rets = new ArrayList<>(templates.size());
-                for (Object o1 : templates) {
-                    rets.add(o1.toString());
-                }
-            }
+
+        List<String> configChunkTemplates = getConfigChunkTemplates(attibutes);
+        if (configChunkTemplates != null) {
+            rets = configChunkTemplates;
         }
         return rets;
     }
@@ -165,9 +134,9 @@ public class QaIndexer implements IIndexer {
     private List<DocReq> toDocReq(String coll, GEntity entity) {
         String chunkMode = getChunkMode(coll);
         List<DocReq> rets = null;
-        if (chunkMode.equals(QaConstants.CHUNK_MODE_MULTIPLE)) {
+        if (chunkMode.equals(IndexConfig.CHUNK_MODE_MULTIPLE)) {
             rets = toDocReq2(coll, entity);
-        } else if (chunkMode.equals(QaConstants.CHUNK_MODE_CUSTOM)) {
+        } else if (chunkMode.equals(IndexConfig.CHUNK_MODE_CUSTOM)) {
             rets = toDocReq3(coll, entity);
         } else {
             rets = toDocReq1(coll, entity);
@@ -400,7 +369,7 @@ public class QaIndexer implements IIndexer {
         boolean b = false;
         try {
             if (isOne2One(collection)) {
-                b = hybridSearch.deleteAll(getMappedIndexName(collection));
+                b = hybridSearch.clearCollection(getMappedIndexName(collection));
             } else {
                 //TODO 特殊处理
                 //  "filterExpression": "metadata[\"__cid\"] == \"qas_2\""
@@ -419,7 +388,7 @@ public class QaIndexer implements IIndexer {
         boolean b = false;
         try {
             if (isOne2One(collection)) {
-                b = hybridSearch.drop(getMappedIndexName(collection));
+                b = hybridSearch.dropCollection(getMappedIndexName(collection));
             } else {
                 //TODO 特殊处理
                 //  "filterExpression": "metadata[\"__cid\"] == \"qas_2\""

server/src/main/java/com/giantan/data/qa/repository/QaTaskRepository.java

@@ -0,0 +1,21 @@
+package com.giantan.data.qa.repository;
+
+import com.giantan.data.tasks.repository.DynamicTaskRepository;
+import jakarta.annotation.PostConstruct;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public class QaTaskRepository extends DynamicTaskRepository {
+    private static final org.slf4j.Logger log
+            = org.slf4j.LoggerFactory.getLogger(QaTaskRepository.class);
+
+    public QaTaskRepository(JdbcTemplate jdbc) {
+        super(jdbc);
+    }
+
+    @PostConstruct
+    public void init() {
+        setSchema("qadb", "tasks_");
+    }
+}

server/src/main/java/com/giantan/data/qa/service/IQaCollectionService.java

@@ -1,5 +1,7 @@
 package com.giantan.data.qa.service;
 
+import com.giantan.data.mds.service.CollectionInstance;
+
 import java.util.Map;
 
 public interface IQaCollectionService {
@@ -7,4 +9,6 @@ public interface IQaCollectionService {
     Map<String,Object> getAttibutes(String collId);
     Map<String,Object> getAttributesByName(String name);
 
+    String getCollectionName(int collId) ;
+
 }

server/src/main/java/com/giantan/data/qa/service/QaCollectionService.java

@@ -5,9 +5,7 @@ import com.giantan.data.kvs.repository.GEntityConfig;
 import com.giantan.data.kvs.repository.GRepository;
 import com.giantan.data.mds.service.CollectionInstance;
 import com.giantan.data.mds.service.impl.MdCores;
-import com.giantan.data.qa.repository.QaDocRepository;
-import com.giantan.data.qa.repository.QaTaxonomyRepository;
-import com.giantan.data.qa.repository.QaIndexer;
+import com.giantan.data.qa.repository.*;
 import jakarta.annotation.PostConstruct;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
@@ -41,6 +39,12 @@ public class QaCollectionService implements IQaCollectionService {
     @Autowired
     QaIndexer qaIndexer;
 
+    @Autowired
+    QaTaskRepository qaTaskRepository;
+
+    @Autowired
+    QaExtraRepository qaExtraRepository;
+
     protected MdCores mdCores;
 
     public QaCollectionService() {
@@ -77,20 +81,37 @@ public class QaCollectionService implements IQaCollectionService {
                 List<GBaseKeyValue> r = collections.findByName(name);
                 if (r != null && r.size() > 0) {
                     GBaseKeyValue kv = r.get(0);
-                    //qaIndexer.init(kv);
-
                     instance = CollectionInstance.build(kv.getIntId(), kv.getGid(), kv.getName(), kv.get(GEntityConfig.ATTRIBUTES));
                     mdCores.put(name, instance);
                     return instance.getId();
+                }else{
+                    log.warn("Collection with name '" + name + "' not exist.");
+                    throw new ResponseStatusException(HttpStatus.NOT_FOUND,
+                            "Collection with name '" + name + "' does not exist.");
                 }
+            } catch (ResponseStatusException e) {
+                // 直接透传，不要覆盖
+                throw e;
             } catch (Throwable e) {
+                log.error("Failed to get collection by name={}", name, e);
+                throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
+                        "Error occurred while retrieving collection with name '" + name + "'", e);
 
             }
-            throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
-                    "Collection with name '" + name + "' not exist.");
+            //throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
+            //        "Collection with name '" + name + "' not exist.");
         }
     }
 
+    @Override
+    public String getCollectionName(int id) {
+        CollectionInstance instance = mdCores.getById(id);
+        if (instance != null) {
+            return instance.getName();
+        }
+        return null;
+    }
+
     public int getCollectionOrNew(String name) {
         CollectionInstance collection = mdCores.getByName(name);
         if (collection == null) {
@@ -182,22 +203,18 @@ public class QaCollectionService implements IQaCollectionService {
         return (Map<String, Object>) ret.get(GEntityConfig.ATTRIBUTES);
     }
 
-    public GBaseKeyValue removeCollectionAttribute(String name, List<String> keys) throws Throwable {
-//        GBaseKeyValue kv = getKvByName(name);
-//        if (kv == null) {
-//            return null;
-//        }
-//        GBaseKeyValue ret = collections.removeAttribute(kv.getIntId(), keys);
+    public Map<String, Object> removeCollectionAttribute(String name, List<String> keys) throws Throwable {
         int id = getCollectionId(name);
         GBaseKeyValue ret = collections.removeAttribute(id, keys);
         onCollectionUpdate(name, ret);
-        return ret;
+        return (Map<String, Object>) ret.get(GEntityConfig.ATTRIBUTES);
     }
 
     private void removeFromStore(String name) {
         CollectionInstance remove = mdCores.remove(name);
     }
 
+
     public long deleteCollection(String name) throws Throwable {
 
         GBaseKeyValue kv = getKvByName(name);
@@ -226,6 +243,18 @@ public class QaCollectionService implements IQaCollectionService {
             log.error(e.getMessage());
         }
 
+        try {
+            qaTaskRepository.deleteTable(id);
+        } catch (Exception e) {
+            log.error(e.getMessage());
+        }
+
+        try {
+            qaExtraRepository.deleteTable(id);
+        } catch (Exception e) {
+            log.error(e.getMessage());
+        }
+
         return deleted;
     }
 
@@ -249,6 +278,17 @@ public class QaCollectionService implements IQaCollectionService {
         } catch (Exception e) {
             log.error(e.getMessage());
         }
+        try {
+            qaTaskRepository.deleteAll(id);
+        } catch (Exception e) {
+            log.error(e.getMessage());
+        }
+
+        try {
+            qaExtraRepository.deleteAll(id);
+        } catch (Exception e) {
+            log.error(e.getMessage());
+        }
         return 1;
     }
 
@@ -276,7 +316,9 @@ public class QaCollectionService implements IQaCollectionService {
         qaRepository.createTable(id);
         qaTaxonomyRepository.createTable(id);
         qaIndexer.createCollection(id);
-        //
+        qaTaskRepository.createTable(id);
+        qaExtraRepository.createTable(id);
+
         return ret;
     }
 
@@ -291,7 +333,9 @@ public class QaCollectionService implements IQaCollectionService {
 
     @Override
     public Map<String, Object> getAttributesByName(String name) {
-        CollectionInstance instance = mdCores.getByName(name);
+        int collId = getCollectionId(name);
+        CollectionInstance instance = mdCores.getById(collId);
+        //CollectionInstance instance = mdCores.getByName(name);
         if (instance == null) {
             return null;
         }

server/src/main/java/com/giantan/data/qa/service/task/QaDynamicTaskRepository.java

@@ -0,0 +1,21 @@
+package com.giantan.data.qa.service.task;
+
+import com.giantan.data.tasks.repository.DynamicTaskRepository;
+import jakarta.annotation.PostConstruct;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public class QaDynamicTaskRepository extends DynamicTaskRepository {
+    private static final org.slf4j.Logger log
+            = org.slf4j.LoggerFactory.getLogger(QaDynamicTaskRepository.class);
+
+    public QaDynamicTaskRepository(JdbcTemplate jdbc) {
+        super(jdbc);
+    }
+
+    @PostConstruct
+    public void init() {
+        setSchema("qadb", "tasks_");
+    }
+}

server/src/main/java/com/giantan/data/qa/service/task/QaPersistentTaskService.java

@@ -0,0 +1,66 @@
+package com.giantan.data.qa.service.task;
+
+import com.giantan.data.qa.service.QaCollectionService;
+import com.giantan.data.tasks.IPersistentTaskService;
+import com.giantan.data.tasks.TaskContext;
+import com.giantan.data.tasks.repository.TaskConverter;
+import com.giantan.data.tasks.repository.TaskStatusHistory;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+//@Service
+public class QaPersistentTaskService implements IPersistentTaskService {
+
+    //@Autowired
+    QaCollectionService qaCollectionService;
+
+    //@Autowired
+    QaDynamicTaskRepository qaDynamicTaskRepository;
+
+    public QaPersistentTaskService() {
+    }
+
+    public QaPersistentTaskService(QaCollectionService qaCollectionService, QaDynamicTaskRepository qaDynamicTaskRepository) {
+        this.qaCollectionService = qaCollectionService;
+        this.qaDynamicTaskRepository = qaDynamicTaskRepository;
+    }
+
+    @Override
+    public int save(String coll, TaskContext task) {
+        int collId = qaCollectionService.getCollectionId(coll);
+        TaskStatusHistory history = TaskConverter.toHistory(task);
+        int saved = qaDynamicTaskRepository.save(Integer.toString(collId), history);
+        return saved;
+    }
+
+    @Override
+    public int updateTaskStatus(String coll, TaskContext task) {
+        int collId = qaCollectionService.getCollectionId(coll);
+        TaskStatusHistory history = TaskConverter.toHistory(task);
+        int saved = qaDynamicTaskRepository.updateTaskStatus(Integer.toString(collId), history);
+        return saved;
+    }
+
+    @Override
+    public List<TaskStatusHistory> getHistoryTasks(String coll, LocalDateTime startTime, LocalDateTime endTime, String status) {
+        int collId = qaCollectionService.getCollectionId(coll);
+        List<TaskStatusHistory> rs = qaDynamicTaskRepository.getHistoryTasks(Integer.toString(collId),startTime,endTime,status);
+        return rs;
+    }
+
+    @Override
+    public TaskStatusHistory find(String coll, String id) {
+        int collId = qaCollectionService.getCollectionId(coll);
+        TaskStatusHistory r = qaDynamicTaskRepository.find(Integer.toString(collId), id);
+        return r;
+    }
+
+    @Override
+    public int deleteHistoryTasks(String coll, LocalDateTime startTime, LocalDateTime endTime, String status) {
+        int collId = qaCollectionService.getCollectionId(coll);
+        int rs = qaDynamicTaskRepository.deleteHistoryTasks(Integer.toString(collId),startTime,endTime,status);
+        return rs;
+    }
+
+}

server/src/main/java/com/giantan/data/qa/service/task/QaTaskManager.java

@@ -0,0 +1,94 @@
+package com.giantan.data.qa.service.task;
+
+import com.giantan.data.index.IHybridSearch;
+import com.giantan.data.qa.service.QaCollectionService;
+import com.giantan.data.qa.service.QaDocsService;
+import com.giantan.data.tasks.ITaskHandler;
+import com.giantan.data.tasks.TaskEventListener;
+import com.giantan.data.tasks.TaskHandlerRegistry;
+import com.giantan.data.tasks.TaskManager;
+import com.google.common.eventbus.AsyncEventBus;
+import jakarta.annotation.PostConstruct;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+//task:
+//  executor:
+//    corePoolSize: 2
+//    maxPoolSize: 20
+//    queueCapacity: 40
+//    keepAliveSeconds: 60
+
+@Service
+public class QaTaskManager extends TaskManager {
+    @Value("${task.executor.corePoolSize}")
+    int corePoolSize = 5;
+    @Value("${task.executor.maxPoolSize}")
+    int maximumPoolSize = 20;
+    @Value("${task.executor.keepAliveSeconds}")
+    int keepAliveSeconds = 60;
+    @Value("${task.executor.queueCapacity}")
+    int queueCapacity = 40;
+
+    @Autowired
+    IHybridSearch hybridSearch;
+
+    QaDocsService qaDocsService;
+
+    @Autowired
+    QaCollectionService qaCollectionService;
+
+    @Autowired
+    QaDynamicTaskRepository qaDynamicTaskRepository;
+
+    public QaTaskManager() {
+
+    }
+
+    @PostConstruct
+    public void init() {
+        Executor executor = taskExecutor();
+        AsyncEventBus eventBus = new AsyncEventBus(executor);
+        setEventBus(eventBus);
+
+        QaPersistentTaskService persistentTaskService = new QaPersistentTaskService(qaCollectionService,qaDynamicTaskRepository);
+        setPersistentTaskService(persistentTaskService);
+
+        TaskHandlerRegistry taskHandlerRegistry = getTaskHandlerRegistry();
+
+        TaskEventListener listener = getTaskEventListener(taskHandlerRegistry, this);
+        eventBus.register(listener); // 注册监听器
+    }
+
+    public Executor taskExecutor() {
+        //return Executors.newFixedThreadPool(10);
+        ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
+        taskExecutor.setCorePoolSize(corePoolSize);
+        taskExecutor.setMaxPoolSize(maximumPoolSize);
+        taskExecutor.setQueueCapacity(queueCapacity);
+        taskExecutor.setKeepAliveSeconds(keepAliveSeconds);
+        taskExecutor.initialize();
+        return taskExecutor;
+    }
+
+    public TaskEventListener getTaskEventListener(TaskHandlerRegistry registry, TaskManager manager) {
+        TaskEventListener listener = new TaskEventListener(registry, manager);
+        return listener;
+    }
+
+    protected TaskHandlerRegistry getTaskHandlerRegistry(){
+        List<ITaskHandler> handlers = new ArrayList<>();
+        QasTaskHandler qasTaskHandler = new QasTaskHandler(qaDocsService);
+        handlers.add(qasTaskHandler);
+
+        TaskHandlerRegistry registry = new TaskHandlerRegistry(handlers);
+        return registry;
+    }
+
+}

server/src/main/java/com/giantan/data/tasks/IPersistentTaskService.java

@@ -13,4 +13,6 @@ public interface IPersistentTaskService {
     List<TaskStatusHistory> getHistoryTasks(String coll, LocalDateTime startTime, LocalDateTime endTime, String status);
 
     int deleteHistoryTasks(String coll, LocalDateTime startTime, LocalDateTime endTime, String status);
+
+    TaskStatusHistory find(String coll, String id);
 }

server/src/main/java/com/giantan/data/tasks/ITaskManager.java

@@ -33,5 +33,9 @@ public interface ITaskManager {
 
     List<TaskStatusHistory> getHistoryTasks(String coll, LocalDateTime startTime, LocalDateTime endTime, String status);
 
-    int deleteHistoryTasks(String collId, LocalDateTime startTime, LocalDateTime endTime, String status);
+    TaskStatusHistory getHistoryTask(String coll, String id);
+
+    int deleteHistoryTasks(String coll, LocalDateTime startTime, LocalDateTime endTime, String status);
+
+
 }

server/src/main/java/com/giantan/data/tasks/TaskManager.java

@@ -118,6 +118,12 @@ public class TaskManager implements ITaskManager {
         return rs;
     }
 
+    @Override
+    public TaskStatusHistory getHistoryTask(String coll, String id) {
+        TaskStatusHistory r = persistentTaskService.find(coll,id);
+        return r;
+    }
+
     @Override
     public int deleteHistoryTasks(String coll, LocalDateTime startTime, LocalDateTime endTime, String status) {
         int r =  persistentTaskService.deleteHistoryTasks(coll,startTime,endTime,status);

server/src/main/java/com/giantan/data/tasks/repository/DynamicTaskRepository.java

@@ -166,6 +166,17 @@ public class DynamicTaskRepository {
         }
     }
 
+    public TaskStatusHistory find(String collId, String taskId) {
+        String sql = String.format("SELECT * FROM %s WHERE task_id = ?", tableName(collId));
+        String[] params = new String[1];
+        params[0] = taskId;
+        List<TaskStatusHistory> rs = jdbc.query(sql.toString(), params, new TaskStatusHistoryRowMapper());
+        if (rs != null && rs.size() > 0) {
+            return rs.get(0);
+        }
+        return null;
+    }
+
     public List<TaskStatusHistory> getHistoryTasks(String collId, LocalDateTime startTime, LocalDateTime endTime, String status) {
         String sql1 = String.format("SELECT * FROM %s WHERE 1=1", tableName(collId));
         StringBuilder sql = new StringBuilder(sql1);

server/src/main/java/com/giantan/data/tasks/repository/PersistentTaskManager.java

@@ -91,6 +91,11 @@ public class PersistentTaskManager implements ITaskManager {
         return List.of();
     }
 
+    @Override
+    public TaskStatusHistory getHistoryTask(String collId, String id) {
+        return null;
+    }
+
     @Override
     public int deleteHistoryTasks(String collId, LocalDateTime startTime, LocalDateTime endTime, String status) {
         return 0;

server/src/test/java/com/giantan/data/mds/MdsApplicationTests.java

@@ -1,16 +1,18 @@
 package com.giantan.data.mds;
 
+import com.giantan.data.index.IndexUtils;
 import com.giantan.data.kvs.kvstore.GBaseKeyValue;
 import com.giantan.data.mds.bot.GChatClient;
 import com.giantan.data.index.HybridSearch;
 import com.giantan.data.mds.service.impl.MdChunksService;
-import com.giantan.data.mds.service.impl.Vectorization;
+import com.giantan.data.index.Vectorization;
 import com.giantan.data.qa.service.IQaDocsService;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -44,6 +46,28 @@ GLiNER 使用的是 DeBERTA v3 large，它对原始 DeBERTA 模型进行了更
 		Map<String, Object> kws = deepseek2.getKeywordsAndQuestions(s, Map.of());
 		System.out.println(kws);
 	}
+
+	@Test
+	public void testVectorization() throws IOException, InterruptedException {
+		String[] ss = {
+				"感冒可以艾灸大椎"
+		};
+		String[] ss2 = {
+				"人名","地名","机构名","药名","症状","穴位"
+		};
+
+
+		List<float[]> floats1 = vectorization.embed(Map.of("inputs",ss));
+		System.out.println(floats1);
+
+		List<float[]> floats2 = vectorization.embed(Map.of("inputs",ss2));
+
+		for (int i = 0; i < floats2.size(); i++) {
+			double v = IndexUtils.cosineSimilarity(floats2.get(i), floats1.get(0));
+			System.out.println(v+" "+ss2[i]);
+		}
+	}
+
 	@Test
 	void contextLoads() throws IOException, InterruptedException {
 		System.out.println("Hello World");

server/src/test/java/com/giantan/data/mds/QasTest.java

@@ -0,0 +1,27 @@
+package com.giantan.data.mds;
+
+import com.giantan.data.index.HybridSearch;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import java.io.IOException;
+
+@SpringBootTest
+public class QasTest {
+
+    @Autowired
+    HybridSearch hybridSearch;
+
+    @Test
+    public void createCollection() throws IOException, InterruptedException {
+        int r = hybridSearch.createCollection("domo21");
+        System.out.println("r="+r);
+    }
+
+    @Test
+    public void dropCollection() throws IOException, InterruptedException {
+        boolean b = hybridSearch.dropCollection("demo21");
+        System.out.println("b="+b);
+    }
+}