增加了 upload 上传文件重名检测

server/pom.xml

@@ -9,7 +9,7 @@
         <version>1.0.0</version>
     </parent>
 
-    <version>3.0.1</version>
+    <version>3.0.3</version>
     <artifactId>mdserver</artifactId>
 
     <properties>

server/src/main/java/com/giantan/data/dk/repository/DkIndexer.java

@@ -398,7 +398,7 @@ public class DkIndexer extends HybridIndexer implements IDkIndexer {
         String gid = entity.getGid();
         int i = 0;
         try {
-            i = hybridSearch.deleteDocumentsByIdFilter(getMappedIndexName(collection), gid);
+            i = hybridSearch.deleteDocumentsByIdLike(getMappedIndexName(collection), gid);
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
@@ -412,7 +412,7 @@ public class DkIndexer extends HybridIndexer implements IDkIndexer {
         try {
             for (GTool e : entities) {
                 String gid = e.getGid();
-                count = count + hybridSearch.deleteDocumentsByIdFilter(getMappedIndexName(collection), gid);
+                count = count + hybridSearch.deleteDocumentsByIdLike(getMappedIndexName(collection), gid);
             }
         } catch (Exception e) {
             throw new RuntimeException(e);

server/src/main/java/com/giantan/data/index/HybridSearch.java

@@ -208,6 +208,11 @@ public class HybridSearch implements IHybridSearch {
         return 0;
     }
 
+    @Override
+    public int deleteDocumentsByIdLike(String coll, String filter) throws IOException, InterruptedException {
+        return deleteDocumentsByIdFilter(coll,filter+"%");
+    }
+
     @Override
     public int deleteDocumentsByFilter(String coll, Map<String, Object> query) throws IOException, InterruptedException {
         String js = JsonUtils.toJson(query);

server/src/main/java/com/giantan/data/index/IHybridSearch.java

@@ -26,6 +26,7 @@ public interface IHybridSearch {
 
     // 可以是  doc_id 也可以是 "doc_id%"，例如 01k2zv5r5f75cz9k305wb8d489%
     int deleteDocumentsByIdFilter(String coll, String filter) throws IOException, InterruptedException;
+    int deleteDocumentsByIdLike(String coll, String filter) throws IOException, InterruptedException;
 
     int deleteDocumentsByFilter(String coll, Map<String, Object> query) throws IOException, InterruptedException;
 

server/src/main/java/com/giantan/data/index/IndexConfig.java

@@ -1,5 +1,19 @@
 package com.giantan.data.index;
 
+/*
+    // 每个collection 一个索引库
+	"indexStrategy": {
+		"indexMode": "table2collection",
+		"collectionPrefix": "qas_"
+	}
+
+	// 共享一个索引库qascolls
+	"indexStrategy": {
+		"indexMode": "singleCollection",
+		"globalCollection": "qascolls"
+	}
+
+ */
 public class IndexConfig {
     //可配置项（用户可控）
     //字段选择：["name", "description", "tags", "attributes.color"]

server/src/main/java/com/giantan/data/mds/MdsApplication.java

@@ -16,7 +16,7 @@ public class MdsApplication {
 
     public static void main(String[] args) {
         SpringApplication.run(MdsApplication.class, args);
-        log.info("Mds server started. Version 3.0.1");
+        log.info("Mds server started. Version 3.0.3");
     }
 
 }

server/src/main/java/com/giantan/data/mds/controller/MdDocsController.java

@@ -15,6 +15,7 @@ import org.springframework.web.multipart.MultipartFile;
 
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.nio.file.FileAlreadyExistsException;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -85,6 +86,7 @@ public class MdDocsController {
 //    }
 
     //@PostMapping("/upload")
+    //objectPath 如果以"/"结尾就是path，否则是文件名
     @PostMapping("/mds")
     public R<Map<String, Object>> upload(@PathVariable String collId,
                                          @RequestParam("file") MultipartFile file,
@@ -94,11 +96,16 @@ public class MdDocsController {
         //System.out.println("taskId = " + taskId);
         //System.out.println("file = " + file.getOriginalFilename());
         log.info("上传文件: {}, taskId: {}", file.getOriginalFilename(), taskId);
-
-        //taskStatusManager.putProcessing(taskId, new TaskStatus(collId, taskId, "处理中", "", System.currentTimeMillis(), 0));
-        Map<String, Object> ret = mdDocsService.processAsyncDirect(collId, taskId, file, params);
-
-        return R.data(ret);
+        try {
+            Map<String, Object> ret = mdDocsService.processAsyncDirect(collId, taskId, file, params);
+            return R.data(ret);
+        } catch (FileAlreadyExistsException e) {
+            log.error("上传文件: {} 失败, taskId: {}", file.getOriginalFilename(), taskId);
+            return R.fail(409, "文件处理失败：" + e.getMessage());
+        } catch (Throwable e) {
+            log.error("任务 {} 失败", taskId, e);
+            return R.fail(500, "文件处理失败：" + e.getMessage());
+        }
     }
 
     @DeleteMapping("/mds/{gid}")
@@ -124,6 +131,14 @@ public class MdDocsController {
         return R.data(ret);
     }
 
+    @GetMapping("/mds/by-name")
+    public R<?> getByName(@PathVariable String collId, @RequestParam("name") String name
+    ) throws Throwable {
+        GBaseKeyValue ret = mdDocsService.findByName(collId, name);
+        return R.data(ret);
+    }
+
+
 //    @DeleteMapping("/mds")
 //    public R<Map<String, Object>> deleteByName(
 //            @PathVariable String collId,

server/src/main/java/com/giantan/data/mds/controller/TaxonomyController.java

@@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.lang.invoke.MethodHandles;
+import java.nio.file.FileAlreadyExistsException;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -78,20 +79,26 @@ public class TaxonomyController {
 
 
     @PostMapping("/{nodeId}/mds")
-    public R<Map<String, Object>> upload(@PathVariable String collName,
+    public R upload(@PathVariable String collName,
                                          @PathVariable String nodeId,
                                          @RequestParam("file") MultipartFile file,
                                          @RequestParam Map<String, String> params
-    ) throws Exception {
+    ) {
         String taskId = UUID.randomUUID().toString();
         //System.out.println("taskId = " + taskId);
         //System.out.println("file = " + file.getOriginalFilename());
         log.info("上传文件: {}, taskId: {}", file.getOriginalFilename(), taskId);
 
-        //taskStatusManager.putProcessing(taskId, new TaskStatus(collId, taskId, "处理中", "", System.currentTimeMillis(), 0));
-        Map<String, Object> ret = dynamicTaxonomyService.processAsyncDirect(collName, nodeId, taskId, file, params);
-
-        return R.data(ret);
+        try {
+            Map<String, Object> ret = dynamicTaxonomyService.processAsyncDirect(collName, nodeId, taskId, file, params);
+            return R.data(ret);
+        }catch (FileAlreadyExistsException e){
+            log.error("上传文件: {} 失败, taskId: {}", file.getOriginalFilename(), taskId);
+            return R.fail(409, "文件处理失败：" + e.getMessage());
+        } catch (Throwable e) {
+            log.error("任务 {} 失败", taskId, e);
+            return R.fail(500, "文件处理失败：" + e.getMessage());
+        }
     }
 
     @DeleteMapping("/{nodeId}/mds")

server/src/main/java/com/giantan/data/mds/service/IMdDocsService.java

@@ -9,7 +9,7 @@ import java.util.Map;
 
 public interface IMdDocsService {
     //@Async
-    Map<String, Object> processAsyncDirect(String coll, String taskId, MultipartFile file, Map<String, String> params);
+    Map<String, Object> processAsyncDirect(String coll, String taskId, MultipartFile file, Map<String, String> params) throws Throwable;
 
     int delete(String coll, String gid) throws Throwable;
 
@@ -27,6 +27,8 @@ public interface IMdDocsService {
 
     GBaseKeyValue findByMdid(String coll, String mdId) throws Throwable;
 
+    GBaseKeyValue findByName(String coll, String name) throws Throwable;
+
     Object getMetadataByKey(String coll, String mdId, String key) throws Throwable;
 
     Object getAttributes(String coll, String mdId) throws Throwable;

server/src/main/java/com/giantan/data/mds/service/impl/MdCollectionsService.java

@@ -314,6 +314,9 @@ public class MdCollectionsService {   //extends KvCollectionService
     public long deleteCollection(String name) throws Throwable {
 
         GBaseKeyValue kv = getKvByName(name);
+        if (kv == null) {
+            return 0;
+        }
         Integer intId = kv.getIntId();
         String id = kv.getId();
 
@@ -431,7 +434,7 @@ public class MdCollectionsService {   //extends KvCollectionService
         chunksRepository.createTable(id);
         extraRepository.createTable(id);
         taskRepository.createTable(id);
-        indexer.createCollection(id);
+        indexer.createCollection(ret.getName());
         return ret;
     }
 

server/src/main/java/com/giantan/data/mds/service/impl/MdDocsService.java

@@ -18,6 +18,7 @@ import org.apache.commons.io.IOUtils;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.FileAlreadyExistsException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -97,75 +98,66 @@ public class MdDocsService implements IMdDocsService {
     }
 
     @Override
-    public Map<String, Object> processAsyncDirect(String coll, String taskId, MultipartFile file, Map<String, String> params) {
+    public Map<String, Object> processAsyncDirect(String coll, String taskId, MultipartFile file, Map<String, String> params) throws Throwable {
 
         Map<String, Object> ret = new HashMap<>();
-        try {
-            String uri = params.get("objectPath");
-            String original = file.getOriginalFilename();
-            String p = normalizePath(uri);
-            //String objectPath = getObjectPath(uri, original);
-            String objectPath = null;
-            if (FileUtil.isEndsWithSeparator(p)) {
-                objectPath = p + original;
-            } else {
-                objectPath = p;
-                int i1 = p.lastIndexOf("/");
-                p = p.substring(0, i1 + 1);
-            }
-            int collId = mdCollectionsService.getCollectionOrNew(coll);
-
-            params.put("path", p);
-            params.put("name", objectPath);
-            //TaskStatus status = new TaskStatus(coll, taskId, "成功", "处理完成", System.currentTimeMillis(), System.currentTimeMillis());
-            //taskStatusManager.markSuccess(taskId, status);
-            ret.put("taskId", taskId);
-            ret.put("uploadFile", original);
-            //List<GBaseKeyValue> kvs = mdCollectionsService.findByName(coll, objectPath);
-            //List<GBaseKeyValue> kvs = collection.findByName(objectPath);
 
-            List<GBaseKeyValue> kvs = mdDynamicRepository.findByPath(Integer.toString(collId), objectPath);
-            boolean isNewFile = true;
-
-            if (kvs != null && kvs.size() > 0) {
-                isNewFile = false;
-                ret.put("gid", kvs.get(0).getGid());
+        String uri = params.get("objectPath");
+        String original = file.getOriginalFilename();
+        String p = normalizePath(uri);
+        //String objectPath = getObjectPath(uri, original);
+        String objectPath = null;
+        if (FileUtil.isEndsWithSeparator(p)) {
+            objectPath = p + original;
+        } else {
+            objectPath = p;
+            int i1 = p.lastIndexOf("/");
+            p = p.substring(0, i1 + 1);
+        }
+        int collId = mdCollectionsService.getCollectionOrNew(coll);
+
+        params.put("path", p);
+        params.put("name", objectPath);
+        //TaskStatus status = new TaskStatus(coll, taskId, "成功", "处理完成", System.currentTimeMillis(), System.currentTimeMillis());
+        //taskStatusManager.markSuccess(taskId, status);
+        ret.put("taskId", taskId);
+        ret.put("uploadFile", original);
+
+        //List<GBaseKeyValue> kvs = mdDynamicRepository.findByPath(Integer.toString(collId), objectPath);
+        List<GBaseKeyValue> kvs = mdDynamicRepository.findByName(Integer.toString(collId), objectPath);
+        boolean isNewFile = true;
+
+        if (kvs != null && kvs.size() > 0) {
+            isNewFile = false;
+            ret.put("gid", kvs.get(0).getGid());
+            log.info("任务 {} 的objectPath {} 已存在,metadata = {}", taskId, objectPath, kvs.get(0));
+            throw new FileAlreadyExistsException("文件已存在，请更换文件名");
+        } else {
+            ret.put("gid", taskId);
+        }
 
-                log.info("任务 {} 的objectPath {} 已存在,metadata = {}", taskId, objectPath, kvs.get(0));
-            } else {
-                ret.put("gid", taskId);
-            }
+        backupAndStore(coll, collId, taskId, file, params, isNewFile);
 
-            backupAndStore(coll, collId, taskId, file, params, isNewFile);
-        } catch (Throwable e) {
-            log.error("任务 {} 失败", taskId, e);
-        }
         return ret;
     }
 
     @Async
-    public void backupAndStore(String coll, int collId, String taskId, MultipartFile file, Map<String, String> params, boolean isNewFile) {
-        try {
-            String uri = params.get("objectPath");
-            String original = file.getOriginalFilename();
-            //String objectPath = gkbStorer.backupAndStore(file.getInputStream(), coll, original, uri);
-            String objectPath = gkbStorer.storeDirect(file.getInputStream(), coll, original, uri);
-            if (isNewFile) {
-                //params.put("name", objectPath);
-                params.put("gid", taskId);
-                //String objectPath2 = getObjectPath(uri, original);
-                //params.put("name", objectPath2);
-
-                //mdCollectionsService.createEntry(coll, toObjectMap(params));
-                Map<String, Object> data = toObjectMap(params);
-                mdDynamicRepository.save(Integer.toString(collId), GBaseKeyValue.build(data));
-            }
-            log.info("任务 {} 成功", taskId);
-        } catch (Throwable e) {
-            //TaskStatus status = new TaskStatus(coll, taskId, "失败", e.getMessage(), System.currentTimeMillis(), System.currentTimeMillis());
-            //taskStatusManager.markFailed(taskId, status);
-            log.error("任务 {} 失败", taskId, e);
-        }
+    public void backupAndStore(String coll, int collId, String taskId, MultipartFile file, Map<String, String> params, boolean isNewFile) throws Throwable {
+
+        String uri = params.get("objectPath");
+        String original = file.getOriginalFilename();
+        //String objectPath = gkbStorer.backupAndStore(file.getInputStream(), coll, original, uri);
+        String objectPath = gkbStorer.storeDirect(file.getInputStream(), coll, original, uri);
+        //if (isNewFile) {
+        params.put("gid", taskId);
+        //String objectPath2 = getObjectPath(uri, original);
+        //params.put("name", objectPath2);
+
+        Map<String, Object> data = toObjectMap(params);
+        mdDynamicRepository.save(Integer.toString(collId), GBaseKeyValue.build(data));
+        //}
+        log.info("任务 {} 成功", taskId);
+
     }
 
     private Map<String, Object> toObjectMap(Map<String, String> params) {
@@ -472,6 +464,16 @@ public class MdDocsService implements IMdDocsService {
         return r;
     }
 
+    @Override
+    public GBaseKeyValue findByName(String coll, String name) throws Throwable {
+        String collId = getStrOfCollId(coll);
+        List<GBaseKeyValue> rs = mdDynamicRepository.findByName(collId, name);
+        if (rs != null && rs.size() > 0) {
+            return rs.get(0);
+        }
+        return null;
+    }
+
     @Override
     public Object getMetadataByKey(String coll, String mdId, String key) throws Throwable {
         GBaseKeyValue kv = findByMdid(coll, mdId);
@@ -615,7 +617,7 @@ public class MdDocsService implements IMdDocsService {
         List<String> uids = mdDynamicChunkRepository.findUidsByMdId(collId, mdId);
         if (uids != null && !uids.isEmpty()) {
             //hybridSearch.delete(coll, uids);
-            mdIndexer.delete(coll,uids);
+            mdIndexer.delete(coll, uids);
             log.info("删除索引文件,mdid: {}", gid);
             long r = mdDynamicChunkRepository.deleteByMdId(collId, mdId);
             log.info("删除chunk文件,mdid: {}", gid);
@@ -636,7 +638,7 @@ public class MdDocsService implements IMdDocsService {
         List<String> uids = mdDynamicChunkRepository.findUidsByMdId(collId, mdId);
         if (uids != null && !uids.isEmpty()) {
             //hybridSearch.delete(coll, uids);
-            mdIndexer.delete(coll,uids);
+            mdIndexer.delete(coll, uids);
             log.info("删除索引文件,mdid: {}", gid);
             //long r = mdDynamicChunkRepository.deleteByMdId(collId, mdId);
             //log.info("删除chunk文件: {}", gid);

server/src/main/java/com/giantan/data/mds/service/impl/MdTaxonomyService.java

@@ -121,19 +121,13 @@ public class MdTaxonomyService {
         return ret;
     }
 
-    public Map<String, Object> processAsyncDirect(String coll, String nodeId, String taskId, MultipartFile file, Map<String, String> params) throws Exception {
-        try {
-            String collId = getStrOfCollId(coll);
-            int intId = getIntId(coll, nodeId);
-            String path = getPath(collId, intId);
-            params.put("objectPath", path);
-            Map<String, Object> ret = mdDocsService.processAsyncDirect(coll, taskId, file, params);
-            return ret;
-        } catch (Exception e) {
-
-        }
-        return null;
-
+    public Map<String, Object> processAsyncDirect(String coll, String nodeId, String taskId, MultipartFile file, Map<String, String> params) throws Throwable {
+        String collId = getStrOfCollId(coll);
+        int intId = getIntId(coll, nodeId);
+        String path = getPath(collId, intId);
+        params.put("objectPath", path);
+        Map<String, Object> ret = mdDocsService.processAsyncDirect(coll, taskId, file, params);
+        return ret;
     }
 
     public int deleteFolder(String coll, String nodeId, String taskId) throws Exception {

server/src/main/java/com/giantan/data/qa/repository/QaIndexer.java

@@ -314,7 +314,7 @@ public class QaIndexer extends HybridIndexer implements IIndexer {
         String gid = entity.getGid();
         int i = 0;
         try {
-            i = hybridSearch.deleteDocumentsByIdFilter(getMappedIndexName(collection), gid);
+            i = hybridSearch.deleteDocumentsByIdLike(getMappedIndexName(collection), gid);
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
@@ -328,7 +328,7 @@ public class QaIndexer extends HybridIndexer implements IIndexer {
         try {
             for (GEntity e : entities) {
                 String gid = e.getGid();
-                count = count + hybridSearch.deleteDocumentsByIdFilter(getMappedIndexName(collection), gid);
+                count = count + hybridSearch.deleteDocumentsByIdLike(getMappedIndexName(collection), gid);
             }
         } catch (Exception e) {
             throw new RuntimeException(e);