Add make clean in Makefile

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>

Makefile

@@ -127,6 +127,7 @@ install: all
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/master $(GOPATH)/bin/master
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/proxy $(GOPATH)/bin/proxy
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/writenode $(GOPATH)/bin/writenode
+	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/indexbuilder $(GOPATH)/bin/indexbuilder
 	@mkdir -p $(LIBRARY_PATH) && cp -f $(PWD)/internal/core/output/lib/* $(LIBRARY_PATH)
 	@echo "Installation successful."
 
@@ -134,7 +135,10 @@ clean:
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@find . -name '*~' | xargs rm -fv
-	@rm -rvf querynode
-	@rm -rvf master
-	@rm -rvf proxy
-	@rm -rvf writenode
+	@rm -rf bin/
+	@rm -rf lib/
+	@rm -rf $(GOPATH)/bin/master
+	@rm -rf $(GOPATH)/bin/proxy
+	@rm -rf $(GOPATH)/bin/querynode
+	@rm -rf $(GOPATH)/bin/writenode
+	@rm -rf $(GOPATH)/bin/indexbuilder

cmd/storage/benchmark.go

+package main
+
+import (
+	"context"
+	"crypto/md5"
+	"flag"
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pivotal-golang/bytefmt"
+	"github.com/zilliztech/milvus-distributed/internal/storage"
+	storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
+)
+
+// Global variables
+var durationSecs, threads, loops, numVersion, batchOpSize int
+var valueSize uint64
+var valueData []byte
+var batchValueData [][]byte
+var counter, totalKeyCount, keyNum int32
+var endTime, setFinish, getFinish, deleteFinish time.Time
+var totalKeys [][]byte
+
+var logFileName = "benchmark.log"
+var logFile *os.File
+
+var store storagetype.Store
+var wg sync.WaitGroup
+
+func runSet() {
+	for time.Now().Before(endTime) {
+		num := atomic.AddInt32(&keyNum, 1)
+		key := []byte(fmt.Sprint("key", num))
+		for ver := 1; ver <= numVersion; ver++ {
+			atomic.AddInt32(&counter, 1)
+			err := store.PutRow(context.Background(), key, valueData, "empty", uint64(ver))
+			if err != nil {
+				log.Fatalf("Error setting key %s, %s", key, err.Error())
+				//atomic.AddInt32(&setCount, -1)
+			}
+		}
+	}
+	// Remember last done time
+	setFinish = time.Now()
+	wg.Done()
+}
+
+func runBatchSet() {
+	for time.Now().Before(endTime) {
+		num := atomic.AddInt32(&keyNum, int32(batchOpSize))
+		keys := make([][]byte, batchOpSize)
+		versions := make([]uint64, batchOpSize)
+		batchSuffix := make([]string, batchOpSize)
+		for n := batchOpSize; n > 0; n-- {
+			keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
+		}
+		for ver := 1; ver <= numVersion; ver++ {
+			atomic.AddInt32(&counter, 1)
+			err := store.PutRows(context.Background(), keys, batchValueData, batchSuffix, versions)
+			if err != nil {
+				log.Fatalf("Error setting batch keys %s %s", keys, err.Error())
+				//atomic.AddInt32(&batchSetCount, -1)
+			}
+		}
+	}
+	setFinish = time.Now()
+	wg.Done()
+}
+
+func runGet() {
+	for time.Now().Before(endTime) {
+		num := atomic.AddInt32(&counter, 1)
+		//num := atomic.AddInt32(&keyNum, 1)
+		//key := []byte(fmt.Sprint("key", num))
+		num = num % totalKeyCount
+		key := totalKeys[num]
+		_, err := store.GetRow(context.Background(), key, uint64(numVersion))
+		if err != nil {
+			log.Fatalf("Error getting key %s, %s", key, err.Error())
+			//atomic.AddInt32(&getCount, -1)
+		}
+	}
+	// Remember last done time
+	getFinish = time.Now()
+	wg.Done()
+}
+
+func runBatchGet() {
+	for time.Now().Before(endTime) {
+		num := atomic.AddInt32(&keyNum, int32(batchOpSize))
+		//keys := make([][]byte, batchOpSize)
+		//for n := batchOpSize; n > 0; n-- {
+		//	keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
+		//}
+		end := num % totalKeyCount
+		if end < int32(batchOpSize) {
+			end = int32(batchOpSize)
+		}
+		start := end - int32(batchOpSize)
+		keys := totalKeys[start:end]
+		versions := make([]uint64, batchOpSize)
+		for i := range versions {
+			versions[i] = uint64(numVersion)
+		}
+		atomic.AddInt32(&counter, 1)
+		_, err := store.GetRows(context.Background(), keys, versions)
+		if err != nil {
+			log.Fatalf("Error getting key %s, %s", keys, err.Error())
+			//atomic.AddInt32(&batchGetCount, -1)
+		}
+	}
+	// Remember last done time
+	getFinish = time.Now()
+	wg.Done()
+}
+
+func runDelete() {
+	for time.Now().Before(endTime) {
+		num := atomic.AddInt32(&counter, 1)
+		//num := atomic.AddInt32(&keyNum, 1)
+		//key := []byte(fmt.Sprint("key", num))
+		num = num % totalKeyCount
+		key := totalKeys[num]
+		err := store.DeleteRow(context.Background(), key, uint64(numVersion))
+		if err != nil {
+			log.Fatalf("Error getting key %s, %s", key, err.Error())
+			//atomic.AddInt32(&deleteCount, -1)
+		}
+	}
+	// Remember last done time
+	deleteFinish = time.Now()
+	wg.Done()
+}
+
+func runBatchDelete() {
+	for time.Now().Before(endTime) {
+		num := atomic.AddInt32(&keyNum, int32(batchOpSize))
+		//keys := make([][]byte, batchOpSize)
+		//for n := batchOpSize; n > 0; n-- {
+		//	keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
+		//}
+		end := num % totalKeyCount
+		if end < int32(batchOpSize) {
+			end = int32(batchOpSize)
+		}
+		start := end - int32(batchOpSize)
+		keys := totalKeys[start:end]
+		atomic.AddInt32(&counter, 1)
+		versions := make([]uint64, batchOpSize)
+		for i := range versions {
+			versions[i] = uint64(numVersion)
+		}
+		err := store.DeleteRows(context.Background(), keys, versions)
+		if err != nil {
+			log.Fatalf("Error getting key %s, %s", keys, err.Error())
+			//atomic.AddInt32(&batchDeleteCount, -1)
+		}
+	}
+	// Remember last done time
+	getFinish = time.Now()
+	wg.Done()
+}
+
+func main() {
+	// Parse command line
+	myflag := flag.NewFlagSet("myflag", flag.ExitOnError)
+	myflag.IntVar(&durationSecs, "d", 5, "Duration of each test in seconds")
+	myflag.IntVar(&threads, "t", 1, "Number of threads to run")
+	myflag.IntVar(&loops, "l", 1, "Number of times to repeat test")
+	var sizeArg string
+	var storeType string
+	myflag.StringVar(&sizeArg, "z", "1k", "Size of objects in bytes with postfix K, M, and G")
+	myflag.StringVar(&storeType, "s", "s3", "Storage type, tikv or minio or s3")
+	myflag.IntVar(&numVersion, "v", 1, "Max versions for each key")
+	myflag.IntVar(&batchOpSize, "b", 100, "Batch operation kv pair number")
+
+	if err := myflag.Parse(os.Args[1:]); err != nil {
+		os.Exit(1)
+	}
+
+	// Check the arguments
+	var err error
+	if valueSize, err = bytefmt.ToBytes(sizeArg); err != nil {
+		log.Fatalf("Invalid -z argument for object size: %v", err)
+	}
+	var option = storagetype.Option{TikvAddress: "localhost:2379", Type: storeType, BucketName: "zilliz-hz"}
+
+	store, err = storage.NewStore(context.Background(), option)
+	if err != nil {
+		log.Fatalf("Error when creating storage " + err.Error())
+	}
+	logFile, err = os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777)
+	if err != nil {
+		log.Fatalf("Prepare log file error, " + err.Error())
+	}
+
+	// Echo the parameters
+	log.Printf("Benchmark log will write to file %s\n", logFile.Name())
+	fmt.Fprintf(logFile, "Parameters: duration=%d, threads=%d, loops=%d, valueSize=%s, batchSize=%d, versions=%d\n", durationSecs, threads, loops, sizeArg, batchOpSize, numVersion)
+	// Init test data
+	valueData = make([]byte, valueSize)
+	rand.Read(valueData)
+	hasher := md5.New()
+	hasher.Write(valueData)
+
+	batchValueData = make([][]byte, batchOpSize)
+	for i := range batchValueData {
+		batchValueData[i] = make([]byte, valueSize)
+		rand.Read(batchValueData[i])
+		hasher := md5.New()
+		hasher.Write(batchValueData[i])
+	}
+
+	// Loop running the tests
+	for loop := 1; loop <= loops; loop++ {
+
+		// reset counters
+		counter = 0
+		keyNum = 0
+		totalKeyCount = 0
+		totalKeys = nil
+
+		// Run the batchSet case
+		// key seq start from setCount
+		counter = 0
+		startTime := time.Now()
+		endTime = startTime.Add(time.Second * time.Duration(durationSecs))
+		for n := 1; n <= threads; n++ {
+			wg.Add(1)
+			go runBatchSet()
+		}
+		wg.Wait()
+
+		setTime := setFinish.Sub(startTime).Seconds()
+		bps := float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
+		fmt.Fprintf(logFile, "Loop %d: BATCH PUT time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
+			loop, setTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/setTime, float64(counter*int32(batchOpSize))/setTime)
+		// Record all test keys
+		//totalKeyCount = keyNum
+		//totalKeys = make([][]byte, totalKeyCount)
+		//for i := int32(0); i < totalKeyCount; i++ {
+		//	totalKeys[i] = []byte(fmt.Sprint("key", i))
+		//}
+		//
+		//// Run the get case
+		//counter = 0
+		//startTime = time.Now()
+		//endTime = startTime.Add(time.Second * time.Duration(durationSecs))
+		//for n := 1; n <= threads; n++ {
+		//	wg.Add(1)
+		//	go runGet()
+		//}
+		//wg.Wait()
+		//
+		//getTime := getFinish.Sub(startTime).Seconds()
+		//bps = float64(uint64(counter)*valueSize) / getTime
+		//fmt.Fprint(logFile, fmt.Sprintf("Loop %d: GET time %.1f secs, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
+		//	loop, getTime, counter, bytefmt.ByteSize(uint64(bps)), float64(counter)/getTime, float64(counter)/getTime))
+
+		// Run the batchGet case
+		//counter = 0
+		//startTime = time.Now()
+		//endTime = startTime.Add(time.Second * time.Duration(durationSecs))
+		//for n := 1; n <= threads; n++ {
+		//	wg.Add(1)
+		//	go runBatchGet()
+		//}
+		//wg.Wait()
+		//
+		//getTime = getFinish.Sub(startTime).Seconds()
+		//bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / getTime
+		//fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH GET time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
+		//	loop, getTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/getTime, float64(counter * int32(batchOpSize))/getTime))
+		//
+		//// Run the delete case
+		//counter = 0
+		//startTime = time.Now()
+		//endTime = startTime.Add(time.Second * time.Duration(durationSecs))
+		//for n := 1; n <= threads; n++ {
+		//	wg.Add(1)
+		//	go runDelete()
+		//}
+		//wg.Wait()
+		//
+		//deleteTime := deleteFinish.Sub(startTime).Seconds()
+		//bps = float64(uint64(counter)*valueSize) / deleteTime
+		//fmt.Fprint(logFile, fmt.Sprintf("Loop %d: Delete time %.1f secs, kv pairs = %d, %.1f operations/sec, %.1f kv/sec.\n",
+		//	loop, deleteTime, counter, float64(counter)/deleteTime, float64(counter)/deleteTime))
+		//
+		//// Run the batchDelete case
+		//counter = 0
+		//startTime = time.Now()
+		//endTime = startTime.Add(time.Second * time.Duration(durationSecs))
+		//for n := 1; n <= threads; n++ {
+		//	wg.Add(1)
+		//	go runBatchDelete()
+		//}
+		//wg.Wait()
+		//
+		//deleteTime = setFinish.Sub(startTime).Seconds()
+		//bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
+		//fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH DELETE time %.1f secs, batchs = %d, kv pairs = %d, %.1f operations/sec, %.1f kv/sec.\n",
+		//	loop, setTime, counter, counter*int32(batchOpSize), float64(counter)/setTime, float64(counter * int32(batchOpSize))/setTime))
+
+		// Print line mark
+		lineMark := "\n"
+		fmt.Fprint(logFile, lineMark)
+	}
+	log.Print("Benchmark test done.")
+}

internal/core/src/query/CMakeLists.txt

@@ -4,15 +4,13 @@ set(MILVUS_QUERY_SRCS
         generated/PlanNode.cpp
         generated/Expr.cpp
         visitors/ShowPlanNodeVisitor.cpp
-        visitors/ShowExprVisitor.cpp
         visitors/ExecPlanNodeVisitor.cpp
+        visitors/ShowExprVisitor.cpp
         visitors/ExecExprVisitor.cpp
-        visitors/VerifyPlanNodeVisitor.cpp
-        visitors/VerifyExprVisitor.cpp
         Plan.cpp
         Search.cpp
         SearchOnSealed.cpp
         BruteForceSearch.cpp
         )
 add_library(milvus_query ${MILVUS_QUERY_SRCS})
-target_link_libraries(milvus_query milvus_proto milvus_utils knowhere)
+target_link_libraries(milvus_query milvus_proto milvus_utils)

internal/core/src/query/Plan.cpp

@@ -21,7 +21,6 @@
 #include <boost/align/aligned_allocator.hpp>
 #include <boost/algorithm/string.hpp>
 #include <algorithm>
-#include "query/generated/VerifyPlanNodeVisitor.h"
 
 namespace milvus::query {
 
@@ -139,8 +138,6 @@ Parser::CreatePlanImpl(const std::string& dsl_str) {
     if (predicate != nullptr) {
         vec_node->predicate_ = std::move(predicate);
     }
-    VerifyPlanNodeVisitor verifier;
-    vec_node->accept(verifier);
 
     auto plan = std::make_unique<Plan>(schema);
     plan->tag2field_ = std::move(tag2field_);

internal/core/src/query/generated/ExecExprVisitor.h

@@ -21,7 +21,7 @@
 #include "ExprVisitor.h"
 
 namespace milvus::query {
-class ExecExprVisitor : public ExprVisitor {
+class ExecExprVisitor : ExprVisitor {
  public:
     void
     visit(BoolUnaryExpr& expr) override;

internal/core/src/query/generated/ExecPlanNodeVisitor.h

@@ -19,7 +19,7 @@
 #include "PlanNodeVisitor.h"
 
 namespace milvus::query {
-class ExecPlanNodeVisitor : public PlanNodeVisitor {
+class ExecPlanNodeVisitor : PlanNodeVisitor {
  public:
     void
     visit(FloatVectorANNS& node) override;

internal/core/src/query/generated/ShowExprVisitor.h

@@ -19,7 +19,7 @@
 #include "ExprVisitor.h"
 
 namespace milvus::query {
-class ShowExprVisitor : public ExprVisitor {
+class ShowExprVisitor : ExprVisitor {
  public:
     void
     visit(BoolUnaryExpr& expr) override;

internal/core/src/query/generated/ShowPlanNodeVisitor.h

@@ -20,7 +20,7 @@
 #include "PlanNodeVisitor.h"
 
 namespace milvus::query {
-class ShowPlanNodeVisitor : public PlanNodeVisitor {
+class ShowPlanNodeVisitor : PlanNodeVisitor {
  public:
     void
     visit(FloatVectorANNS& node) override;

internal/core/src/query/generated/VerifyExprVisitor.cpp

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#error TODO: copy this file out, and modify the content.
-#include "query/generated/VerifyExprVisitor.h"
-
-namespace milvus::query {
-void
-VerifyExprVisitor::visit(BoolUnaryExpr& expr) {
-    // TODO
-}
-
-void
-VerifyExprVisitor::visit(BoolBinaryExpr& expr) {
-    // TODO
-}
-
-void
-VerifyExprVisitor::visit(TermExpr& expr) {
-    // TODO
-}
-
-void
-VerifyExprVisitor::visit(RangeExpr& expr) {
-    // TODO
-}
-
-}  // namespace milvus::query

internal/core/src/query/generated/VerifyExprVisitor.h

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#pragma once
-// Generated File
-// DO NOT EDIT
-#include <optional>
-#include <boost/dynamic_bitset.hpp>
-#include <utility>
-#include <deque>
-#include "segcore/SegmentSmallIndex.h"
-#include "query/ExprImpl.h"
-#include "ExprVisitor.h"
-
-namespace milvus::query {
-class VerifyExprVisitor : public ExprVisitor {
- public:
-    void
-    visit(BoolUnaryExpr& expr) override;
-
-    void
-    visit(BoolBinaryExpr& expr) override;
-
-    void
-    visit(TermExpr& expr) override;
-
-    void
-    visit(RangeExpr& expr) override;
-
- public:
-};
-}  // namespace milvus::query

internal/core/src/query/generated/VerifyPlanNodeVisitor.cpp

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#error TODO: copy this file out, and modify the content.
-#include "query/generated/VerifyPlanNodeVisitor.h"
-
-namespace milvus::query {
-void
-VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
-    // TODO
-}
-
-void
-VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
-    // TODO
-}
-
-}  // namespace milvus::query

internal/core/src/query/generated/VerifyPlanNodeVisitor.h

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#pragma once
-// Generated File
-// DO NOT EDIT
-#include "utils/Json.h"
-#include "query/PlanImpl.h"
-#include "segcore/SegmentBase.h"
-#include <utility>
-#include "PlanNodeVisitor.h"
-
-namespace milvus::query {
-class VerifyPlanNodeVisitor : public PlanNodeVisitor {
- public:
-    void
-    visit(FloatVectorANNS& node) override;
-
-    void
-    visit(BinaryVectorANNS& node) override;
-
- public:
-    using RetType = QueryResult;
-    VerifyPlanNodeVisitor() = default;
-
- private:
-    std::optional<RetType> ret_;
-};
-}  // namespace milvus::query

internal/core/src/query/visitors/VerifyExprVisitor.cpp

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#include "query/generated/VerifyExprVisitor.h"
-
-namespace milvus::query {
-void
-VerifyExprVisitor::visit(BoolUnaryExpr& expr) {
-    // TODO
-}
-
-void
-VerifyExprVisitor::visit(BoolBinaryExpr& expr) {
-    // TODO
-}
-
-void
-VerifyExprVisitor::visit(TermExpr& expr) {
-    // TODO
-}
-
-void
-VerifyExprVisitor::visit(RangeExpr& expr) {
-    // TODO
-}
-
-}  // namespace milvus::query

internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#include "query/generated/VerifyPlanNodeVisitor.h"
-#include "knowhere/index/vector_index/ConfAdapterMgr.h"
-#include "segcore/SegmentSmallIndex.h"
-#include "knowhere/index/vector_index/ConfAdapter.h"
-#include "knowhere/index/vector_index/helpers/IndexParameter.h"
-
-namespace milvus::query {
-
-#if 1
-namespace impl {
-// THIS CONTAINS EXTRA BODY FOR VISITOR
-// WILL BE USED BY GENERATOR UNDER suvlim/core_gen/
-class VerifyPlanNodeVisitor : PlanNodeVisitor {
- public:
-    using RetType = QueryResult;
-    VerifyPlanNodeVisitor() = default;
-
- private:
-    std::optional<RetType> ret_;
-};
-}  // namespace impl
-#endif
-
-static knowhere::IndexType
-InferIndexType(const Json& search_params) {
-    // ivf -> nprobe
-    // nsg -> search_length
-    // hnsw/rhnsw/*pq/*sq -> ef
-    // annoy -> search_k
-    // ngtpanng / ngtonng -> max_search_edges / epsilon
-    static const std::map<std::string, knowhere::IndexType> key_list = [] {
-        std::map<std::string, knowhere::IndexType> list;
-        namespace ip = knowhere::IndexParams;
-        namespace ie = knowhere::IndexEnum;
-        list.emplace(ip::nprobe, ie::INDEX_FAISS_IVFFLAT);
-        list.emplace(ip::search_length, ie::INDEX_NSG);
-        list.emplace(ip::ef, ie::INDEX_HNSW);
-        list.emplace(ip::search_k, ie::INDEX_ANNOY);
-        list.emplace(ip::max_search_edges, ie::INDEX_NGTONNG);
-        list.emplace(ip::epsilon, ie::INDEX_NGTONNG);
-        return list;
-    }();
-    auto dbg_str = search_params.dump();
-    for (auto& kv : search_params.items()) {
-        std::string key = kv.key();
-        if (key_list.count(key)) {
-            return key_list.at(key);
-        }
-    }
-    PanicInfo("failed to infer index type");
-}
-
-void
-VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
-    auto& search_params = node.query_info_.search_params_;
-    auto inferred_type = InferIndexType(search_params);
-    auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(inferred_type);
-    auto index_mode = knowhere::IndexMode::MODE_CPU;
-
-    // mock the api, topk will be passed from placeholder
-    auto params_copy = search_params;
-    params_copy[knowhere::meta::TOPK] = 10;
-
-    // NOTE: the second parameter is not checked in knowhere, may be redundant
-    auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode);
-    AssertInfo(passed, "invalid search params");
-}
-
-void
-VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
-    // TODO
-}
-
-}  // namespace milvus::query

internal/core/src/segcore/CMakeLists.txt

@@ -24,6 +24,5 @@ target_link_libraries(milvus_segcore
         dl backtrace
         milvus_common
         milvus_query
-        milvus_utils
         )
 

internal/core/unittest/CMakeLists.txt

@@ -24,8 +24,10 @@ target_link_libraries(all_tests
         gtest_main
         milvus_segcore
         milvus_indexbuilder
+        knowhere
         log
         pthread
+        milvus_utils
         )
 
 install (TARGETS all_tests DESTINATION unittest)

internal/core/unittest/test_c_api.cpp

@@ -137,7 +137,7 @@ TEST(CApiTest, SearchTest) {
     auto offset = PreInsert(segment, N);
 
     auto ins_res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
-    ASSERT_EQ(ins_res.error_code, Success);
+    assert(ins_res.error_code == Success);
 
     const char* dsl_string = R"(
     {
@@ -176,11 +176,11 @@ TEST(CApiTest, SearchTest) {
     void* plan = nullptr;
 
     auto status = CreatePlan(collection, dsl_string, &plan);
-    ASSERT_EQ(status.error_code, Success);
+    assert(status.error_code == Success);
 
     void* placeholderGroup = nullptr;
     status = ParsePlaceholderGroup(plan, blob.data(), blob.length(), &placeholderGroup);
-    ASSERT_EQ(status.error_code, Success);
+    assert(status.error_code == Success);
 
     std::vector<CPlaceholderGroup> placeholderGroups;
     placeholderGroups.push_back(placeholderGroup);
@@ -189,7 +189,7 @@ TEST(CApiTest, SearchTest) {
 
     CQueryResult search_result;
     auto res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1, &search_result);
-    ASSERT_EQ(res.error_code, Success);
+    assert(res.error_code == Success);
 
     DeletePlan(plan);
     DeletePlaceholderGroup(placeholderGroup);

internal/indexbuilder/indexbuilder.go

@@ -11,6 +11,9 @@ import (
 
 	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+
 	"go.etcd.io/etcd/clientv3"
 
 	"github.com/zilliztech/milvus-distributed/internal/allocator"
@@ -68,16 +71,19 @@ func CreateBuilder(ctx context.Context) (*Builder, error) {
 
 	idAllocator, err := allocator.NewIDAllocator(b.loopCtx, Params.MasterAddress)
 
-	option := &miniokv.Option{
-		Address:           Params.MinIOAddress,
-		AccessKeyID:       Params.MinIOAccessKeyID,
-		SecretAccessKeyID: Params.MinIOSecretAccessKey,
-		UseSSL:            Params.MinIOUseSSL,
-		BucketName:        Params.MinioBucketName,
-		CreateBucket:      true,
+	minIOEndPoint := Params.MinIOAddress
+	minIOAccessKeyID := Params.MinIOAccessKeyID
+	minIOSecretAccessKey := Params.MinIOSecretAccessKey
+	minIOUseSSL := Params.MinIOUseSSL
+	minIOClient, err := minio.New(minIOEndPoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(minIOAccessKeyID, minIOSecretAccessKey, ""),
+		Secure: minIOUseSSL,
+	})
+	if err != nil {
+		return nil, err
 	}
 
-	b.kv, err = miniokv.NewMinIOKV(b.loopCtx, option)
+	b.kv, err = miniokv.NewMinIOKV(b.loopCtx, minIOClient, Params.MinioBucketName)
 	if err != nil {
 		return nil, err
 	}

internal/kv/minio/minio_kv.go

@@ -2,15 +2,11 @@ package miniokv
 
 import (
 	"context"
-	"fmt"
-
 	"io"
 	"log"
 	"strings"
 
 	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/zilliztech/milvus-distributed/internal/errors"
 )
 
 type MinIOKV struct {
@@ -19,46 +15,24 @@ type MinIOKV struct {
 	bucketName  string
 }
 
-type Option struct {
-	Address           string
-	AccessKeyID       string
-	BucketName        string
-	SecretAccessKeyID string
-	UseSSL            bool
-	CreateBucket      bool // when bucket not existed, create it
-}
-
-func NewMinIOKV(ctx context.Context, option *Option) (*MinIOKV, error) {
-	minIOClient, err := minio.New(option.Address, &minio.Options{
-		Creds:  credentials.NewStaticV4(option.AccessKeyID, option.SecretAccessKeyID, ""),
-		Secure: option.UseSSL,
-	})
-	if err != nil {
-		return nil, err
-	}
+// NewMinIOKV creates a new MinIO kv.
+func NewMinIOKV(ctx context.Context, client *minio.Client, bucketName string) (*MinIOKV, error) {
 
-	bucketExists, err := minIOClient.BucketExists(ctx, option.BucketName)
+	bucketExists, err := client.BucketExists(ctx, bucketName)
 	if err != nil {
 		return nil, err
 	}
 
-	if option.CreateBucket {
-		if !bucketExists {
-			err = minIOClient.MakeBucket(ctx, option.BucketName, minio.MakeBucketOptions{})
-			if err != nil {
-				return nil, err
-			}
-		}
-	} else {
-		if !bucketExists {
-			return nil, errors.New(fmt.Sprintf("Bucket %s not Existed.", option.BucketName))
+	if !bucketExists {
+		err = client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
+		if err != nil {
+			return nil, err
 		}
 	}
-
 	return &MinIOKV{
 		ctx:         ctx,
-		minioClient: minIOClient,
-		bucketName:  option.BucketName,
+		minioClient: client,
+		bucketName:  bucketName,
 	}, nil
 }
 

internal/kv/minio/minio_kv_test.go

@@ -5,6 +5,8 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 	"github.com/zilliztech/milvus-distributed/internal/util/paramtable"
 
@@ -13,31 +15,24 @@ import (
 
 var Params paramtable.BaseTable
 
-func newMinIOKVClient(ctx context.Context, bucketName string) (*miniokv.MinIOKV, error) {
+func TestMinIOKV_Load(t *testing.T) {
+	Params.Init()
 	endPoint, _ := Params.Load("_MinioAddress")
 	accessKeyID, _ := Params.Load("minio.accessKeyID")
 	secretAccessKey, _ := Params.Load("minio.secretAccessKey")
 	useSSLStr, _ := Params.Load("minio.useSSL")
-	useSSL, _ := strconv.ParseBool(useSSLStr)
-	option := &miniokv.Option{
-		Address:           endPoint,
-		AccessKeyID:       accessKeyID,
-		SecretAccessKeyID: secretAccessKey,
-		UseSSL:            useSSL,
-		BucketName:        bucketName,
-		CreateBucket:      true,
-	}
-	client, err := miniokv.NewMinIOKV(ctx, option)
-	return client, err
-}
-
-func TestMinIOKV_Load(t *testing.T) {
-	Params.Init()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
+	useSSL, _ := strconv.ParseBool(useSSLStr)
+
+	minioClient, err := minio.New(endPoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: useSSL,
+	})
+	assert.Nil(t, err)
 
 	bucketName := "fantastic-tech-test"
-	MinIOKV, err := newMinIOKVClient(ctx, bucketName)
+	MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
 	assert.Nil(t, err)
 	defer MinIOKV.RemoveWithPrefix("")
 
@@ -84,14 +79,25 @@ func TestMinIOKV_Load(t *testing.T) {
 }
 
 func TestMinIOKV_MultiSave(t *testing.T) {
-	Params.Init()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	bucketName := "fantastic-tech-test"
-	MinIOKV, err := newMinIOKVClient(ctx, bucketName)
+	Params.Init()
+	endPoint, _ := Params.Load("_MinioAddress")
+	accessKeyID, _ := Params.Load("minio.accessKeyID")
+	secretAccessKey, _ := Params.Load("minio.secretAccessKey")
+	useSSLStr, _ := Params.Load("minio.useSSL")
+	useSSL, _ := strconv.ParseBool(useSSLStr)
+
+	minioClient, err := minio.New(endPoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: useSSL,
+	})
 	assert.Nil(t, err)
 
+	bucketName := "fantastic-tech-test"
+	MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
+	assert.Nil(t, err)
 	defer MinIOKV.RemoveWithPrefix("")
 
 	err = MinIOKV.Save("key_1", "111")
@@ -111,13 +117,25 @@ func TestMinIOKV_MultiSave(t *testing.T) {
 }
 
 func TestMinIOKV_Remove(t *testing.T) {
-	Params.Init()
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	Params.Init()
+	endPoint, _ := Params.Load("_MinioAddress")
+	accessKeyID, _ := Params.Load("minio.accessKeyID")
+	secretAccessKey, _ := Params.Load("minio.secretAccessKey")
+	useSSLStr, _ := Params.Load("minio.useSSL")
+	useSSL, _ := strconv.ParseBool(useSSLStr)
+
+	minioClient, err := minio.New(endPoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: useSSL,
+	})
+	assert.Nil(t, err)
+
 	bucketName := "fantastic-tech-test"
-	MinIOKV, err := newMinIOKVClient(ctx, bucketName)
+	MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
 	assert.Nil(t, err)
 	defer MinIOKV.RemoveWithPrefix("")