diff --git a/cmd/storage/benchmark.go b/cmd/storage/benchmark.go
deleted file mode 100644
index 6a7c06f9a209b25df74545d1f23f58b2d1c80594..0000000000000000000000000000000000000000
--- a/cmd/storage/benchmark.go
+++ /dev/null
@@ -1,315 +0,0 @@
-package main
-
-import (
- "context"
- "crypto/md5"
- "flag"
- "fmt"
- "log"
- "math/rand"
- "os"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/pivotal-golang/bytefmt"
- "github.com/zilliztech/milvus-distributed/internal/storage"
- storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-// Global variables
-var durationSecs, threads, loops, numVersion, batchOpSize int
-var valueSize uint64
-var valueData []byte
-var batchValueData [][]byte
-var counter, totalKeyCount, keyNum int32
-var endTime, setFinish, getFinish, deleteFinish time.Time
-var totalKeys [][]byte
-
-var logFileName = "benchmark.log"
-var logFile *os.File
-
-var store storagetype.Store
-var wg sync.WaitGroup
-
-func runSet() {
- for time.Now().Before(endTime) {
- num := atomic.AddInt32(&keyNum, 1)
- key := []byte(fmt.Sprint("key", num))
- for ver := 1; ver <= numVersion; ver++ {
- atomic.AddInt32(&counter, 1)
- err := store.PutRow(context.Background(), key, valueData, "empty", uint64(ver))
- if err != nil {
- log.Fatalf("Error setting key %s, %s", key, err.Error())
- //atomic.AddInt32(&setCount, -1)
- }
- }
- }
- // Remember last done time
- setFinish = time.Now()
- wg.Done()
-}
-
-func runBatchSet() {
- for time.Now().Before(endTime) {
- num := atomic.AddInt32(&keyNum, int32(batchOpSize))
- keys := make([][]byte, batchOpSize)
- versions := make([]uint64, batchOpSize)
- batchSuffix := make([]string, batchOpSize)
- for n := batchOpSize; n > 0; n-- {
- keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
- }
- for ver := 1; ver <= numVersion; ver++ {
- atomic.AddInt32(&counter, 1)
- err := store.PutRows(context.Background(), keys, batchValueData, batchSuffix, versions)
- if err != nil {
- log.Fatalf("Error setting batch keys %s %s", keys, err.Error())
- //atomic.AddInt32(&batchSetCount, -1)
- }
- }
- }
- setFinish = time.Now()
- wg.Done()
-}
-
-func runGet() {
- for time.Now().Before(endTime) {
- num := atomic.AddInt32(&counter, 1)
- //num := atomic.AddInt32(&keyNum, 1)
- //key := []byte(fmt.Sprint("key", num))
- num = num % totalKeyCount
- key := totalKeys[num]
- _, err := store.GetRow(context.Background(), key, uint64(numVersion))
- if err != nil {
- log.Fatalf("Error getting key %s, %s", key, err.Error())
- //atomic.AddInt32(&getCount, -1)
- }
- }
- // Remember last done time
- getFinish = time.Now()
- wg.Done()
-}
-
-func runBatchGet() {
- for time.Now().Before(endTime) {
- num := atomic.AddInt32(&keyNum, int32(batchOpSize))
- //keys := make([][]byte, batchOpSize)
- //for n := batchOpSize; n > 0; n-- {
- // keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
- //}
- end := num % totalKeyCount
- if end < int32(batchOpSize) {
- end = int32(batchOpSize)
- }
- start := end - int32(batchOpSize)
- keys := totalKeys[start:end]
- versions := make([]uint64, batchOpSize)
- for i := range versions {
- versions[i] = uint64(numVersion)
- }
- atomic.AddInt32(&counter, 1)
- _, err := store.GetRows(context.Background(), keys, versions)
- if err != nil {
- log.Fatalf("Error getting key %s, %s", keys, err.Error())
- //atomic.AddInt32(&batchGetCount, -1)
- }
- }
- // Remember last done time
- getFinish = time.Now()
- wg.Done()
-}
-
-func runDelete() {
- for time.Now().Before(endTime) {
- num := atomic.AddInt32(&counter, 1)
- //num := atomic.AddInt32(&keyNum, 1)
- //key := []byte(fmt.Sprint("key", num))
- num = num % totalKeyCount
- key := totalKeys[num]
- err := store.DeleteRow(context.Background(), key, uint64(numVersion))
- if err != nil {
- log.Fatalf("Error getting key %s, %s", key, err.Error())
- //atomic.AddInt32(&deleteCount, -1)
- }
- }
- // Remember last done time
- deleteFinish = time.Now()
- wg.Done()
-}
-
-func runBatchDelete() {
- for time.Now().Before(endTime) {
- num := atomic.AddInt32(&keyNum, int32(batchOpSize))
- //keys := make([][]byte, batchOpSize)
- //for n := batchOpSize; n > 0; n-- {
- // keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
- //}
- end := num % totalKeyCount
- if end < int32(batchOpSize) {
- end = int32(batchOpSize)
- }
- start := end - int32(batchOpSize)
- keys := totalKeys[start:end]
- atomic.AddInt32(&counter, 1)
- versions := make([]uint64, batchOpSize)
- for i := range versions {
- versions[i] = uint64(numVersion)
- }
- err := store.DeleteRows(context.Background(), keys, versions)
- if err != nil {
- log.Fatalf("Error getting key %s, %s", keys, err.Error())
- //atomic.AddInt32(&batchDeleteCount, -1)
- }
- }
- // Remember last done time
- getFinish = time.Now()
- wg.Done()
-}
-
-func main() {
- // Parse command line
- myflag := flag.NewFlagSet("myflag", flag.ExitOnError)
- myflag.IntVar(&durationSecs, "d", 5, "Duration of each test in seconds")
- myflag.IntVar(&threads, "t", 1, "Number of threads to run")
- myflag.IntVar(&loops, "l", 1, "Number of times to repeat test")
- var sizeArg string
- var storeType string
- myflag.StringVar(&sizeArg, "z", "1k", "Size of objects in bytes with postfix K, M, and G")
- myflag.StringVar(&storeType, "s", "s3", "Storage type, tikv or minio or s3")
- myflag.IntVar(&numVersion, "v", 1, "Max versions for each key")
- myflag.IntVar(&batchOpSize, "b", 100, "Batch operation kv pair number")
-
- if err := myflag.Parse(os.Args[1:]); err != nil {
- os.Exit(1)
- }
-
- // Check the arguments
- var err error
- if valueSize, err = bytefmt.ToBytes(sizeArg); err != nil {
- log.Fatalf("Invalid -z argument for object size: %v", err)
- }
- var option = storagetype.Option{TikvAddress: "localhost:2379", Type: storeType, BucketName: "zilliz-hz"}
-
- store, err = storage.NewStore(context.Background(), option)
- if err != nil {
- log.Fatalf("Error when creating storage " + err.Error())
- }
- logFile, err = os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777)
- if err != nil {
- log.Fatalf("Prepare log file error, " + err.Error())
- }
-
- // Echo the parameters
- log.Printf("Benchmark log will write to file %s\n", logFile.Name())
- fmt.Fprintf(logFile, "Parameters: duration=%d, threads=%d, loops=%d, valueSize=%s, batchSize=%d, versions=%d\n", durationSecs, threads, loops, sizeArg, batchOpSize, numVersion)
- // Init test data
- valueData = make([]byte, valueSize)
- rand.Read(valueData)
- hasher := md5.New()
- hasher.Write(valueData)
-
- batchValueData = make([][]byte, batchOpSize)
- for i := range batchValueData {
- batchValueData[i] = make([]byte, valueSize)
- rand.Read(batchValueData[i])
- hasher := md5.New()
- hasher.Write(batchValueData[i])
- }
-
- // Loop running the tests
- for loop := 1; loop <= loops; loop++ {
-
- // reset counters
- counter = 0
- keyNum = 0
- totalKeyCount = 0
- totalKeys = nil
-
- // Run the batchSet case
- // key seq start from setCount
- counter = 0
- startTime := time.Now()
- endTime = startTime.Add(time.Second * time.Duration(durationSecs))
- for n := 1; n <= threads; n++ {
- wg.Add(1)
- go runBatchSet()
- }
- wg.Wait()
-
- setTime := setFinish.Sub(startTime).Seconds()
- bps := float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
- fmt.Fprintf(logFile, "Loop %d: BATCH PUT time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
- loop, setTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/setTime, float64(counter*int32(batchOpSize))/setTime)
- // Record all test keys
- //totalKeyCount = keyNum
- //totalKeys = make([][]byte, totalKeyCount)
- //for i := int32(0); i < totalKeyCount; i++ {
- // totalKeys[i] = []byte(fmt.Sprint("key", i))
- //}
- //
- //// Run the get case
- //counter = 0
- //startTime = time.Now()
- //endTime = startTime.Add(time.Second * time.Duration(durationSecs))
- //for n := 1; n <= threads; n++ {
- // wg.Add(1)
- // go runGet()
- //}
- //wg.Wait()
- //
- //getTime := getFinish.Sub(startTime).Seconds()
- //bps = float64(uint64(counter)*valueSize) / getTime
- //fmt.Fprint(logFile, fmt.Sprintf("Loop %d: GET time %.1f secs, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
- // loop, getTime, counter, bytefmt.ByteSize(uint64(bps)), float64(counter)/getTime, float64(counter)/getTime))
-
- // Run the batchGet case
- //counter = 0
- //startTime = time.Now()
- //endTime = startTime.Add(time.Second * time.Duration(durationSecs))
- //for n := 1; n <= threads; n++ {
- // wg.Add(1)
- // go runBatchGet()
- //}
- //wg.Wait()
- //
- //getTime = getFinish.Sub(startTime).Seconds()
- //bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / getTime
- //fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH GET time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
- // loop, getTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/getTime, float64(counter * int32(batchOpSize))/getTime))
- //
- //// Run the delete case
- //counter = 0
- //startTime = time.Now()
- //endTime = startTime.Add(time.Second * time.Duration(durationSecs))
- //for n := 1; n <= threads; n++ {
- // wg.Add(1)
- // go runDelete()
- //}
- //wg.Wait()
- //
- //deleteTime := deleteFinish.Sub(startTime).Seconds()
- //bps = float64(uint64(counter)*valueSize) / deleteTime
- //fmt.Fprint(logFile, fmt.Sprintf("Loop %d: Delete time %.1f secs, kv pairs = %d, %.1f operations/sec, %.1f kv/sec.\n",
- // loop, deleteTime, counter, float64(counter)/deleteTime, float64(counter)/deleteTime))
- //
- //// Run the batchDelete case
- //counter = 0
- //startTime = time.Now()
- //endTime = startTime.Add(time.Second * time.Duration(durationSecs))
- //for n := 1; n <= threads; n++ {
- // wg.Add(1)
- // go runBatchDelete()
- //}
- //wg.Wait()
- //
- //deleteTime = setFinish.Sub(startTime).Seconds()
- //bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
- //fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH DELETE time %.1f secs, batchs = %d, kv pairs = %d, %.1f operations/sec, %.1f kv/sec.\n",
- // loop, setTime, counter, counter*int32(batchOpSize), float64(counter)/setTime, float64(counter * int32(batchOpSize))/setTime))
-
- // Print line mark
- lineMark := "\n"
- fmt.Fprint(logFile, lineMark)
- }
- log.Print("Benchmark test done.")
-}
diff --git a/deployments/docker/docker-compose.yml b/deployments/docker/docker-compose.yml
index 60bf5d9fff1f2a0aa47274becc742f778aaeb77a..0ae708a19ecb9ababafe5fcdb6bd5f9d5eac529e 100644
--- a/deployments/docker/docker-compose.yml
+++ b/deployments/docker/docker-compose.yml
@@ -36,6 +36,14 @@ services:
networks:
- milvus
+ jaeger:
+ image: jaegertracing/all-in-one:latest
+ ports:
+ - "6831:6831/udp"
+ - "16686:16686"
+ networks:
+ - milvus
+
networks:
milvus:
diff --git a/docker-compose.yml b/docker-compose.yml
index cba23befabc4c39314b179b3227a1d3570ee3c31..9f3599abb9a5b139323717b4e98e4a9d7e91b8f4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -83,5 +83,10 @@ services:
networks:
- milvus
+ jaeger:
+ image: jaegertracing/all-in-one:latest
+ networks:
+ - milvus
+
networks:
milvus:
diff --git a/go.mod b/go.mod
index 47afde2bff278667721c394a64396793788e92f2..bb426c8ba01c5a6bcfdd5778e54e5f1aba3f7a79 100644
--- a/go.mod
+++ b/go.mod
@@ -4,14 +4,17 @@ go 1.15
require (
code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
+ github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
github.com/apache/pulsar-client-go v0.1.1
- github.com/aws/aws-sdk-go v1.30.8
+ github.com/apache/thrift v0.13.0
+ github.com/aws/aws-sdk-go v1.30.8 // indirect
github.com/coreos/etcd v3.3.25+incompatible // indirect
- github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548
+ github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/frankban/quicktest v1.10.2 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/git-hooks/git-hooks v1.3.1 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+ github.com/golang/mock v1.3.1
github.com/golang/protobuf v1.3.2
github.com/google/btree v1.0.0
github.com/klauspost/compress v1.10.11 // indirect
@@ -20,12 +23,12 @@ require (
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/onsi/ginkgo v1.12.1 // indirect
github.com/onsi/gomega v1.10.0 // indirect
- github.com/opentracing/opentracing-go v1.2.0 // indirect
+ github.com/opentracing/opentracing-go v1.2.0
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 // indirect
github.com/pingcap/errors v0.11.4 // indirect
github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 // indirect
- github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48
+ github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
github.com/prometheus/client_golang v1.5.1 // indirect
github.com/prometheus/common v0.10.0 // indirect
github.com/prometheus/procfs v0.1.3 // indirect
@@ -35,7 +38,9 @@ require (
github.com/spf13/cast v1.3.0
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.6.1
- github.com/tikv/client-go v0.0.0-20200824032810-95774393107b
+ github.com/tikv/client-go v0.0.0-20200824032810-95774393107b // indirect
+ github.com/uber/jaeger-client-go v2.25.0+incompatible
+ github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
github.com/urfave/cli v1.22.5 // indirect
github.com/yahoo/athenz v1.9.16 // indirect
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
@@ -50,7 +55,7 @@ require (
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150 // indirect
google.golang.org/grpc v1.31.0
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
- gopkg.in/yaml.v2 v2.3.0
+ gopkg.in/yaml.v2 v2.3.0 // indirect
honnef.co/go/tools v0.0.1-2020.1.4 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)
diff --git a/go.sum b/go.sum
index eb4ef6b6a4059c712a42b92603ceb5d687117066..14c1fca608a146e91e855a8c730df2ad5684861a 100644
--- a/go.sum
+++ b/go.sum
@@ -15,6 +15,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw=
+github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
@@ -24,6 +26,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1C
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apache/pulsar-client-go v0.1.1 h1:v/kU+2ZCC6yFIcbZrFtWa9/nvVzVr18L+xYJUvZSxEQ=
github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2czVPb27D477YpyU=
+github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI=
+github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4=
github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI=
github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk=
@@ -117,6 +121,7 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18h
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -343,6 +348,7 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/protocolbuffers/protobuf v3.14.0+incompatible h1:8r0H76h/Q/lEnFFY60AuM23NOnaDMi6bd7zuboSYM+o=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -403,6 +409,12 @@ github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrX
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/uber/jaeger-client-go v1.6.0 h1:3+zLlq+4npI5fg8IsgAje3YsP7TcEdNzJScyqFIzxEQ=
+github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
+github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
+github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo=
+github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ=
+github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4=
diff --git a/internal/core/src/common/FieldMeta.h b/internal/core/src/common/FieldMeta.h
index 0adf92bd49b42d1c0945c19e19a7508a0ab57937..9486ed2524095fd9e9ddb31af26012227215ecbc 100644
--- a/internal/core/src/common/FieldMeta.h
+++ b/internal/core/src/common/FieldMeta.h
@@ -18,7 +18,7 @@
namespace milvus {
inline int
-field_sizeof(DataType data_type, int dim = 1) {
+datatype_sizeof(DataType data_type, int dim = 1) {
switch (data_type) {
case DataType::BOOL:
return sizeof(bool);
@@ -78,7 +78,7 @@ datatype_name(DataType data_type) {
}
inline bool
-field_is_vector(DataType datatype) {
+datatype_is_vector(DataType datatype) {
return datatype == DataType::VECTOR_BINARY || datatype == DataType::VECTOR_FLOAT;
}
@@ -119,9 +119,9 @@ struct FieldMeta {
int
get_sizeof() const {
if (is_vector()) {
- return field_sizeof(type_, get_dim());
+ return datatype_sizeof(type_, get_dim());
} else {
- return field_sizeof(type_, 1);
+ return datatype_sizeof(type_, 1);
}
}
diff --git a/internal/core/src/common/Schema.cpp b/internal/core/src/common/Schema.cpp
index 9b5a0c3b8a623d1bd87dea62e234d133792436c0..c7b91a7b4c03d2bd0d1873a2382bd25489944870 100644
--- a/internal/core/src/common/Schema.cpp
+++ b/internal/core/src/common/Schema.cpp
@@ -50,7 +50,7 @@ Schema::ParseFrom(const milvus::proto::schema::CollectionSchema& schema_proto) {
schema->primary_key_offset_opt_ = schema->size();
}
- if (field_is_vector(data_type)) {
+ if (datatype_is_vector(data_type)) {
auto type_map = RepeatedKeyValToMap(child.type_params());
auto index_map = RepeatedKeyValToMap(child.index_params());
if (!index_map.count("metric_type")) {
diff --git a/internal/core/src/common/Types.h b/internal/core/src/common/Types.h
index aee866d457978f6a57e8b4e2a7369a63f872e06c..9d36a9c505920e73270040413b3f9a0b11529c8b 100644
--- a/internal/core/src/common/Types.h
+++ b/internal/core/src/common/Types.h
@@ -14,13 +14,15 @@
#include <faiss/MetricType.h>
#include <string>
#include <boost/align/aligned_allocator.hpp>
+#include <memory>
#include <vector>
namespace milvus {
using Timestamp = uint64_t; // TODO: use TiKV-like timestamp
using engine::DataType;
using engine::FieldElementType;
-using engine::QueryResult;
+using engine::idx_t;
+
using MetricType = faiss::MetricType;
MetricType
@@ -39,4 +41,33 @@ constexpr std::false_type always_false{};
template <typename T>
using aligned_vector = std::vector<T, boost::alignment::aligned_allocator<T, 512>>;
+///////////////////////////////////////////////////////////////////////////////////////////////////
+struct QueryResult {
+ QueryResult() = default;
+ QueryResult(uint64_t num_queries, uint64_t topK) : topK_(topK), num_queries_(num_queries) {
+ auto count = get_row_count();
+ result_distances_.resize(count);
+ internal_seg_offsets_.resize(count);
+ }
+
+ [[nodiscard]] uint64_t
+ get_row_count() const {
+ return topK_ * num_queries_;
+ }
+
+ public:
+ uint64_t num_queries_;
+ uint64_t topK_;
+ uint64_t seg_id_;
+ std::vector<float> result_distances_;
+
+ public:
+ // TODO(gexi): utilize these field
+ std::vector<int64_t> internal_seg_offsets_;
+ std::vector<int64_t> result_offsets_;
+ std::vector<std::vector<char>> row_data_;
+};
+
+using QueryResultPtr = std::shared_ptr<QueryResult>;
+
} // namespace milvus
diff --git a/internal/core/src/indexbuilder/IndexWrapper.cpp b/internal/core/src/indexbuilder/IndexWrapper.cpp
index fcced635e0e508894a70995b68002310c7b8182f..5d95eabf3d97ba40fcb9ee646b480a9c5a280254 100644
--- a/internal/core/src/indexbuilder/IndexWrapper.cpp
+++ b/internal/core/src/indexbuilder/IndexWrapper.cpp
@@ -55,6 +55,7 @@ IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Con
}
auto stoi_closure = [](const std::string& s) -> int { return std::stoi(s); };
+ auto stof_closure = [](const std::string& s) -> int { return std::stof(s); };
/***************************** meta *******************************/
check_parameter<int>(conf, milvus::knowhere::meta::DIM, stoi_closure, std::nullopt);
@@ -88,7 +89,7 @@ IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Con
check_parameter<int>(conf, milvus::knowhere::IndexParams::edge_size, stoi_closure, std::nullopt);
/************************** NGT Search Params *****************************/
- check_parameter<int>(conf, milvus::knowhere::IndexParams::epsilon, stoi_closure, std::nullopt);
+ check_parameter<float>(conf, milvus::knowhere::IndexParams::epsilon, stof_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt);
/************************** NGT_PANNG Params *****************************/
@@ -274,6 +275,12 @@ IndexWrapper::QueryWithParam(const knowhere::DatasetPtr& dataset, const char* se
std::unique_ptr<IndexWrapper::QueryResult>
IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Config& conf) {
+ auto load_raw_data_closure = [&]() { LoadRawData(); }; // hide this pointer
+ auto index_type = get_index_type();
+ if (is_in_nm_list(index_type)) {
+ std::call_once(raw_data_loaded_, load_raw_data_closure);
+ }
+
auto res = index_->Query(dataset, conf, nullptr);
auto ids = res->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto distances = res->Get<float*>(milvus::knowhere::meta::DISTANCE);
@@ -291,5 +298,19 @@ IndexWrapper::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::Con
return std::move(query_res);
}
+void
+IndexWrapper::LoadRawData() {
+ auto index_type = get_index_type();
+ if (is_in_nm_list(index_type)) {
+ auto bs = index_->Serialize(config_);
+ auto bptr = std::make_shared<milvus::knowhere::Binary>();
+ auto deleter = [&](uint8_t*) {}; // avoid repeated deconstruction
+ bptr->data = std::shared_ptr<uint8_t[]>(static_cast<uint8_t*>(raw_data_.data()), deleter);
+ bptr->size = raw_data_.size();
+ bs.Append(RAW_DATA, bptr);
+ index_->Load(bs);
+ }
+}
+
} // namespace indexbuilder
} // namespace milvus
diff --git a/internal/core/src/indexbuilder/IndexWrapper.h b/internal/core/src/indexbuilder/IndexWrapper.h
index 65c6f149febf89bd30521e0478ba4eb2782b8583..16f2721712c655bff7b2e7d53a235e32ed1d6458 100644
--- a/internal/core/src/indexbuilder/IndexWrapper.h
+++ b/internal/core/src/indexbuilder/IndexWrapper.h
@@ -66,6 +66,9 @@ class IndexWrapper {
void
StoreRawData(const knowhere::DatasetPtr& dataset);
+ void
+ LoadRawData();
+
template <typename T>
void
check_parameter(knowhere::Config& conf,
@@ -92,6 +95,7 @@ class IndexWrapper {
milvus::json index_config_;
knowhere::Config config_;
std::vector<uint8_t> raw_data_;
+ std::once_flag raw_data_loaded_;
};
} // namespace indexbuilder
diff --git a/internal/core/src/query/CMakeLists.txt b/internal/core/src/query/CMakeLists.txt
index a1de1d4ed502053407f16f1fc6e107c163cda653..7488270fee4817935004d4754359bea7a4b6329b 100644
--- a/internal/core/src/query/CMakeLists.txt
+++ b/internal/core/src/query/CMakeLists.txt
@@ -4,13 +4,16 @@ set(MILVUS_QUERY_SRCS
generated/PlanNode.cpp
generated/Expr.cpp
visitors/ShowPlanNodeVisitor.cpp
- visitors/ExecPlanNodeVisitor.cpp
visitors/ShowExprVisitor.cpp
+ visitors/ExecPlanNodeVisitor.cpp
visitors/ExecExprVisitor.cpp
+ visitors/VerifyPlanNodeVisitor.cpp
+ visitors/VerifyExprVisitor.cpp
Plan.cpp
Search.cpp
SearchOnSealed.cpp
- BruteForceSearch.cpp
+ SearchBruteForce.cpp
+ SubQueryResult.cpp
)
add_library(milvus_query ${MILVUS_QUERY_SRCS})
-target_link_libraries(milvus_query milvus_proto milvus_utils)
+target_link_libraries(milvus_query milvus_proto milvus_utils knowhere)
diff --git a/internal/core/src/query/Plan.cpp b/internal/core/src/query/Plan.cpp
index 96653593516c15f98797a9f2d4c18e316b0161e1..d2a8bf04b33237bea0770dbd7d9a576e6838f4d0 100644
--- a/internal/core/src/query/Plan.cpp
+++ b/internal/core/src/query/Plan.cpp
@@ -21,6 +21,7 @@
#include <boost/align/aligned_allocator.hpp>
#include <boost/algorithm/string.hpp>
#include <algorithm>
+#include "query/generated/VerifyPlanNodeVisitor.h"
namespace milvus::query {
@@ -106,7 +107,7 @@ Parser::ParseRangeNode(const Json& out_body) {
auto field_name = out_iter.key();
auto body = out_iter.value();
auto data_type = schema[field_name].get_data_type();
- Assert(!field_is_vector(data_type));
+ Assert(!datatype_is_vector(data_type));
switch (data_type) {
case DataType::BOOL:
@@ -138,6 +139,8 @@ Parser::CreatePlanImpl(const std::string& dsl_str) {
if (predicate != nullptr) {
vec_node->predicate_ = std::move(predicate);
}
+ VerifyPlanNodeVisitor verifier;
+ vec_node->accept(verifier);
auto plan = std::make_unique<Plan>(schema);
plan->tag2field_ = std::move(tag2field_);
@@ -152,7 +155,7 @@ Parser::ParseTermNode(const Json& out_body) {
auto field_name = out_iter.key();
auto body = out_iter.value();
auto data_type = schema[field_name].get_data_type();
- Assert(!field_is_vector(data_type));
+ Assert(!datatype_is_vector(data_type));
switch (data_type) {
case DataType::BOOL: {
return ParseTermNodeImpl<bool>(field_name, body);
diff --git a/internal/core/src/query/Search.cpp b/internal/core/src/query/Search.cpp
index e1c0135f5f95cd62d641fb8635a2901f95f75a1b..35c9ec0c51eecfdda943654fadbbfa9566f7259d 100644
--- a/internal/core/src/query/Search.cpp
+++ b/internal/core/src/query/Search.cpp
@@ -16,7 +16,7 @@
#include <faiss/utils/distances.h>
#include "utils/tools.h"
-#include "query/BruteForceSearch.h"
+#include "query/SearchBruteForce.h"
namespace milvus::query {
@@ -34,13 +34,13 @@ create_bitmap_view(std::optional<const BitmapSimple*> bitmaps_opt, int64_t chunk
}
Status
-QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
- const query::QueryInfo& info,
- const float* query_data,
- int64_t num_queries,
- Timestamp timestamp,
- std::optional<const BitmapSimple*> bitmaps_opt,
- QueryResult& results) {
+FloatSearch(const segcore::SegmentSmallIndex& segment,
+ const query::QueryInfo& info,
+ const float* query_data,
+ int64_t num_queries,
+ Timestamp timestamp,
+ std::optional<const BitmapSimple*> bitmaps_opt,
+ QueryResult& results) {
auto& schema = segment.get_schema();
auto& indexing_record = segment.get_indexing_record();
auto& record = segment.get_insert_record();
@@ -75,6 +75,7 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
const auto& indexing_entry = indexing_record.get_vec_entry(vecfield_offset);
auto search_conf = indexing_entry.get_search_conf(topK);
+ // TODO: use sub_qr
for (int chunk_id = 0; chunk_id < max_indexed_id; ++chunk_id) {
auto indexing = indexing_entry.get_vec_indexing(chunk_id);
auto dataset = knowhere::GenDataset(num_queries, dim, query_data);
@@ -99,10 +100,12 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
Assert(vec_chunk_size == indexing_entry.get_chunk_size());
auto max_chunk = upper_div(ins_barrier, vec_chunk_size);
+ // TODO: use sub_qr
for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
std::vector<int64_t> buf_uids(total_count, -1);
std::vector<float> buf_dis(total_count, std::numeric_limits<float>::max());
+ // should be not visitable
faiss::float_maxheap_array_t buf = {(size_t)num_queries, (size_t)topK, buf_uids.data(), buf_dis.data()};
auto& chunk = vec_ptr->get_chunk(chunk_id);
@@ -112,6 +115,7 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
auto nsize = element_end - element_begin;
auto bitmap_view = create_bitmap_view(bitmaps_opt, chunk_id);
+ // TODO: make it wrapped
faiss::knn_L2sqr(query_data, chunk.data(), dim, num_queries, nsize, &buf, bitmap_view);
Assert(buf_uids.size() == total_count);
@@ -134,13 +138,13 @@ QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
}
Status
-BinaryQueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
- const query::QueryInfo& info,
- const uint8_t* query_data,
- int64_t num_queries,
- Timestamp timestamp,
- std::optional<const BitmapSimple*> bitmaps_opt,
- QueryResult& results) {
+BinarySearch(const segcore::SegmentSmallIndex& segment,
+ const query::QueryInfo& info,
+ const uint8_t* query_data,
+ int64_t num_queries,
+ Timestamp timestamp,
+ std::optional<const BitmapSimple*> bitmaps_opt,
+ QueryResult& results) {
auto& schema = segment.get_schema();
auto& indexing_record = segment.get_indexing_record();
auto& record = segment.get_insert_record();
@@ -169,8 +173,8 @@ BinaryQueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
auto total_count = topK * num_queries;
// step 3: small indexing search
- std::vector<int64_t> final_uids(total_count, -1);
- std::vector<float> final_dis(total_count, std::numeric_limits<float>::max());
+ // TODO: this is too intrusive
+ // TODO: use QuerySubResult instead
query::dataset::BinaryQueryDataset query_dataset{metric_type, num_queries, topK, code_size, query_data};
using segcore::BinaryVector;
@@ -181,30 +185,27 @@ BinaryQueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
auto vec_chunk_size = vec_ptr->get_chunk_size();
auto max_chunk = upper_div(ins_barrier, vec_chunk_size);
+ SubQueryResult final_result(num_queries, topK, metric_type);
for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
- std::vector<int64_t> buf_uids(total_count, -1);
- std::vector<float> buf_dis(total_count, std::numeric_limits<float>::max());
-
auto& chunk = vec_ptr->get_chunk(chunk_id);
auto element_begin = chunk_id * vec_chunk_size;
auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_chunk_size);
auto nsize = element_end - element_begin;
auto bitmap_view = create_bitmap_view(bitmaps_opt, chunk_id);
- BinarySearchBruteForce(query_dataset, chunk.data(), nsize, buf_dis.data(), buf_uids.data(), bitmap_view);
+ auto sub_result = BinarySearchBruteForce(query_dataset, chunk.data(), nsize, bitmap_view);
// convert chunk uid to segment uid
- for (auto& x : buf_uids) {
+ for (auto& x : sub_result.mutable_labels()) {
if (x != -1) {
x += chunk_id * vec_chunk_size;
}
}
-
- segcore::merge_into(num_queries, topK, final_dis.data(), final_uids.data(), buf_dis.data(), buf_uids.data());
+ final_result.merge(sub_result);
}
- results.result_distances_ = std::move(final_dis);
- results.internal_seg_offsets_ = std::move(final_uids);
+ results.result_distances_ = std::move(final_result.mutable_values());
+ results.internal_seg_offsets_ = std::move(final_result.mutable_labels());
results.topK_ = topK;
results.num_queries_ = num_queries;
diff --git a/internal/core/src/query/Search.h b/internal/core/src/query/Search.h
index a5334038ac570e50bd143cdc1d0731ffd2a1e009..f130b9aacae84e93e733b5436f359ff03f9b77f3 100644
--- a/internal/core/src/query/Search.h
+++ b/internal/core/src/query/Search.h
@@ -14,27 +14,29 @@
#include "segcore/SegmentSmallIndex.h"
#include <deque>
#include <boost/dynamic_bitset.hpp>
+#include "query/SubQueryResult.h"
namespace milvus::query {
using BitmapChunk = boost::dynamic_bitset<>;
using BitmapSimple = std::deque<BitmapChunk>;
+// TODO: merge these two search into one
// note: c++17 don't support optional ref
Status
-QueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
- const QueryInfo& info,
- const float* query_data,
- int64_t num_queries,
- Timestamp timestamp,
- std::optional<const BitmapSimple*> bitmap_opt,
- QueryResult& results);
+FloatSearch(const segcore::SegmentSmallIndex& segment,
+ const QueryInfo& info,
+ const float* query_data,
+ int64_t num_queries,
+ Timestamp timestamp,
+ std::optional<const BitmapSimple*> bitmap_opt,
+ QueryResult& results);
Status
-BinaryQueryBruteForceImpl(const segcore::SegmentSmallIndex& segment,
- const query::QueryInfo& info,
- const uint8_t* query_data,
- int64_t num_queries,
- Timestamp timestamp,
- std::optional<const BitmapSimple*> bitmaps_opt,
- QueryResult& results);
+BinarySearch(const segcore::SegmentSmallIndex& segment,
+ const query::QueryInfo& info,
+ const uint8_t* query_data,
+ int64_t num_queries,
+ Timestamp timestamp,
+ std::optional<const BitmapSimple*> bitmaps_opt,
+ QueryResult& results);
} // namespace milvus::query
diff --git a/internal/core/src/query/BruteForceSearch.cpp b/internal/core/src/query/SearchBruteForce.cpp
similarity index 62%
rename from internal/core/src/query/BruteForceSearch.cpp
rename to internal/core/src/query/SearchBruteForce.cpp
index de9e0143b1861895204405c55e93b3ae3bce99d4..4d46a1ee12038a7af90208b2e4a0554f48690680 100644
--- a/internal/core/src/query/BruteForceSearch.cpp
+++ b/internal/core/src/query/SearchBruteForce.cpp
@@ -9,58 +9,16 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
-#include "BruteForceSearch.h"
+#include "SearchBruteForce.h"
#include <vector>
#include <common/Types.h>
#include <boost/dynamic_bitset.hpp>
#include <queue>
+#include "SubQueryResult.h"
namespace milvus::query {
-void
-BinarySearchBruteForceNaive(MetricType metric_type,
- int64_t code_size,
- const uint8_t* binary_chunk,
- int64_t chunk_size,
- int64_t topk,
- int64_t num_queries,
- const uint8_t* query_data,
- float* result_distances,
- idx_t* result_labels,
- faiss::ConcurrentBitsetPtr bitset) {
- // THIS IS A NAIVE IMPLEMENTATION, ready for optimize
- Assert(metric_type == faiss::METRIC_Jaccard);
- Assert(code_size % 4 == 0);
-
- using T = std::tuple<float, int>;
-
- for (int64_t q = 0; q < num_queries; ++q) {
- auto query_ptr = query_data + code_size * q;
- auto query = boost::dynamic_bitset(query_ptr, query_ptr + code_size);
- std::vector<T> max_heap(topk + 1, std::make_tuple(std::numeric_limits<float>::max(), -1));
-
- for (int64_t i = 0; i < chunk_size; ++i) {
- auto element_ptr = binary_chunk + code_size * i;
- auto element = boost::dynamic_bitset(element_ptr, element_ptr + code_size);
- auto the_and = (query & element).count();
- auto the_or = (query | element).count();
- auto distance = the_or ? (float)(the_or - the_and) / the_or : 0;
- if (distance < std::get<0>(max_heap[0])) {
- max_heap[topk] = std::make_tuple(distance, i);
- std::push_heap(max_heap.begin(), max_heap.end());
- std::pop_heap(max_heap.begin(), max_heap.end());
- }
- }
- std::sort(max_heap.begin(), max_heap.end());
- for (int k = 0; k < topk; ++k) {
- auto info = max_heap[k];
- result_distances[k + q * topk] = std::get<0>(info);
- result_labels[k + q * topk] = std::get<1>(info);
- }
- }
-}
-
-void
+SubQueryResult
BinarySearchBruteForceFast(MetricType metric_type,
int64_t code_size,
const uint8_t* binary_chunk,
@@ -68,9 +26,11 @@ BinarySearchBruteForceFast(MetricType metric_type,
int64_t topk,
int64_t num_queries,
const uint8_t* query_data,
- float* result_distances,
- idx_t* result_labels,
- faiss::ConcurrentBitsetPtr bitset) {
+ const faiss::BitsetView& bitset) {
+ SubQueryResult sub_result(num_queries, topk, metric_type);
+ float* result_distances = sub_result.get_values();
+ idx_t* result_labels = sub_result.get_labels();
+
const idx_t block_size = chunk_size;
bool use_heap = true;
@@ -132,18 +92,26 @@ BinarySearchBruteForceFast(MetricType metric_type,
} else {
PanicInfo("Unsupported metric type");
}
+ return sub_result;
}
void
+FloatSearchBruteForceFast(MetricType metric_type,
+ const float* chunk_data,
+ int64_t chunk_size,
+ float* result_distances,
+ idx_t* result_labels,
+ const faiss::BitsetView& bitset) {
+ // TODO
+}
+
+SubQueryResult
BinarySearchBruteForce(const dataset::BinaryQueryDataset& query_dataset,
const uint8_t* binary_chunk,
int64_t chunk_size,
- float* result_distances,
- idx_t* result_labels,
- faiss::ConcurrentBitsetPtr bitset) {
+ const faiss::BitsetView& bitset) {
// TODO: refactor the internal function
- BinarySearchBruteForceFast(query_dataset.metric_type, query_dataset.code_size, binary_chunk, chunk_size,
- query_dataset.topk, query_dataset.num_queries, query_dataset.query_data,
- result_distances, result_labels, bitset);
+ return BinarySearchBruteForceFast(query_dataset.metric_type, query_dataset.code_size, binary_chunk, chunk_size,
+ query_dataset.topk, query_dataset.num_queries, query_dataset.query_data, bitset);
}
} // namespace milvus::query
diff --git a/internal/core/src/query/BruteForceSearch.h b/internal/core/src/query/SearchBruteForce.h
similarity index 87%
rename from internal/core/src/query/BruteForceSearch.h
rename to internal/core/src/query/SearchBruteForce.h
index 4d9cba96df3b84579d4a0c18c74b6a25cdba3546..d8114e19d6a13d8c48a57e8c34cd42f4ce68575e 100644
--- a/internal/core/src/query/BruteForceSearch.h
+++ b/internal/core/src/query/SearchBruteForce.h
@@ -13,6 +13,7 @@
#include <faiss/utils/BinaryDistance.h>
#include "segcore/ConcurrentVector.h"
#include "common/Schema.h"
+#include "query/SubQueryResult.h"
namespace milvus::query {
using MetricType = faiss::MetricType;
@@ -28,12 +29,10 @@ struct BinaryQueryDataset {
} // namespace dataset
-void
+SubQueryResult
BinarySearchBruteForce(const dataset::BinaryQueryDataset& query_dataset,
const uint8_t* binary_chunk,
int64_t chunk_size,
- float* result_distances,
- idx_t* result_labels,
- faiss::ConcurrentBitsetPtr bitset = nullptr);
+ const faiss::BitsetView& bitset = nullptr);
} // namespace milvus::query
diff --git a/internal/core/src/query/SubQueryResult.cpp b/internal/core/src/query/SubQueryResult.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..660d7b44f2cf2f11103972221b1a0ef1438cd5bc
--- /dev/null
+++ b/internal/core/src/query/SubQueryResult.cpp
@@ -0,0 +1,77 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "utils/EasyAssert.h"
+#include "query/SubQueryResult.h"
+#include "segcore/Reduce.h"
+
+namespace milvus::query {
+
+template <bool is_desc>
+void
+SubQueryResult::merge_impl(const SubQueryResult& right) {
+ Assert(num_queries_ == right.num_queries_);
+ Assert(topk_ == right.topk_);
+ Assert(metric_type_ == right.metric_type_);
+ Assert(is_desc == is_descending(metric_type_));
+
+ for (int64_t qn = 0; qn < num_queries_; ++qn) {
+ auto offset = qn * topk_;
+
+ int64_t* __restrict__ left_labels = this->get_labels() + offset;
+ float* __restrict__ left_values = this->get_values() + offset;
+
+ auto right_labels = right.get_labels() + offset;
+ auto right_values = right.get_values() + offset;
+
+ std::vector<float> buf_values(topk_);
+ std::vector<int64_t> buf_labels(topk_);
+
+ auto lit = 0; // left iter
+ auto rit = 0; // right iter
+
+ for (auto buf_iter = 0; buf_iter < topk_; ++buf_iter) {
+ auto left_v = left_values[lit];
+ auto right_v = right_values[rit];
+ // optimize out at compiling
+ if (is_desc ? (left_v >= right_v) : (left_v <= right_v)) {
+ buf_values[buf_iter] = left_values[lit];
+ buf_labels[buf_iter] = left_labels[lit];
+ ++lit;
+ } else {
+ buf_values[buf_iter] = right_values[rit];
+ buf_labels[buf_iter] = right_labels[rit];
+ ++rit;
+ }
+ }
+ std::copy_n(buf_values.data(), topk_, left_values);
+ std::copy_n(buf_labels.data(), topk_, left_labels);
+ }
+}
+
+void
+SubQueryResult::merge(const SubQueryResult& sub_result) {
+ Assert(metric_type_ == sub_result.metric_type_);
+ if (is_descending(metric_type_)) {
+ this->merge_impl<true>(sub_result);
+ } else {
+ this->merge_impl<false>(sub_result);
+ }
+}
+
+SubQueryResult
+SubQueryResult::merge(const SubQueryResult& left, const SubQueryResult& right) {
+ auto left_copy = left;
+ left_copy.merge(right);
+ return left_copy;
+}
+
+} // namespace milvus::query
diff --git a/internal/core/src/query/SubQueryResult.h b/internal/core/src/query/SubQueryResult.h
new file mode 100644
index 0000000000000000000000000000000000000000..6cf7aace5850cff86633218997316d4563cd791c
--- /dev/null
+++ b/internal/core/src/query/SubQueryResult.h
@@ -0,0 +1,98 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#pragma once
+#include "common/Types.h"
+#include <limits>
+#include <vector>
+namespace milvus::query {
+
+class SubQueryResult {
+ public:
+ SubQueryResult(int64_t num_queries, int64_t topk, MetricType metric_type)
+ : metric_type_(metric_type),
+ num_queries_(num_queries),
+ topk_(topk),
+ labels_(num_queries * topk, -1),
+ values_(num_queries * topk, init_value(metric_type)) {
+ }
+
+ public:
+ static constexpr float
+ init_value(MetricType metric_type) {
+ return (is_descending(metric_type) ? -1 : 1) * std::numeric_limits<float>::max();
+ }
+
+ static constexpr bool
+ is_descending(MetricType metric_type) {
+ // TODO
+ if (metric_type == MetricType::METRIC_INNER_PRODUCT) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public:
+ int64_t
+ get_num_queries() const {
+ return num_queries_;
+ }
+ int64_t
+ get_topk() const {
+ return topk_;
+ }
+
+ const int64_t*
+ get_labels() const {
+ return labels_.data();
+ }
+ int64_t*
+ get_labels() {
+ return labels_.data();
+ }
+ const float*
+ get_values() const {
+ return values_.data();
+ }
+ float*
+ get_values() {
+ return values_.data();
+ }
+ auto&
+ mutable_labels() {
+ return labels_;
+ }
+ auto&
+ mutable_values() {
+ return values_;
+ }
+
+ static SubQueryResult
+ merge(const SubQueryResult& left, const SubQueryResult& right);
+
+ void
+ merge(const SubQueryResult& sub_result);
+
+ private:
+ template <bool is_desc>
+ void
+ merge_impl(const SubQueryResult& sub_result);
+
+ private:
+ int64_t num_queries_;
+ int64_t topk_;
+ MetricType metric_type_;
+ std::vector<int64_t> labels_;
+ std::vector<float> values_;
+};
+
+} // namespace milvus::query
diff --git a/internal/core/src/query/generated/ExecExprVisitor.h b/internal/core/src/query/generated/ExecExprVisitor.h
index 250d68a6e567a52f9cf9c6bc8c7c886abf12f3f3..a9e0574a6e527a6f8fe5856ec640f15072824390 100644
--- a/internal/core/src/query/generated/ExecExprVisitor.h
+++ b/internal/core/src/query/generated/ExecExprVisitor.h
@@ -21,7 +21,7 @@
#include "ExprVisitor.h"
namespace milvus::query {
-class ExecExprVisitor : ExprVisitor {
+class ExecExprVisitor : public ExprVisitor {
public:
void
visit(BoolUnaryExpr& expr) override;
diff --git a/internal/core/src/query/generated/ExecPlanNodeVisitor.h b/internal/core/src/query/generated/ExecPlanNodeVisitor.h
index 0eb33384d71eec5e01a486014c27c1049e442c46..c026c689857958c27b44daf2c160b51592b26c44 100644
--- a/internal/core/src/query/generated/ExecPlanNodeVisitor.h
+++ b/internal/core/src/query/generated/ExecPlanNodeVisitor.h
@@ -19,7 +19,7 @@
#include "PlanNodeVisitor.h"
namespace milvus::query {
-class ExecPlanNodeVisitor : PlanNodeVisitor {
+class ExecPlanNodeVisitor : public PlanNodeVisitor {
public:
void
visit(FloatVectorANNS& node) override;
diff --git a/internal/core/src/query/generated/ShowExprVisitor.h b/internal/core/src/query/generated/ShowExprVisitor.h
index 55659e24c04e4a419a97b50ec39cfaffb1bcb558..6a1ed2646fc641b7670a0c7f100da9ed8408dc06 100644
--- a/internal/core/src/query/generated/ShowExprVisitor.h
+++ b/internal/core/src/query/generated/ShowExprVisitor.h
@@ -19,7 +19,7 @@
#include "ExprVisitor.h"
namespace milvus::query {
-class ShowExprVisitor : ExprVisitor {
+class ShowExprVisitor : public ExprVisitor {
public:
void
visit(BoolUnaryExpr& expr) override;
diff --git a/internal/core/src/query/generated/ShowPlanNodeVisitor.h b/internal/core/src/query/generated/ShowPlanNodeVisitor.h
index b921ec81fc5aa3eb29eb15c091a72738cfc57d4b..c518c3f7d0b23204f804c035db3471bcf08c4831 100644
--- a/internal/core/src/query/generated/ShowPlanNodeVisitor.h
+++ b/internal/core/src/query/generated/ShowPlanNodeVisitor.h
@@ -20,7 +20,7 @@
#include "PlanNodeVisitor.h"
namespace milvus::query {
-class ShowPlanNodeVisitor : PlanNodeVisitor {
+class ShowPlanNodeVisitor : public PlanNodeVisitor {
public:
void
visit(FloatVectorANNS& node) override;
diff --git a/internal/core/src/query/generated/VerifyExprVisitor.cpp b/internal/core/src/query/generated/VerifyExprVisitor.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..44af4dde81bdeea864b8bef34d064e4fbd4f2fee
--- /dev/null
+++ b/internal/core/src/query/generated/VerifyExprVisitor.cpp
@@ -0,0 +1,36 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#error TODO: copy this file out, and modify the content.
+#include "query/generated/VerifyExprVisitor.h"
+
+namespace milvus::query {
+void
+VerifyExprVisitor::visit(BoolUnaryExpr& expr) {
+ // TODO
+}
+
+void
+VerifyExprVisitor::visit(BoolBinaryExpr& expr) {
+ // TODO
+}
+
+void
+VerifyExprVisitor::visit(TermExpr& expr) {
+ // TODO
+}
+
+void
+VerifyExprVisitor::visit(RangeExpr& expr) {
+ // TODO
+}
+
+} // namespace milvus::query
diff --git a/internal/core/src/query/generated/VerifyExprVisitor.h b/internal/core/src/query/generated/VerifyExprVisitor.h
new file mode 100644
index 0000000000000000000000000000000000000000..6b04a76978d7db2c8247ac45399b50b85f44309d
--- /dev/null
+++ b/internal/core/src/query/generated/VerifyExprVisitor.h
@@ -0,0 +1,40 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include <optional>
+#include <boost/dynamic_bitset.hpp>
+#include <utility>
+#include <deque>
+#include "segcore/SegmentSmallIndex.h"
+#include "query/ExprImpl.h"
+#include "ExprVisitor.h"
+
+namespace milvus::query {
+class VerifyExprVisitor : public ExprVisitor {
+ public:
+ void
+ visit(BoolUnaryExpr& expr) override;
+
+ void
+ visit(BoolBinaryExpr& expr) override;
+
+ void
+ visit(TermExpr& expr) override;
+
+ void
+ visit(RangeExpr& expr) override;
+
+ public:
+};
+} // namespace milvus::query
diff --git a/internal/core/src/query/generated/VerifyPlanNodeVisitor.cpp b/internal/core/src/query/generated/VerifyPlanNodeVisitor.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c7b0656f57041427e8159e899f891ca0f391c901
--- /dev/null
+++ b/internal/core/src/query/generated/VerifyPlanNodeVisitor.cpp
@@ -0,0 +1,26 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#error TODO: copy this file out, and modify the content.
+#include "query/generated/VerifyPlanNodeVisitor.h"
+
+namespace milvus::query {
+void
+VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
+ // TODO
+}
+
+void
+VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
+ // TODO
+}
+
+} // namespace milvus::query
diff --git a/internal/core/src/query/generated/VerifyPlanNodeVisitor.h b/internal/core/src/query/generated/VerifyPlanNodeVisitor.h
new file mode 100644
index 0000000000000000000000000000000000000000..a964e6c08f920bcf3b2b1f4e7f70ebbddb0e264a
--- /dev/null
+++ b/internal/core/src/query/generated/VerifyPlanNodeVisitor.h
@@ -0,0 +1,37 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "utils/Json.h"
+#include "query/PlanImpl.h"
+#include "segcore/SegmentBase.h"
+#include <utility>
+#include "PlanNodeVisitor.h"
+
+namespace milvus::query {
+class VerifyPlanNodeVisitor : public PlanNodeVisitor {
+ public:
+ void
+ visit(FloatVectorANNS& node) override;
+
+ void
+ visit(BinaryVectorANNS& node) override;
+
+ public:
+ using RetType = QueryResult;
+ VerifyPlanNodeVisitor() = default;
+
+ private:
+ std::optional<RetType> ret_;
+};
+} // namespace milvus::query
diff --git a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
index 6409f248ac89750704c3fb448775e9f5508e379f..c7cb08a247edd2d8edfbce95625db37a71a66958 100644
--- a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
+++ b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
@@ -79,7 +79,7 @@ ExecPlanNodeVisitor::visit(FloatVectorANNS& node) {
SearchOnSealed(segment->get_schema(), sealed_indexing, node.query_info_, src_data, num_queries, timestamp_,
bitset_pack, ret);
} else {
- QueryBruteForceImpl(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
+ FloatSearch(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
}
ret_ = ret;
@@ -104,7 +104,7 @@ ExecPlanNodeVisitor::visit(BinaryVectorANNS& node) {
bitset_pack = &bitmap_holder;
}
- BinaryQueryBruteForceImpl(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
+ BinarySearch(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
ret_ = ret;
}
diff --git a/internal/core/src/query/visitors/ShowExprVisitor.cpp b/internal/core/src/query/visitors/ShowExprVisitor.cpp
index 79ae412c7a140c1041d0ad3b727b8f4dfc746a95..c9c52718546bfd9a209bf29ac84af3d8cd8af181 100644
--- a/internal/core/src/query/visitors/ShowExprVisitor.cpp
+++ b/internal/core/src/query/visitors/ShowExprVisitor.cpp
@@ -112,7 +112,7 @@ TermExtract(const TermExpr& expr_raw) {
void
ShowExprVisitor::visit(TermExpr& expr) {
Assert(!ret_.has_value());
- Assert(field_is_vector(expr.data_type_) == false);
+ Assert(datatype_is_vector(expr.data_type_) == false);
auto terms = [&] {
switch (expr.data_type_) {
case DataType::BOOL:
@@ -161,7 +161,7 @@ ConditionExtract(const RangeExpr& expr_raw) {
void
ShowExprVisitor::visit(RangeExpr& expr) {
Assert(!ret_.has_value());
- Assert(field_is_vector(expr.data_type_) == false);
+ Assert(datatype_is_vector(expr.data_type_) == false);
auto conditions = [&] {
switch (expr.data_type_) {
case DataType::BOOL:
diff --git a/internal/core/src/query/visitors/VerifyExprVisitor.cpp b/internal/core/src/query/visitors/VerifyExprVisitor.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9b3326c74a60b9f604de92a24fb71c4156a60cad
--- /dev/null
+++ b/internal/core/src/query/visitors/VerifyExprVisitor.cpp
@@ -0,0 +1,35 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "query/generated/VerifyExprVisitor.h"
+
+namespace milvus::query {
+void
+VerifyExprVisitor::visit(BoolUnaryExpr& expr) {
+ // TODO
+}
+
+void
+VerifyExprVisitor::visit(BoolBinaryExpr& expr) {
+ // TODO
+}
+
+void
+VerifyExprVisitor::visit(TermExpr& expr) {
+ // TODO
+}
+
+void
+VerifyExprVisitor::visit(RangeExpr& expr) {
+ // TODO
+}
+
+} // namespace milvus::query
diff --git a/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp b/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..263390a39a52831c0eb7f1bd71f8dc0cc1cecdb5
--- /dev/null
+++ b/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
@@ -0,0 +1,85 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "query/generated/VerifyPlanNodeVisitor.h"
+#include "knowhere/index/vector_index/ConfAdapterMgr.h"
+#include "segcore/SegmentSmallIndex.h"
+#include "knowhere/index/vector_index/ConfAdapter.h"
+#include "knowhere/index/vector_index/helpers/IndexParameter.h"
+
+namespace milvus::query {
+
+#if 1
+namespace impl {
+// THIS CONTAINS EXTRA BODY FOR VISITOR
+// WILL BE USED BY GENERATOR UNDER suvlim/core_gen/
+class VerifyPlanNodeVisitor : PlanNodeVisitor {
+ public:
+ using RetType = QueryResult;
+ VerifyPlanNodeVisitor() = default;
+
+ private:
+ std::optional<RetType> ret_;
+};
+} // namespace impl
+#endif
+
+static knowhere::IndexType
+InferIndexType(const Json& search_params) {
+ // ivf -> nprobe
+ // nsg -> search_length
+ // hnsw/rhnsw/*pq/*sq -> ef
+ // annoy -> search_k
+ // ngtpanng / ngtonng -> max_search_edges / epsilon
+ static const std::map<std::string, knowhere::IndexType> key_list = [] {
+ std::map<std::string, knowhere::IndexType> list;
+ namespace ip = knowhere::IndexParams;
+ namespace ie = knowhere::IndexEnum;
+ list.emplace(ip::nprobe, ie::INDEX_FAISS_IVFFLAT);
+ list.emplace(ip::search_length, ie::INDEX_NSG);
+ list.emplace(ip::ef, ie::INDEX_HNSW);
+ list.emplace(ip::search_k, ie::INDEX_ANNOY);
+ list.emplace(ip::max_search_edges, ie::INDEX_NGTONNG);
+ list.emplace(ip::epsilon, ie::INDEX_NGTONNG);
+ return list;
+ }();
+ auto dbg_str = search_params.dump();
+ for (auto& kv : search_params.items()) {
+ std::string key = kv.key();
+ if (key_list.count(key)) {
+ return key_list.at(key);
+ }
+ }
+ PanicInfo("failed to infer index type");
+}
+
+void
+VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
+ auto& search_params = node.query_info_.search_params_;
+ auto inferred_type = InferIndexType(search_params);
+ auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(inferred_type);
+ auto index_mode = knowhere::IndexMode::MODE_CPU;
+
+ // mock the api, topk will be passed from placeholder
+ auto params_copy = search_params;
+ params_copy[knowhere::meta::TOPK] = 10;
+
+ // NOTE: the second parameter is not checked in knowhere, may be redundant
+ auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode);
+ AssertInfo(passed, "invalid search params");
+}
+
+void
+VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
+ // TODO
+}
+
+} // namespace milvus::query
diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt
index 709a983c977aceacc0c049b070887044701840f6..1a011c984b690d92e62dc6bb172aa245cbfd140f 100644
--- a/internal/core/src/segcore/CMakeLists.txt
+++ b/internal/core/src/segcore/CMakeLists.txt
@@ -24,5 +24,6 @@ target_link_libraries(milvus_segcore
dl backtrace
milvus_common
milvus_query
+ milvus_utils
)
diff --git a/internal/core/src/segcore/SegmentBase.h b/internal/core/src/segcore/SegmentBase.h
index 57918d96a1203c6565be5446a22f457ea7d05f04..a7d427d66517e3ea10451e1c70d5130b9acbb33e 100644
--- a/internal/core/src/segcore/SegmentBase.h
+++ b/internal/core/src/segcore/SegmentBase.h
@@ -13,7 +13,7 @@
#include <vector>
#include "IndexMeta.h"
-#include "utils/Types.h"
+#include "common/Types.h"
#include "common/Schema.h"
#include <memory>
diff --git a/internal/core/src/segcore/SegmentNaive.cpp b/internal/core/src/segcore/SegmentNaive.cpp
index f20c0ce4ab5d4e1e9b4c606708d4b7f634775786..b513247c7bc4c01e9d2a00080668c0fc4caf7784 100644
--- a/internal/core/src/segcore/SegmentNaive.cpp
+++ b/internal/core/src/segcore/SegmentNaive.cpp
@@ -274,19 +274,14 @@ SegmentNaive::QueryImpl(query::QueryDeprecatedPtr query_info, Timestamp timestam
auto distances = final->Get<float*>(knowhere::meta::DISTANCE);
auto total_num = num_queries * topK;
- result.result_ids_.resize(total_num);
result.result_distances_.resize(total_num);
result.num_queries_ = num_queries;
result.topK_ = topK;
- std::copy_n(ids, total_num, result.result_ids_.data());
+ std::copy_n(ids, total_num, result.internal_seg_offsets_.data());
std::copy_n(distances, total_num, result.result_distances_.data());
- for (auto& id : result.result_ids_) {
- id = record_.uids_[id];
- }
-
return Status::OK();
}
@@ -347,7 +342,7 @@ SegmentNaive::QuerySlowImpl(query::QueryDeprecatedPtr query_info, Timestamp time
result.topK_ = topK;
auto row_num = topK * num_queries;
- result.result_ids_.resize(row_num);
+ result.internal_seg_offsets_.resize(row_num);
result.result_distances_.resize(row_num);
for (int q_id = 0; q_id < num_queries; ++q_id) {
@@ -356,7 +351,7 @@ SegmentNaive::QuerySlowImpl(query::QueryDeprecatedPtr query_info, Timestamp time
auto dst_id = topK - 1 - i + q_id * topK;
auto [dis, offset] = records[q_id].top();
records[q_id].pop();
- result.result_ids_[dst_id] = record_.uids_[offset];
+ result.internal_seg_offsets_[dst_id] = offset;
result.result_distances_[dst_id] = dis;
}
}
diff --git a/internal/core/src/segcore/SegmentSmallIndex.cpp b/internal/core/src/segcore/SegmentSmallIndex.cpp
index a6eccb9440440062a17d83956e68d876f200540a..fa7b0a3164cf0f4f17a785f8d1d181c8be88af6f 100644
--- a/internal/core/src/segcore/SegmentSmallIndex.cpp
+++ b/internal/core/src/segcore/SegmentSmallIndex.cpp
@@ -349,19 +349,12 @@ SegmentSmallIndex::FillTargetEntry(const query::Plan* plan, QueryResult& results
Assert(results.result_offsets_.size() == size);
Assert(results.row_data_.size() == 0);
- // TODO: deprecate
- results.result_ids_.clear();
- results.result_ids_.resize(size);
-
if (plan->schema_.get_is_auto_id()) {
auto& uids = record_.uids_;
for (int64_t i = 0; i < size; ++i) {
auto seg_offset = results.internal_seg_offsets_[i];
auto row_id = seg_offset == -1 ? -1 : uids[seg_offset];
- // TODO: deprecate
- results.result_ids_[i] = row_id;
-
std::vector<char> blob(sizeof(row_id));
memcpy(blob.data(), &row_id, sizeof(row_id));
results.row_data_.emplace_back(std::move(blob));
@@ -377,9 +370,6 @@ SegmentSmallIndex::FillTargetEntry(const query::Plan* plan, QueryResult& results
auto seg_offset = results.internal_seg_offsets_[i];
auto row_id = seg_offset == -1 ? -1 : uids->operator[](seg_offset);
- // TODO: deprecate
- results.result_ids_[i] = row_id;
-
std::vector<char> blob(sizeof(row_id));
memcpy(blob.data(), &row_id, sizeof(row_id));
results.row_data_.emplace_back(std::move(blob));
diff --git a/internal/core/src/segcore/reduce_c.cpp b/internal/core/src/segcore/reduce_c.cpp
index 68af9246497b11dab63e56a121f877f6a8e35da3..cbd88a13b9b0209cbeb8d247857fc9b3a9c847d9 100644
--- a/internal/core/src/segcore/reduce_c.cpp
+++ b/internal/core/src/segcore/reduce_c.cpp
@@ -14,10 +14,10 @@
#include "segcore/reduce_c.h"
#include "segcore/Reduce.h"
-#include "utils/Types.h"
+#include "common/Types.h"
#include "pb/service_msg.pb.h"
-using SearchResult = milvus::engine::QueryResult;
+using SearchResult = milvus::QueryResult;
int
MergeInto(int64_t num_queries, int64_t topk, float* distances, int64_t* uids, float* new_distances, int64_t* new_uids) {
diff --git a/internal/core/src/segcore/segment_c.cpp b/internal/core/src/segcore/segment_c.cpp
index 2bed416f9397cb3f8e86acb2af7f56d8f8c4fa1f..c38b3921be18f2e8240320389b43d31ec56ad75d 100644
--- a/internal/core/src/segcore/segment_c.cpp
+++ b/internal/core/src/segcore/segment_c.cpp
@@ -165,7 +165,7 @@ CStatus
FillTargetEntry(CSegmentBase c_segment, CPlan c_plan, CQueryResult c_result) {
auto segment = (milvus::segcore::SegmentBase*)c_segment;
auto plan = (milvus::query::Plan*)c_plan;
- auto result = (milvus::engine::QueryResult*)c_result;
+ auto result = (milvus::QueryResult*)c_result;
auto status = CStatus();
try {
diff --git a/internal/core/src/utils/Types.h b/internal/core/src/utils/Types.h
index 33643f32580541180cbc87c865dfd7c6ec5852cc..4e404c8035c3521a71d054edcce48ec5b50438e5 100644
--- a/internal/core/src/utils/Types.h
+++ b/internal/core/src/utils/Types.h
@@ -136,45 +136,5 @@ struct AttrsData {
IDNumbers id_array_;
};
-///////////////////////////////////////////////////////////////////////////////////////////////////
-struct QueryResult {
- QueryResult() = default;
- QueryResult(uint64_t num_queries, uint64_t topK) : topK_(topK), num_queries_(num_queries) {
- auto count = get_row_count();
- result_distances_.resize(count);
- internal_seg_offsets_.resize(count);
-
- // TODO: deprecated
- result_ids_.resize(count);
- }
-
- [[nodiscard]] uint64_t
- get_row_count() const {
- return topK_ * num_queries_;
- }
-
- uint64_t num_queries_;
- uint64_t topK_;
- // uint64_t total_row_count_; // total_row_count_ = topK * num_queries_
-
- // vector<tuple<Score, SegId, Offset>> data_reduced;
-
- // vector<tuple<Score, SegId, Offset, RawData>>
-
- // map<SegId, vector<tuple<DataOffset, ResLoc>>>
- uint64_t seg_id_;
- std::vector<float> result_distances_;
-
- // TODO(gexi): utilize these field
- std::vector<int64_t> internal_seg_offsets_;
- std::vector<int64_t> result_offsets_;
- std::vector<std::vector<char>> row_data_;
-
- // TODO: deprecated, use row_data directly
- std::vector<idx_t> result_ids_;
-};
-
-using QueryResultPtr = std::shared_ptr<QueryResult>;
-
} // namespace engine
} // namespace milvus
diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt
index a69690728065e26d373ed961c0fd4ccd4817e10b..ce3f8ab702d9a08c64fcba79d77af77a728cc960 100644
--- a/internal/core/unittest/CMakeLists.txt
+++ b/internal/core/unittest/CMakeLists.txt
@@ -14,6 +14,7 @@ set(MILVUS_TEST_FILES
test_binary.cpp
test_index_wrapper.cpp
test_sealed.cpp
+ test_reduce.cpp
)
add_executable(all_tests
${MILVUS_TEST_FILES}
@@ -24,10 +25,8 @@ target_link_libraries(all_tests
gtest_main
milvus_segcore
milvus_indexbuilder
- knowhere
log
pthread
- milvus_utils
)
install (TARGETS all_tests DESTINATION unittest)
diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp
index 0c5dbe83a8ca657cfef829b5767577312124f416..65866a60b7f39538dcb5ef2cba8a872f23da8689 100644
--- a/internal/core/unittest/test_c_api.cpp
+++ b/internal/core/unittest/test_c_api.cpp
@@ -137,7 +137,7 @@ TEST(CApiTest, SearchTest) {
auto offset = PreInsert(segment, N);
auto ins_res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
- assert(ins_res.error_code == Success);
+ ASSERT_EQ(ins_res.error_code, Success);
const char* dsl_string = R"(
{
@@ -176,11 +176,11 @@ TEST(CApiTest, SearchTest) {
void* plan = nullptr;
auto status = CreatePlan(collection, dsl_string, &plan);
- assert(status.error_code == Success);
+ ASSERT_EQ(status.error_code, Success);
void* placeholderGroup = nullptr;
status = ParsePlaceholderGroup(plan, blob.data(), blob.length(), &placeholderGroup);
- assert(status.error_code == Success);
+ ASSERT_EQ(status.error_code, Success);
std::vector<CPlaceholderGroup> placeholderGroups;
placeholderGroups.push_back(placeholderGroup);
@@ -189,7 +189,7 @@ TEST(CApiTest, SearchTest) {
CQueryResult search_result;
auto res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1, &search_result);
- assert(res.error_code == Success);
+ ASSERT_EQ(res.error_code, Success);
DeletePlan(plan);
DeletePlaceholderGroup(placeholderGroup);
diff --git a/internal/core/unittest/test_index_wrapper.cpp b/internal/core/unittest/test_index_wrapper.cpp
index a885c837a096b9558da69ec74c73d2c9c019e510..bd335951f8053029f720e368d7907cb0d65d451d 100644
--- a/internal/core/unittest/test_index_wrapper.cpp
+++ b/internal/core/unittest/test_index_wrapper.cpp
@@ -11,6 +11,8 @@
#include <tuple>
#include <map>
+#include <limits>
+#include <math.h>
#include <gtest/gtest.h>
#include <google/protobuf/text_format.h>
@@ -41,16 +43,16 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, K},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, K},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nlist, 100},
- // {milvus::knowhere::IndexParams::nprobe, 4},
+ {milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, metric_type},
@@ -59,9 +61,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, K},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nlist, 100},
- // {milvus::knowhere::IndexParams::nprobe, 4},
+ {milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
#ifdef MILVUS_GPU_VERSION
@@ -71,9 +73,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, K},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nlist, 100},
- // {milvus::knowhere::IndexParams::nprobe, 4},
+ {milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
@@ -84,9 +86,9 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, K},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nlist, 100},
- // {milvus::knowhere::IndexParams::nprobe, 4},
+ {milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, metric_type},
@@ -95,13 +97,14 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, K},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::Metric::TYPE, metric_type},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_NSG) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::IndexParams::nlist, 163},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nprobe, 8},
{milvus::knowhere::IndexParams::knng, 20},
{milvus::knowhere::IndexParams::search_length, 40},
@@ -127,17 +130,14 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
#endif
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_HNSW) {
return milvus::knowhere::Config{
- {milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, 10},
- {milvus::knowhere::IndexParams::M, 16},
- {milvus::knowhere::IndexParams::efConstruction, 200},
- {milvus::knowhere::IndexParams::ef, 200},
- {milvus::knowhere::Metric::TYPE, metric_type},
+ {milvus::knowhere::meta::DIM, DIM}, {milvus::knowhere::meta::TOPK, K},
+ {milvus::knowhere::IndexParams::M, 16}, {milvus::knowhere::IndexParams::efConstruction, 200},
+ {milvus::knowhere::IndexParams::ef, 200}, {milvus::knowhere::Metric::TYPE, metric_type},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_ANNOY) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, 10},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::n_trees, 4},
{milvus::knowhere::IndexParams::search_k, 100},
{milvus::knowhere::Metric::TYPE, metric_type},
@@ -146,7 +146,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWFlat) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, 10},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::M, 16},
{milvus::knowhere::IndexParams::efConstruction, 200},
{milvus::knowhere::IndexParams::ef, 200},
@@ -156,7 +156,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWPQ) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, 10},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::M, 16},
{milvus::knowhere::IndexParams::efConstruction, 200},
{milvus::knowhere::IndexParams::ef, 200},
@@ -167,7 +167,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWSQ) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, 10},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::M, 16},
{milvus::knowhere::IndexParams::efConstruction, 200},
{milvus::knowhere::IndexParams::ef, 200},
@@ -177,7 +177,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTPANNG) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, 10},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::IndexParams::edge_size, 10},
{milvus::knowhere::IndexParams::epsilon, 0.1},
@@ -189,7 +189,7 @@ generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowh
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTONNG) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
- // {milvus::knowhere::meta::TOPK, 10},
+ {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::IndexParams::edge_size, 20},
{milvus::knowhere::IndexParams::epsilon, 0.1},
@@ -234,6 +234,99 @@ GenDataset(int64_t N, const milvus::knowhere::MetricType& metric_type, bool is_b
return milvus::segcore::DataGen(schema, N);
}
}
+
+using QueryResultPtr = std::unique_ptr<milvus::indexbuilder::IndexWrapper::QueryResult>;
+void
+PrintQueryResult(const QueryResultPtr& result) {
+ auto nq = result->nq;
+ auto k = result->topk;
+
+ std::stringstream ss_id;
+ std::stringstream ss_dist;
+
+ for (auto i = 0; i < nq; i++) {
+ for (auto j = 0; j < k; ++j) {
+ ss_id << result->ids[i * k + j] << " ";
+ ss_dist << result->distances[i * k + j] << " ";
+ }
+ ss_id << std::endl;
+ ss_dist << std::endl;
+ }
+ std::cout << "id\n" << ss_id.str() << std::endl;
+ std::cout << "dist\n" << ss_dist.str() << std::endl;
+}
+
+float
+L2(const float* point_a, const float* point_b, int dim) {
+ float dis = 0;
+ for (auto i = 0; i < dim; i++) {
+ auto c_a = point_a[i];
+ auto c_b = point_b[i];
+ dis += pow(c_b - c_a, 2);
+ }
+ return dis;
+}
+
+int hamming_weight(uint8_t n) {
+ int count=0;
+ while(n != 0){
+ count += n&1;
+ n >>= 1;
+ }
+ return count;
+}
+float
+Jaccard(const uint8_t* point_a, const uint8_t* point_b, int dim) {
+ float dis;
+ int len = dim / 8;
+ float intersection = 0;
+ float union_num = 0;
+ for (int i = 0; i < len; i++) {
+ intersection += hamming_weight(point_a[i] & point_b[i]);
+ union_num += hamming_weight(point_a[i] | point_b[i]);
+ }
+ dis = 1 - (intersection / union_num);
+ return dis;
+}
+
+float
+CountDistance(const void* point_a,
+ const void* point_b,
+ int dim,
+ const milvus::knowhere::MetricType& metric,
+ bool is_binary = false) {
+ if (point_a == nullptr || point_b == nullptr) {
+ return std::numeric_limits<float>::max();
+ }
+ if (metric == milvus::knowhere::Metric::L2) {
+ return L2(static_cast<const float*>(point_a), static_cast<const float*>(point_b), dim);
+ } else if (metric == milvus::knowhere::Metric::JACCARD) {
+ return Jaccard(static_cast<const uint8_t*>(point_a), static_cast<const uint8_t*>(point_b), dim);
+ } else {
+ return std::numeric_limits<float>::max();
+ }
+}
+
+void
+CheckDistances(const QueryResultPtr& result,
+ const milvus::knowhere::DatasetPtr& base_dataset,
+ const milvus::knowhere::DatasetPtr& query_dataset,
+ const milvus::knowhere::MetricType& metric,
+ const float threshold = 1.0e-5) {
+ auto base_vecs = base_dataset->Get<float*>(milvus::knowhere::meta::TENSOR);
+ auto query_vecs = query_dataset->Get<float*>(milvus::knowhere::meta::TENSOR);
+ auto dim = base_dataset->Get<int64_t>(milvus::knowhere::meta::DIM);
+ auto nq = result->nq;
+ auto k = result->topk;
+ for (auto i = 0; i < nq; i++) {
+ for (auto j = 0; j < k; ++j) {
+ auto dis = result->distances[i * k + j];
+ auto id = result->ids[i * k + j];
+ auto count_dis = CountDistance(query_vecs + i * dim, base_vecs + id * dim, dim, metric);
+ // assert(std::abs(dis - count_dis) < threshold);
+ }
+ }
+}
} // namespace
using Param = std::pair<milvus::knowhere::IndexType, milvus::knowhere::MetricType>;
@@ -247,8 +340,26 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
metric_type = param.second;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
- std::map<std::string, bool> is_binary_map = {{milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false},
- {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true}};
+ std::map<std::string, bool> is_binary_map = {
+ {milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP, false},
+ {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false},
+ {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false},
+ {milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, false},
+ {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true},
+ {milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true},
+#ifdef MILVUS_SUPPORT_SPTAG
+ {milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, false},
+ {milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, false},
+#endif
+ {milvus::knowhere::IndexEnum::INDEX_HNSW, false},
+ {milvus::knowhere::IndexEnum::INDEX_ANNOY, false},
+ {milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, false},
+ {milvus::knowhere::IndexEnum::INDEX_RHNSWPQ, false},
+ {milvus::knowhere::IndexEnum::INDEX_RHNSWSQ, false},
+ {milvus::knowhere::IndexEnum::INDEX_NGTPANNG, false},
+ {milvus::knowhere::IndexEnum::INDEX_NGTONNG, false},
+ {milvus::knowhere::IndexEnum::INDEX_NSG, false},
+ };
is_binary = is_binary_map[index_type];
@@ -262,9 +373,13 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
if (!is_binary) {
xb_data = dataset.get_col<float>(0);
xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
+ xq_data = dataset.get_col<float>(0);
+ xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
} else {
xb_bin_data = dataset.get_col<uint8_t>(0);
xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_bin_data.data());
+ xq_bin_data = dataset.get_col<uint8_t>(0);
+ xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_bin_data.data());
}
}
@@ -282,6 +397,9 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
std::vector<float> xb_data;
std::vector<uint8_t> xb_bin_data;
std::vector<milvus::knowhere::IDType> ids;
+ milvus::knowhere::DatasetPtr xq_dataset;
+ std::vector<float> xq_data;
+ std::vector<uint8_t> xq_bin_data;
};
TEST(PQ, Build) {
@@ -308,6 +426,47 @@ TEST(IVFFLATNM, Build) {
ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf));
}
+TEST(IVFFLATNM, Query) {
+ auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
+ auto metric_type = milvus::knowhere::Metric::L2;
+ auto conf = generate_conf(index_type, metric_type);
+ auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
+ auto dataset = GenDataset(NB, metric_type, false);
+ auto xb_data = dataset.get_col<float>(0);
+ auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
+ ASSERT_NO_THROW(index->Train(xb_dataset, conf));
+ ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf));
+ auto bs = index->Serialize(conf);
+ auto bptr = std::make_shared<milvus::knowhere::Binary>();
+ bptr->data = std::shared_ptr<uint8_t[]>((uint8_t*)xb_data.data(), [&](uint8_t*) {});
+ bptr->size = DIM * NB * sizeof(float);
+ bs.Append(RAW_DATA, bptr);
+ index->Load(bs);
+ auto xq_data = dataset.get_col<float>(0);
+ auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
+ auto result = index->Query(xq_dataset, conf, nullptr);
+}
+
+TEST(NSG, Query) {
+ auto index_type = milvus::knowhere::IndexEnum::INDEX_NSG;
+ auto metric_type = milvus::knowhere::Metric::L2;
+ auto conf = generate_conf(index_type, metric_type);
+ auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
+ auto dataset = GenDataset(NB, metric_type, false);
+ auto xb_data = dataset.get_col<float>(0);
+ auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
+ index->BuildAll(xb_dataset, conf);
+ auto bs = index->Serialize(conf);
+ auto bptr = std::make_shared<milvus::knowhere::Binary>();
+ bptr->data = std::shared_ptr<uint8_t[]>((uint8_t*)xb_data.data(), [&](uint8_t*) {});
+ bptr->size = DIM * NB * sizeof(float);
+ bs.Append(RAW_DATA, bptr);
+ index->Load(bs);
+ auto xq_data = dataset.get_col<float>(0);
+ auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
+ auto result = index->Query(xq_dataset, conf, nullptr);
+}
+
TEST(BINFLAT, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::JACCARD;
@@ -485,12 +644,7 @@ TEST_P(IndexWrapperTest, Dim) {
TEST_P(IndexWrapperTest, BuildWithoutIds) {
auto index =
std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str.c_str(), index_params_str.c_str());
-
- if (milvus::indexbuilder::is_in_need_id_list(index_type)) {
- ASSERT_ANY_THROW(index->BuildWithoutIds(xb_dataset));
- } else {
- ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
- }
+ ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
}
TEST_P(IndexWrapperTest, Codec) {
@@ -511,3 +665,16 @@ TEST_P(IndexWrapperTest, Codec) {
ASSERT_EQ(strcmp(binary.data, copy_binary.data), 0);
}
}
+
+TEST_P(IndexWrapperTest, Query) {
+ auto index_wrapper =
+ std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str.c_str(), index_params_str.c_str());
+
+ index_wrapper->BuildWithoutIds(xb_dataset);
+
+ std::unique_ptr<milvus::indexbuilder::IndexWrapper::QueryResult> query_result = index_wrapper->Query(xq_dataset);
+ ASSERT_EQ(query_result->topk, K);
+ ASSERT_EQ(query_result->nq, NQ);
+ ASSERT_EQ(query_result->distances.size(), query_result->topk * query_result->nq);
+ ASSERT_EQ(query_result->ids.size(), query_result->topk * query_result->nq);
+}
diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp
index 1030938cefe2414223b7b51870f679743061b4d0..d5712c1ae680f1bed9b2c49e180905efae646f45 100644
--- a/internal/core/unittest/test_indexing.cpp
+++ b/internal/core/unittest/test_indexing.cpp
@@ -33,7 +33,7 @@
#include "test_utils/Timer.h"
#include "segcore/Reduce.h"
#include "test_utils/DataGen.h"
-#include "query/BruteForceSearch.h"
+#include "query/SearchBruteForce.h"
using std::cin;
using std::cout;
@@ -245,8 +245,6 @@ TEST(Indexing, BinaryBruteForce) {
schema->AddField("vecbin", DataType::VECTOR_BINARY, dim, MetricType::METRIC_Jaccard);
schema->AddField("age", DataType::INT64);
auto dataset = DataGen(schema, N, 10);
- vector<float> distances(result_count);
- vector<int64_t> ids(result_count);
auto bin_vec = dataset.get_col<uint8_t>(0);
auto line_sizeof = schema->operator[](0).get_sizeof();
auto query_data = 1024 * line_sizeof + bin_vec.data();
@@ -258,13 +256,13 @@ TEST(Indexing, BinaryBruteForce) {
query_data //
};
- query::BinarySearchBruteForce(query_dataset, bin_vec.data(), N, distances.data(), ids.data());
+ auto sub_result = query::BinarySearchBruteForce(query_dataset, bin_vec.data(), N);
QueryResult qr;
qr.num_queries_ = num_queries;
qr.topK_ = topk;
- qr.internal_seg_offsets_ = ids;
- qr.result_distances_ = distances;
+ qr.internal_seg_offsets_ = std::move(sub_result.mutable_labels());
+ qr.result_distances_ = std::move(sub_result.mutable_values());
auto json = QueryResultToJson(qr);
auto ref = json::parse(R"(
diff --git a/internal/core/unittest/test_query.cpp b/internal/core/unittest/test_query.cpp
index ce914a5c9eb7c2afc6de8ccf43132685311b4aa2..121f5932ae2e2fdc5d35a87361c8bbcd2b854808 100644
--- a/internal/core/unittest/test_query.cpp
+++ b/internal/core/unittest/test_query.cpp
@@ -402,7 +402,7 @@ TEST(Query, FillSegment) {
pb::schema::CollectionSchema proto;
proto.set_name("col");
proto.set_description("asdfhsalkgfhsadg");
- proto.set_autoid(true);
+ proto.set_autoid(false);
{
auto field = proto.add_fields();
@@ -425,7 +425,7 @@ TEST(Query, FillSegment) {
field->set_fieldid(101);
field->set_is_primary_key(true);
field->set_description("asdgfsagf");
- field->set_data_type(pb::schema::DataType::INT32);
+ field->set_data_type(pb::schema::DataType::INT64);
}
auto schema = Schema::ParseFrom(proto);
@@ -466,18 +466,17 @@ TEST(Query, FillSegment) {
result.result_offsets_.resize(topk * num_queries);
segment->FillTargetEntry(plan.get(), result);
- // TODO: deprecated result_ids_
- ASSERT_EQ(result.result_ids_, result.internal_seg_offsets_);
-
auto ans = result.row_data_;
ASSERT_EQ(ans.size(), topk * num_queries);
int64_t std_index = 0;
+ auto std_vec = dataset.get_col<int64_t>(1);
for (auto& vec : ans) {
ASSERT_EQ(vec.size(), sizeof(int64_t));
int64_t val;
memcpy(&val, vec.data(), sizeof(int64_t));
- auto std_val = result.result_ids_[std_index];
- ASSERT_EQ(val, std_val);
+ auto internal_offset = result.internal_seg_offsets_[std_index];
+ auto std_val = std_vec[internal_offset];
+ ASSERT_EQ(val, std_val) << "io:" << internal_offset;
++std_index;
}
}
diff --git a/internal/core/unittest/test_reduce.cpp b/internal/core/unittest/test_reduce.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..985586128c2bab385497d723b6c334de0483cf94
--- /dev/null
+++ b/internal/core/unittest/test_reduce.cpp
@@ -0,0 +1,120 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include <gtest/gtest.h>
+#include "query/SubQueryResult.h"
+#include <vector>
+#include <queue>
+#include <random>
+
+using namespace milvus;
+using namespace milvus::query;
+
+TEST(Reduce, SubQueryResult) {
+ int64_t num_queries = 512;
+ int64_t topk = 32;
+ int64_t iteration = 50;
+ constexpr int64_t limit = 100000000L;
+ auto metric_type = MetricType::METRIC_L2;
+ using queue_type = std::priority_queue<int64_t>;
+
+ std::vector<queue_type> ref_results(num_queries);
+ for (auto& ref_result : ref_results) {
+ for (int i = 0; i < topk; ++i) {
+ ref_result.push(limit);
+ }
+ }
+ std::default_random_engine e(42);
+ SubQueryResult final_result(num_queries, topk, metric_type);
+ for (int i = 0; i < iteration; ++i) {
+ std::vector<int64_t> labels;
+ std::vector<float> values;
+ for (int n = 0; n < num_queries; ++n) {
+ for (int k = 0; k < topk; ++k) {
+ auto gen_x = e() % limit;
+ ref_results[n].push(gen_x);
+ ref_results[n].pop();
+ labels.push_back(gen_x);
+ values.push_back(gen_x);
+ }
+ std::sort(labels.begin() + n * topk, labels.begin() + n * topk + topk);
+ std::sort(values.begin() + n * topk, values.begin() + n * topk + topk);
+ }
+ SubQueryResult sub_result(num_queries, topk, metric_type);
+ sub_result.mutable_values() = values;
+ sub_result.mutable_labels() = labels;
+ final_result.merge(sub_result);
+ }
+
+ for (int n = 0; n < num_queries; ++n) {
+ ASSERT_EQ(ref_results[n].size(), topk);
+ for (int k = 0; k < topk; ++k) {
+ auto ref_x = ref_results[n].top();
+ ref_results[n].pop();
+ auto index = n * topk + topk - 1 - k;
+ auto label = final_result.get_labels()[index];
+ auto value = final_result.get_values()[index];
+ ASSERT_EQ(label, ref_x);
+ ASSERT_EQ(value, ref_x);
+ }
+ }
+}
+
+TEST(Reduce, SubQueryResultDesc) {
+ int64_t num_queries = 512;
+ int64_t topk = 32;
+ int64_t iteration = 50;
+ constexpr int64_t limit = 100000000L;
+ constexpr int64_t init_value = 0;
+ auto metric_type = MetricType::METRIC_INNER_PRODUCT;
+ using queue_type = std::priority_queue<int64_t, std::vector<int64_t>, std::greater<int64_t>>;
+
+ std::vector<queue_type> ref_results(num_queries);
+ for (auto& ref_result : ref_results) {
+ for (int i = 0; i < topk; ++i) {
+ ref_result.push(init_value);
+ }
+ }
+ std::default_random_engine e(42);
+ SubQueryResult final_result(num_queries, topk, metric_type);
+ for (int i = 0; i < iteration; ++i) {
+ std::vector<int64_t> labels;
+ std::vector<float> values;
+ for (int n = 0; n < num_queries; ++n) {
+ for (int k = 0; k < topk; ++k) {
+ auto gen_x = e() % limit;
+ ref_results[n].push(gen_x);
+ ref_results[n].pop();
+ labels.push_back(gen_x);
+ values.push_back(gen_x);
+ }
+ std::sort(labels.begin() + n * topk, labels.begin() + n * topk + topk, std::greater<int64_t>());
+ std::sort(values.begin() + n * topk, values.begin() + n * topk + topk, std::greater<float>());
+ }
+ SubQueryResult sub_result(num_queries, topk, metric_type);
+ sub_result.mutable_values() = values;
+ sub_result.mutable_labels() = labels;
+ final_result.merge(sub_result);
+ }
+
+ for (int n = 0; n < num_queries; ++n) {
+ ASSERT_EQ(ref_results[n].size(), topk);
+ for (int k = 0; k < topk; ++k) {
+ auto ref_x = ref_results[n].top();
+ ref_results[n].pop();
+ auto index = n * topk + topk - 1 - k;
+ auto label = final_result.get_labels()[index];
+ auto value = final_result.get_values()[index];
+ ASSERT_EQ(label, ref_x);
+ ASSERT_EQ(value, ref_x);
+ }
+ }
+}
\ No newline at end of file
diff --git a/internal/indexbuilder/indexbuilder.go b/internal/indexbuilder/indexbuilder.go
index 5b21e68dd4eebd11079784d799aaf63679a28359..4acfffc3d284665158d9f3c237682a5792257d50 100644
--- a/internal/indexbuilder/indexbuilder.go
+++ b/internal/indexbuilder/indexbuilder.go
@@ -11,9 +11,6 @@ import (
miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
-
"go.etcd.io/etcd/clientv3"
"github.com/zilliztech/milvus-distributed/internal/allocator"
@@ -71,19 +68,16 @@ func CreateBuilder(ctx context.Context) (*Builder, error) {
idAllocator, err := allocator.NewIDAllocator(b.loopCtx, Params.MasterAddress)
- minIOEndPoint := Params.MinIOAddress
- minIOAccessKeyID := Params.MinIOAccessKeyID
- minIOSecretAccessKey := Params.MinIOSecretAccessKey
- minIOUseSSL := Params.MinIOUseSSL
- minIOClient, err := minio.New(minIOEndPoint, &minio.Options{
- Creds: credentials.NewStaticV4(minIOAccessKeyID, minIOSecretAccessKey, ""),
- Secure: minIOUseSSL,
- })
- if err != nil {
- return nil, err
+ option := &miniokv.Option{
+ Address: Params.MinIOAddress,
+ AccessKeyID: Params.MinIOAccessKeyID,
+ SecretAccessKeyID: Params.MinIOSecretAccessKey,
+ UseSSL: Params.MinIOUseSSL,
+ BucketName: Params.MinioBucketName,
+ CreateBucket: true,
}
- b.kv, err = miniokv.NewMinIOKV(b.loopCtx, minIOClient, Params.MinioBucketName)
+ b.kv, err = miniokv.NewMinIOKV(b.loopCtx, option)
if err != nil {
return nil, err
}
diff --git a/internal/kv/minio/minio_kv.go b/internal/kv/minio/minio_kv.go
index 68bb3a3438bbd771a2d5afc98c23988703db72ea..6b3522fb454273f24a08e11d83f4f64e43a40381 100644
--- a/internal/kv/minio/minio_kv.go
+++ b/internal/kv/minio/minio_kv.go
@@ -2,11 +2,15 @@ package miniokv
import (
"context"
+ "fmt"
+
"io"
"log"
"strings"
"github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/zilliztech/milvus-distributed/internal/errors"
)
type MinIOKV struct {
@@ -15,24 +19,46 @@ type MinIOKV struct {
bucketName string
}
-// NewMinIOKV creates a new MinIO kv.
-func NewMinIOKV(ctx context.Context, client *minio.Client, bucketName string) (*MinIOKV, error) {
+type Option struct {
+ Address string
+ AccessKeyID string
+ BucketName string
+ SecretAccessKeyID string
+ UseSSL bool
+ CreateBucket bool // when bucket not existed, create it
+}
- bucketExists, err := client.BucketExists(ctx, bucketName)
+func NewMinIOKV(ctx context.Context, option *Option) (*MinIOKV, error) {
+ minIOClient, err := minio.New(option.Address, &minio.Options{
+ Creds: credentials.NewStaticV4(option.AccessKeyID, option.SecretAccessKeyID, ""),
+ Secure: option.UseSSL,
+ })
if err != nil {
return nil, err
}
- if !bucketExists {
- err = client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
- if err != nil {
- return nil, err
+ bucketExists, err := minIOClient.BucketExists(ctx, option.BucketName)
+ if err != nil {
+ return nil, err
+ }
+
+ if option.CreateBucket {
+ if !bucketExists {
+ err = minIOClient.MakeBucket(ctx, option.BucketName, minio.MakeBucketOptions{})
+ if err != nil {
+ return nil, err
+ }
+ }
+ } else {
+ if !bucketExists {
+ return nil, errors.New(fmt.Sprintf("Bucket %s not Existed.", option.BucketName))
}
}
+
return &MinIOKV{
ctx: ctx,
- minioClient: client,
- bucketName: bucketName,
+ minioClient: minIOClient,
+ bucketName: option.BucketName,
}, nil
}
diff --git a/internal/kv/minio/minio_kv_test.go b/internal/kv/minio/minio_kv_test.go
index ac2a3180b2966bbdc09158ff70ec8baaef23dedb..2e50545b40a83516aa16cb0991bbc64cf34360cc 100644
--- a/internal/kv/minio/minio_kv_test.go
+++ b/internal/kv/minio/minio_kv_test.go
@@ -5,8 +5,6 @@ import (
"strconv"
"testing"
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/util/paramtable"
@@ -15,24 +13,31 @@ import (
var Params paramtable.BaseTable
-func TestMinIOKV_Load(t *testing.T) {
- Params.Init()
+func newMinIOKVClient(ctx context.Context, bucketName string) (*miniokv.MinIOKV, error) {
endPoint, _ := Params.Load("_MinioAddress")
accessKeyID, _ := Params.Load("minio.accessKeyID")
secretAccessKey, _ := Params.Load("minio.secretAccessKey")
useSSLStr, _ := Params.Load("minio.useSSL")
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
useSSL, _ := strconv.ParseBool(useSSLStr)
+ option := &miniokv.Option{
+ Address: endPoint,
+ AccessKeyID: accessKeyID,
+ SecretAccessKeyID: secretAccessKey,
+ UseSSL: useSSL,
+ BucketName: bucketName,
+ CreateBucket: true,
+ }
+ client, err := miniokv.NewMinIOKV(ctx, option)
+ return client, err
+}
- minioClient, err := minio.New(endPoint, &minio.Options{
- Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
- Secure: useSSL,
- })
- assert.Nil(t, err)
+func TestMinIOKV_Load(t *testing.T) {
+ Params.Init()
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
bucketName := "fantastic-tech-test"
- MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
+ MinIOKV, err := newMinIOKVClient(ctx, bucketName)
assert.Nil(t, err)
defer MinIOKV.RemoveWithPrefix("")
@@ -79,25 +84,14 @@ func TestMinIOKV_Load(t *testing.T) {
}
func TestMinIOKV_MultiSave(t *testing.T) {
+ Params.Init()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- Params.Init()
- endPoint, _ := Params.Load("_MinioAddress")
- accessKeyID, _ := Params.Load("minio.accessKeyID")
- secretAccessKey, _ := Params.Load("minio.secretAccessKey")
- useSSLStr, _ := Params.Load("minio.useSSL")
- useSSL, _ := strconv.ParseBool(useSSLStr)
-
- minioClient, err := minio.New(endPoint, &minio.Options{
- Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
- Secure: useSSL,
- })
- assert.Nil(t, err)
-
bucketName := "fantastic-tech-test"
- MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
+ MinIOKV, err := newMinIOKVClient(ctx, bucketName)
assert.Nil(t, err)
+
defer MinIOKV.RemoveWithPrefix("")
err = MinIOKV.Save("key_1", "111")
@@ -117,25 +111,13 @@ func TestMinIOKV_MultiSave(t *testing.T) {
}
func TestMinIOKV_Remove(t *testing.T) {
+ Params.Init()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- Params.Init()
- endPoint, _ := Params.Load("_MinioAddress")
- accessKeyID, _ := Params.Load("minio.accessKeyID")
- secretAccessKey, _ := Params.Load("minio.secretAccessKey")
- useSSLStr, _ := Params.Load("minio.useSSL")
- useSSL, _ := strconv.ParseBool(useSSLStr)
-
- minioClient, err := minio.New(endPoint, &minio.Options{
- Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
- Secure: useSSL,
- })
- assert.Nil(t, err)
-
bucketName := "fantastic-tech-test"
- MinIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, bucketName)
+ MinIOKV, err := newMinIOKVClient(ctx, bucketName)
assert.Nil(t, err)
defer MinIOKV.RemoveWithPrefix("")
diff --git a/internal/master/client.go b/internal/master/client.go
index 7c644e02acddf72292636baac9d011272bc19574..88e44d8f70a02aba66217094d8f2327c6b06c814 100644
--- a/internal/master/client.go
+++ b/internal/master/client.go
@@ -9,19 +9,25 @@ import (
)
type WriteNodeClient interface {
- FlushSegment(segmentID UniqueID) error
+ FlushSegment(segmentID UniqueID, collectionID UniqueID, partitionTag string, timestamp Timestamp) error
DescribeSegment(segmentID UniqueID) (*writerclient.SegmentDescription, error)
GetInsertBinlogPaths(segmentID UniqueID) (map[UniqueID][]string, error)
}
type MockWriteNodeClient struct {
- segmentID UniqueID
- flushTime time.Time
+ segmentID UniqueID
+ flushTime time.Time
+ partitionTag string
+ timestamp Timestamp
+ collectionID UniqueID
}
-func (m *MockWriteNodeClient) FlushSegment(segmentID UniqueID) error {
+func (m *MockWriteNodeClient) FlushSegment(segmentID UniqueID, collectionID UniqueID, partitionTag string, timestamp Timestamp) error {
m.flushTime = time.Now()
m.segmentID = segmentID
+ m.collectionID = collectionID
+ m.partitionTag = partitionTag
+ m.timestamp = timestamp
return nil
}
diff --git a/internal/master/flush_scheduler.go b/internal/master/flush_scheduler.go
index d27511c2c868907be6fb3e551d04695455dbf1b4..7ed3d9a5689383f05d10a9ab598f9e4f293540c5 100644
--- a/internal/master/flush_scheduler.go
+++ b/internal/master/flush_scheduler.go
@@ -17,11 +17,12 @@ type FlushScheduler struct {
segmentDescribeChan chan UniqueID
indexBuilderSch persistenceScheduler
- ctx context.Context
- cancel context.CancelFunc
+ ctx context.Context
+ cancel context.CancelFunc
+ globalTSOAllocator func() (Timestamp, error)
}
-func NewFlushScheduler(ctx context.Context, client WriteNodeClient, metaTable *metaTable, buildScheduler *IndexBuildScheduler) *FlushScheduler {
+func NewFlushScheduler(ctx context.Context, client WriteNodeClient, metaTable *metaTable, buildScheduler *IndexBuildScheduler, globalTSOAllocator func() (Timestamp, error)) *FlushScheduler {
ctx2, cancel := context.WithCancel(ctx)
return &FlushScheduler{
@@ -32,12 +33,23 @@ func NewFlushScheduler(ctx context.Context, client WriteNodeClient, metaTable *m
segmentDescribeChan: make(chan UniqueID, 100),
ctx: ctx2,
cancel: cancel,
+ globalTSOAllocator: globalTSOAllocator,
}
}
func (scheduler *FlushScheduler) schedule(id interface{}) error {
segmentID := id.(UniqueID)
- err := scheduler.client.FlushSegment(segmentID)
+ segmentMeta, err := scheduler.metaTable.GetSegmentByID(segmentID)
+ if err != nil {
+ return err
+ }
+
+ ts, err := scheduler.globalTSOAllocator()
+ if err != nil {
+ return err
+ }
+ // todo set corrent timestamp
+ err = scheduler.client.FlushSegment(segmentID, segmentMeta.CollectionID, segmentMeta.PartitionTag, ts)
log.Printf("flush segment %d", segmentID)
if err != nil {
return err
diff --git a/internal/master/master.go b/internal/master/master.go
index 5476a7c8dc177e249c6cca4103a264f8c83d1e17..12f6e53086c74854e747c5969376260de5130f41 100644
--- a/internal/master/master.go
+++ b/internal/master/master.go
@@ -193,7 +193,7 @@ func CreateServer(ctx context.Context) (*Master, error) {
m.indexLoadSch = NewIndexLoadScheduler(ctx, loadIndexClient, m.metaTable)
m.indexBuildSch = NewIndexBuildScheduler(ctx, buildIndexClient, m.metaTable, m.indexLoadSch)
- m.flushSch = NewFlushScheduler(ctx, flushClient, m.metaTable, m.indexBuildSch)
+ m.flushSch = NewFlushScheduler(ctx, flushClient, m.metaTable, m.indexBuildSch, func() (Timestamp, error) { return m.tsoAllocator.AllocOne() })
m.segmentAssigner = NewSegmentAssigner(ctx, metakv,
func() (Timestamp, error) { return m.tsoAllocator.AllocOne() },
@@ -218,6 +218,7 @@ func CreateServer(ctx context.Context) (*Master, error) {
m.grpcServer = grpc.NewServer()
masterpb.RegisterMasterServer(m.grpcServer, m)
+
return m, nil
}
diff --git a/internal/master/master_test.go b/internal/master/master_test.go
index 0a44ed90e886b55d6b7bd5bec9e2d1842041fd2c..a605e73aa76127c24918cdd3826fae0d0d186ad8 100644
--- a/internal/master/master_test.go
+++ b/internal/master/master_test.go
@@ -110,6 +110,7 @@ func TestMaster(t *testing.T) {
conn, err := grpc.DialContext(ctx, Params.Address, grpc.WithInsecure(), grpc.WithBlock())
require.Nil(t, err)
+
cli := masterpb.NewMasterClient(conn)
t.Run("TestConfigTask", func(t *testing.T) {
@@ -886,12 +887,6 @@ func TestMaster(t *testing.T) {
var k2sMsgstream ms.MsgStream = k2sMs
assert.True(t, receiveTimeTickMsg(&k2sMsgstream))
- conn, err := grpc.DialContext(ctx, Params.Address, grpc.WithInsecure(), grpc.WithBlock())
- assert.Nil(t, err)
- defer conn.Close()
-
- cli := masterpb.NewMasterClient(conn)
-
sch := schemapb.CollectionSchema{
Name: "name" + strconv.FormatUint(rand.Uint64(), 10),
Description: "test collection",
diff --git a/internal/master/persistence_scheduler_test.go b/internal/master/persistence_scheduler_test.go
index 44110f3a0b94a4f29ba5dc7036bf42bf41616ab5..917e520f29757d91e3b5ada6993b5d28860373bd 100644
--- a/internal/master/persistence_scheduler_test.go
+++ b/internal/master/persistence_scheduler_test.go
@@ -59,7 +59,11 @@ func TestPersistenceScheduler(t *testing.T) {
//Init scheduler
indexLoadSch := NewIndexLoadScheduler(ctx, loadIndexClient, meta)
indexBuildSch := NewIndexBuildScheduler(ctx, buildIndexClient, meta, indexLoadSch)
- flushSch := NewFlushScheduler(ctx, flushClient, meta, indexBuildSch)
+ cnt := 0
+ flushSch := NewFlushScheduler(ctx, flushClient, meta, indexBuildSch, func() (Timestamp, error) {
+ cnt++
+ return Timestamp(cnt), nil
+ })
//scheduler start
err = indexLoadSch.Start()
diff --git a/internal/msgstream/msg.go b/internal/msgstream/msg.go
index 518bcfa7afe56a34cf86ff1464eb309a42560a9c..a71d1cabfe89745cfdfaec2d3411c4e3521c8b19 100644
--- a/internal/msgstream/msg.go
+++ b/internal/msgstream/msg.go
@@ -1,6 +1,8 @@
package msgstream
import (
+ "context"
+
"github.com/golang/protobuf/proto"
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
)
@@ -8,6 +10,8 @@ import (
type MsgType = internalPb.MsgType
type TsMsg interface {
+ GetContext() context.Context
+ SetContext(context.Context)
BeginTs() Timestamp
EndTs() Timestamp
Type() MsgType
@@ -17,6 +21,7 @@ type TsMsg interface {
}
type BaseMsg struct {
+ ctx context.Context
BeginTimestamp Timestamp
EndTimestamp Timestamp
HashValues []uint32
@@ -44,6 +49,14 @@ func (it *InsertMsg) Type() MsgType {
return it.MsgType
}
+func (it *InsertMsg) GetContext() context.Context {
+ return it.ctx
+}
+
+func (it *InsertMsg) SetContext(ctx context.Context) {
+ it.ctx = ctx
+}
+
func (it *InsertMsg) Marshal(input TsMsg) ([]byte, error) {
insertMsg := input.(*InsertMsg)
insertRequest := &insertMsg.InsertRequest
@@ -88,6 +101,13 @@ func (fl *FlushMsg) Type() MsgType {
return fl.GetMsgType()
}
+func (fl *FlushMsg) GetContext() context.Context {
+ return fl.ctx
+}
+func (fl *FlushMsg) SetContext(ctx context.Context) {
+ fl.ctx = ctx
+}
+
func (fl *FlushMsg) Marshal(input TsMsg) ([]byte, error) {
flushMsgTask := input.(*FlushMsg)
flushMsg := &flushMsgTask.FlushMsg
@@ -121,6 +141,14 @@ func (dt *DeleteMsg) Type() MsgType {
return dt.MsgType
}
+func (dt *DeleteMsg) GetContext() context.Context {
+ return dt.ctx
+}
+
+func (dt *DeleteMsg) SetContext(ctx context.Context) {
+ dt.ctx = ctx
+}
+
func (dt *DeleteMsg) Marshal(input TsMsg) ([]byte, error) {
deleteTask := input.(*DeleteMsg)
deleteRequest := &deleteTask.DeleteRequest
@@ -165,6 +193,14 @@ func (st *SearchMsg) Type() MsgType {
return st.MsgType
}
+func (st *SearchMsg) GetContext() context.Context {
+ return st.ctx
+}
+
+func (st *SearchMsg) SetContext(ctx context.Context) {
+ st.ctx = ctx
+}
+
func (st *SearchMsg) Marshal(input TsMsg) ([]byte, error) {
searchTask := input.(*SearchMsg)
searchRequest := &searchTask.SearchRequest
@@ -198,6 +234,14 @@ func (srt *SearchResultMsg) Type() MsgType {
return srt.MsgType
}
+func (srt *SearchResultMsg) GetContext() context.Context {
+ return srt.ctx
+}
+
+func (srt *SearchResultMsg) SetContext(ctx context.Context) {
+ srt.ctx = ctx
+}
+
func (srt *SearchResultMsg) Marshal(input TsMsg) ([]byte, error) {
searchResultTask := input.(*SearchResultMsg)
searchResultRequest := &searchResultTask.SearchResult
@@ -231,6 +275,14 @@ func (tst *TimeTickMsg) Type() MsgType {
return tst.MsgType
}
+func (tst *TimeTickMsg) GetContext() context.Context {
+ return tst.ctx
+}
+
+func (tst *TimeTickMsg) SetContext(ctx context.Context) {
+ tst.ctx = ctx
+}
+
func (tst *TimeTickMsg) Marshal(input TsMsg) ([]byte, error) {
timeTickTask := input.(*TimeTickMsg)
timeTick := &timeTickTask.TimeTickMsg
@@ -264,6 +316,14 @@ func (qs *QueryNodeStatsMsg) Type() MsgType {
return qs.MsgType
}
+func (qs *QueryNodeStatsMsg) GetContext() context.Context {
+ return qs.ctx
+}
+
+func (qs *QueryNodeStatsMsg) SetContext(ctx context.Context) {
+ qs.ctx = ctx
+}
+
func (qs *QueryNodeStatsMsg) Marshal(input TsMsg) ([]byte, error) {
queryNodeSegStatsTask := input.(*QueryNodeStatsMsg)
queryNodeSegStats := &queryNodeSegStatsTask.QueryNodeStats
@@ -305,6 +365,14 @@ func (cc *CreateCollectionMsg) Type() MsgType {
return cc.MsgType
}
+func (cc *CreateCollectionMsg) GetContext() context.Context {
+ return cc.ctx
+}
+
+func (cc *CreateCollectionMsg) SetContext(ctx context.Context) {
+ cc.ctx = ctx
+}
+
func (cc *CreateCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
createCollectionMsg := input.(*CreateCollectionMsg)
createCollectionRequest := &createCollectionMsg.CreateCollectionRequest
@@ -337,6 +405,13 @@ type DropCollectionMsg struct {
func (dc *DropCollectionMsg) Type() MsgType {
return dc.MsgType
}
+func (dc *DropCollectionMsg) GetContext() context.Context {
+ return dc.ctx
+}
+
+func (dc *DropCollectionMsg) SetContext(ctx context.Context) {
+ dc.ctx = ctx
+}
func (dc *DropCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
dropCollectionMsg := input.(*DropCollectionMsg)
@@ -361,109 +436,18 @@ func (dc *DropCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
return dropCollectionMsg, nil
}
-/////////////////////////////////////////HasCollection//////////////////////////////////////////
-type HasCollectionMsg struct {
- BaseMsg
- internalPb.HasCollectionRequest
-}
-
-func (hc *HasCollectionMsg) Type() MsgType {
- return hc.MsgType
-}
-
-func (hc *HasCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
- hasCollectionMsg := input.(*HasCollectionMsg)
- hasCollectionRequest := &hasCollectionMsg.HasCollectionRequest
- mb, err := proto.Marshal(hasCollectionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (hc *HasCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
- hasCollectionRequest := internalPb.HasCollectionRequest{}
- err := proto.Unmarshal(input, &hasCollectionRequest)
- if err != nil {
- return nil, err
- }
- hasCollectionMsg := &HasCollectionMsg{HasCollectionRequest: hasCollectionRequest}
- hasCollectionMsg.BeginTimestamp = hasCollectionMsg.Timestamp
- hasCollectionMsg.EndTimestamp = hasCollectionMsg.Timestamp
-
- return hasCollectionMsg, nil
-}
-
-/////////////////////////////////////////DescribeCollection//////////////////////////////////////////
-type DescribeCollectionMsg struct {
- BaseMsg
- internalPb.DescribeCollectionRequest
-}
-
-func (dc *DescribeCollectionMsg) Type() MsgType {
- return dc.MsgType
-}
-
-func (dc *DescribeCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
- describeCollectionMsg := input.(*DescribeCollectionMsg)
- describeCollectionRequest := &describeCollectionMsg.DescribeCollectionRequest
- mb, err := proto.Marshal(describeCollectionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (dc *DescribeCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
- describeCollectionRequest := internalPb.DescribeCollectionRequest{}
- err := proto.Unmarshal(input, &describeCollectionRequest)
- if err != nil {
- return nil, err
- }
- describeCollectionMsg := &DescribeCollectionMsg{DescribeCollectionRequest: describeCollectionRequest}
- describeCollectionMsg.BeginTimestamp = describeCollectionMsg.Timestamp
- describeCollectionMsg.EndTimestamp = describeCollectionMsg.Timestamp
-
- return describeCollectionMsg, nil
-}
-
-/////////////////////////////////////////ShowCollection//////////////////////////////////////////
-type ShowCollectionMsg struct {
+/////////////////////////////////////////CreatePartition//////////////////////////////////////////
+type CreatePartitionMsg struct {
BaseMsg
- internalPb.ShowCollectionRequest
-}
-
-func (sc *ShowCollectionMsg) Type() MsgType {
- return sc.MsgType
-}
-
-func (sc *ShowCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
- showCollectionMsg := input.(*ShowCollectionMsg)
- showCollectionRequest := &showCollectionMsg.ShowCollectionRequest
- mb, err := proto.Marshal(showCollectionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
+ internalPb.CreatePartitionRequest
}
-func (sc *ShowCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
- showCollectionRequest := internalPb.ShowCollectionRequest{}
- err := proto.Unmarshal(input, &showCollectionRequest)
- if err != nil {
- return nil, err
- }
- showCollectionMsg := &ShowCollectionMsg{ShowCollectionRequest: showCollectionRequest}
- showCollectionMsg.BeginTimestamp = showCollectionMsg.Timestamp
- showCollectionMsg.EndTimestamp = showCollectionMsg.Timestamp
-
- return showCollectionMsg, nil
+func (cc *CreatePartitionMsg) GetContext() context.Context {
+ return cc.ctx
}
-/////////////////////////////////////////CreatePartition//////////////////////////////////////////
-type CreatePartitionMsg struct {
- BaseMsg
- internalPb.CreatePartitionRequest
+func (cc *CreatePartitionMsg) SetContext(ctx context.Context) {
+ cc.ctx = ctx
}
func (cc *CreatePartitionMsg) Type() MsgType {
@@ -499,6 +483,14 @@ type DropPartitionMsg struct {
internalPb.DropPartitionRequest
}
+func (dc *DropPartitionMsg) GetContext() context.Context {
+ return dc.ctx
+}
+
+func (dc *DropPartitionMsg) SetContext(ctx context.Context) {
+ dc.ctx = ctx
+}
+
func (dc *DropPartitionMsg) Type() MsgType {
return dc.MsgType
}
@@ -526,105 +518,6 @@ func (dc *DropPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
return dropPartitionMsg, nil
}
-/////////////////////////////////////////HasPartition//////////////////////////////////////////
-type HasPartitionMsg struct {
- BaseMsg
- internalPb.HasPartitionRequest
-}
-
-func (hc *HasPartitionMsg) Type() MsgType {
- return hc.MsgType
-}
-
-func (hc *HasPartitionMsg) Marshal(input TsMsg) ([]byte, error) {
- hasPartitionMsg := input.(*HasPartitionMsg)
- hasPartitionRequest := &hasPartitionMsg.HasPartitionRequest
- mb, err := proto.Marshal(hasPartitionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (hc *HasPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
- hasPartitionRequest := internalPb.HasPartitionRequest{}
- err := proto.Unmarshal(input, &hasPartitionRequest)
- if err != nil {
- return nil, err
- }
- hasPartitionMsg := &HasPartitionMsg{HasPartitionRequest: hasPartitionRequest}
- hasPartitionMsg.BeginTimestamp = hasPartitionMsg.Timestamp
- hasPartitionMsg.EndTimestamp = hasPartitionMsg.Timestamp
-
- return hasPartitionMsg, nil
-}
-
-/////////////////////////////////////////DescribePartition//////////////////////////////////////////
-type DescribePartitionMsg struct {
- BaseMsg
- internalPb.DescribePartitionRequest
-}
-
-func (dc *DescribePartitionMsg) Type() MsgType {
- return dc.MsgType
-}
-
-func (dc *DescribePartitionMsg) Marshal(input TsMsg) ([]byte, error) {
- describePartitionMsg := input.(*DescribePartitionMsg)
- describePartitionRequest := &describePartitionMsg.DescribePartitionRequest
- mb, err := proto.Marshal(describePartitionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (dc *DescribePartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
- describePartitionRequest := internalPb.DescribePartitionRequest{}
- err := proto.Unmarshal(input, &describePartitionRequest)
- if err != nil {
- return nil, err
- }
- describePartitionMsg := &DescribePartitionMsg{DescribePartitionRequest: describePartitionRequest}
- describePartitionMsg.BeginTimestamp = describePartitionMsg.Timestamp
- describePartitionMsg.EndTimestamp = describePartitionMsg.Timestamp
-
- return describePartitionMsg, nil
-}
-
-/////////////////////////////////////////ShowPartition//////////////////////////////////////////
-type ShowPartitionMsg struct {
- BaseMsg
- internalPb.ShowPartitionRequest
-}
-
-func (sc *ShowPartitionMsg) Type() MsgType {
- return sc.MsgType
-}
-
-func (sc *ShowPartitionMsg) Marshal(input TsMsg) ([]byte, error) {
- showPartitionMsg := input.(*ShowPartitionMsg)
- showPartitionRequest := &showPartitionMsg.ShowPartitionRequest
- mb, err := proto.Marshal(showPartitionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (sc *ShowPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
- showPartitionRequest := internalPb.ShowPartitionRequest{}
- err := proto.Unmarshal(input, &showPartitionRequest)
- if err != nil {
- return nil, err
- }
- showPartitionMsg := &ShowPartitionMsg{ShowPartitionRequest: showPartitionRequest}
- showPartitionMsg.BeginTimestamp = showPartitionMsg.Timestamp
- showPartitionMsg.EndTimestamp = showPartitionMsg.Timestamp
-
- return showPartitionMsg, nil
-}
-
/////////////////////////////////////////LoadIndex//////////////////////////////////////////
type LoadIndexMsg struct {
BaseMsg
@@ -635,6 +528,14 @@ func (lim *LoadIndexMsg) Type() MsgType {
return lim.MsgType
}
+func (lim *LoadIndexMsg) GetContext() context.Context {
+ return lim.ctx
+}
+
+func (lim *LoadIndexMsg) SetContext(ctx context.Context) {
+ lim.ctx = ctx
+}
+
func (lim *LoadIndexMsg) Marshal(input TsMsg) ([]byte, error) {
loadIndexMsg := input.(*LoadIndexMsg)
loadIndexRequest := &loadIndexMsg.LoadIndex
diff --git a/internal/msgstream/msgstream.go b/internal/msgstream/msgstream.go
index 37dd71c053441673334cbda1c60adac9fdc5fc5f..969755feb322a3ac33619500a061dd9a0f984c00 100644
--- a/internal/msgstream/msgstream.go
+++ b/internal/msgstream/msgstream.go
@@ -4,9 +4,13 @@ import (
"context"
"log"
"reflect"
+ "strings"
"sync"
"time"
+ "github.com/opentracing/opentracing-go"
+ "github.com/opentracing/opentracing-go/ext"
+
"github.com/apache/pulsar-client-go/pulsar"
"github.com/golang/protobuf/proto"
@@ -151,6 +155,29 @@ func (ms *PulsarMsgStream) Close() {
}
}
+type propertiesReaderWriter struct {
+ ppMap map[string]string
+}
+
+func (ppRW *propertiesReaderWriter) Set(key, val string) {
+ // The GRPC HPACK implementation rejects any uppercase keys here.
+ //
+ // As such, since the HTTP_HEADERS format is case-insensitive anyway, we
+ // blindly lowercase the key (which is guaranteed to work in the
+ // Inject/Extract sense per the OpenTracing spec).
+ key = strings.ToLower(key)
+ ppRW.ppMap[key] = val
+}
+
+func (ppRW *propertiesReaderWriter) ForeachKey(handler func(key, val string) error) error {
+ for k, val := range ppRW.ppMap {
+ if err := handler(k, val); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
tsMsgs := msgPack.Msgs
if len(tsMsgs) <= 0 {
@@ -200,12 +227,41 @@ func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
if err != nil {
return err
}
+
+ msg := &pulsar.ProducerMessage{Payload: mb}
+ var child opentracing.Span
+ if v.Msgs[i].Type() == internalPb.MsgType_kInsert || v.Msgs[i].Type() == internalPb.MsgType_kSearch {
+ tracer := opentracing.GlobalTracer()
+ ctx := v.Msgs[i].GetContext()
+ if ctx == nil {
+ ctx = context.Background()
+ }
+
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan("start send pulsar msg",
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan("start send pulsar msg")
+ }
+ child.SetTag("hash keys", v.Msgs[i].HashKeys())
+ child.SetTag("start time", v.Msgs[i].BeginTs())
+ child.SetTag("end time", v.Msgs[i].EndTs())
+ msg.Properties = make(map[string]string)
+ err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
+ if err != nil {
+ return err
+ }
+ }
+
if _, err := (*ms.producers[k]).Send(
context.Background(),
- &pulsar.ProducerMessage{Payload: mb},
+ msg,
); err != nil {
return err
}
+ if child != nil {
+ child.Finish()
+ }
}
}
return nil
@@ -218,10 +274,34 @@ func (ms *PulsarMsgStream) Broadcast(msgPack *MsgPack) error {
if err != nil {
return err
}
+ msg := &pulsar.ProducerMessage{Payload: mb}
+ if v.Type() == internalPb.MsgType_kInsert || v.Type() == internalPb.MsgType_kSearch {
+ tracer := opentracing.GlobalTracer()
+ ctx := v.GetContext()
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ var child opentracing.Span
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan("start send pulsar msg",
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan("start send pulsar msg, start time: %d")
+ }
+ child.SetTag("hash keys", v.HashKeys())
+ child.SetTag("start time", v.BeginTs())
+ child.SetTag("end time", v.EndTs())
+ msg.Properties = make(map[string]string)
+ err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
+ if err != nil {
+ return err
+ }
+ child.Finish()
+ }
for i := 0; i < producerLen; i++ {
if _, err := (*ms.producers[i]).Send(
context.Background(),
- &pulsar.ProducerMessage{Payload: mb},
+ msg,
); err != nil {
return err
}
@@ -258,6 +338,7 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
for {
select {
case <-ms.ctx.Done():
+ log.Println("done")
return
default:
tsMsgList := make([]TsMsg, 0)
@@ -270,6 +351,7 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
}
pulsarMsg, ok := value.Interface().(pulsar.ConsumerMessage)
+
if !ok {
log.Printf("type assertion failed, not consumer message type")
continue
@@ -283,6 +365,21 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
continue
}
tsMsg, err := ms.unmarshal.Unmarshal(pulsarMsg.Payload(), headerMsg.MsgType)
+ if tsMsg.Type() == internalPb.MsgType_kInsert || tsMsg.Type() == internalPb.MsgType_kSearch {
+ tracer := opentracing.GlobalTracer()
+ spanContext, err := tracer.Extract(opentracing.HTTPHeaders, &propertiesReaderWriter{pulsarMsg.Properties()})
+ if err != nil {
+ log.Println("extract message err")
+ log.Println(err.Error())
+ }
+ span := opentracing.StartSpan("pulsar msg received",
+ ext.RPCServerOption(spanContext))
+ span.SetTag("hash keys", tsMsg.HashKeys())
+ span.SetTag("start time", tsMsg.BeginTs())
+ span.SetTag("end time", tsMsg.EndTs())
+ tsMsg.SetContext(opentracing.ContextWithSpan(context.Background(), span))
+ span.Finish()
+ }
if err != nil {
log.Printf("Failed to unmarshal tsMsg, error = %v", err)
continue
@@ -420,6 +517,23 @@ func (ms *PulsarTtMsgStream) findTimeTick(channelIndex int,
if err != nil {
log.Printf("Failed to unmarshal, error = %v", err)
}
+
+ if tsMsg.Type() == internalPb.MsgType_kInsert || tsMsg.Type() == internalPb.MsgType_kSearch {
+ tracer := opentracing.GlobalTracer()
+ spanContext, err := tracer.Extract(opentracing.HTTPHeaders, &propertiesReaderWriter{pulsarMsg.Properties()})
+ if err != nil {
+ log.Println("extract message err")
+ log.Println(err.Error())
+ }
+ span := opentracing.StartSpan("pulsar msg received",
+ ext.RPCServerOption(spanContext))
+ span.SetTag("hash keys", tsMsg.HashKeys())
+ span.SetTag("start time", tsMsg.BeginTs())
+ span.SetTag("end time", tsMsg.EndTs())
+ tsMsg.SetContext(opentracing.ContextWithSpan(context.Background(), span))
+ span.Finish()
+ }
+
if headerMsg.MsgType == internalPb.MsgType_kTimeTick {
eofMsgMap[channelIndex] = tsMsg.(*TimeTickMsg).Timestamp
return
@@ -500,7 +614,7 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
result := make(map[int32]*MsgPack)
for i, request := range tsMsgs {
if request.Type() != internalPb.MsgType_kInsert {
- return nil, errors.New(string("msg's must be Insert"))
+ return nil, errors.New("msg's must be Insert")
}
insertRequest := request.(*InsertMsg)
keys := hashKeys[i]
@@ -511,7 +625,7 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
keysLen := len(keys)
if keysLen != timestampLen || keysLen != rowIDLen || keysLen != rowDataLen {
- return nil, errors.New(string("the length of hashValue, timestamps, rowIDs, RowData are not equal"))
+ return nil, errors.New("the length of hashValue, timestamps, rowIDs, RowData are not equal")
}
for index, key := range keys {
_, ok := result[key]
@@ -534,6 +648,9 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
}
insertMsg := &InsertMsg{
+ BaseMsg: BaseMsg{
+ ctx: request.GetContext(),
+ },
InsertRequest: sliceRequest,
}
result[key].Msgs = append(result[key].Msgs, insertMsg)
@@ -546,7 +663,7 @@ func deleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
result := make(map[int32]*MsgPack)
for i, request := range tsMsgs {
if request.Type() != internalPb.MsgType_kDelete {
- return nil, errors.New(string("msg's must be Delete"))
+ return nil, errors.New("msg's must be Delete")
}
deleteRequest := request.(*DeleteMsg)
keys := hashKeys[i]
@@ -556,7 +673,7 @@ func deleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
keysLen := len(keys)
if keysLen != timestampLen || keysLen != primaryKeysLen {
- return nil, errors.New(string("the length of hashValue, timestamps, primaryKeys are not equal"))
+ return nil, errors.New("the length of hashValue, timestamps, primaryKeys are not equal")
}
for index, key := range keys {
@@ -590,7 +707,7 @@ func defaultRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack,
for i, request := range tsMsgs {
keys := hashKeys[i]
if len(keys) != 1 {
- return nil, errors.New(string("len(msg.hashValue) must equal 1"))
+ return nil, errors.New("len(msg.hashValue) must equal 1")
}
key := keys[0]
_, ok := result[key]
diff --git a/internal/proto/internal_msg.proto b/internal/proto/internal_msg.proto
index cb934231a2acdce48430f9355f4707dea9ccfd82..6dbfb483536c716f3c07338e26293da2ae14208d 100644
--- a/internal/proto/internal_msg.proto
+++ b/internal/proto/internal_msg.proto
@@ -272,6 +272,8 @@ message FlushMsg {
MsgType msg_type = 1;
int64 segmentID = 2;
uint64 timestamp = 3;
+ int64 collectionID = 4;
+ string partitionTag = 5;
}
diff --git a/internal/proto/internalpb/internal_msg.pb.go b/internal/proto/internalpb/internal_msg.pb.go
index d159b04dde1455ca0828481abee5a287c49a72c4..d290d5e2745d21ae359f5dd5760be813515d1c24 100644
--- a/internal/proto/internalpb/internal_msg.pb.go
+++ b/internal/proto/internalpb/internal_msg.pb.go
@@ -1881,6 +1881,8 @@ type FlushMsg struct {
MsgType MsgType `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType" json:"msg_type,omitempty"`
SegmentID int64 `protobuf:"varint,2,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+ CollectionID int64 `protobuf:"varint,4,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
+ PartitionTag string `protobuf:"bytes,5,opt,name=partitionTag,proto3" json:"partitionTag,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@@ -1932,6 +1934,20 @@ func (m *FlushMsg) GetTimestamp() uint64 {
return 0
}
+func (m *FlushMsg) GetCollectionID() int64 {
+ if m != nil {
+ return m.CollectionID
+ }
+ return 0
+}
+
+func (m *FlushMsg) GetPartitionTag() string {
+ if m != nil {
+ return m.PartitionTag
+ }
+ return ""
+}
+
type Key2Seg struct {
RowID int64 `protobuf:"varint,1,opt,name=rowID,proto3" json:"rowID,omitempty"`
PrimaryKey int64 `protobuf:"varint,2,opt,name=primary_key,json=primaryKey,proto3" json:"primary_key,omitempty"`
@@ -2661,121 +2677,122 @@ func init() {
func init() { proto.RegisterFile("internal_msg.proto", fileDescriptor_7eb37f6b80b23116) }
var fileDescriptor_7eb37f6b80b23116 = []byte{
- // 1852 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x99, 0xcd, 0x6f, 0x23, 0x49,
- 0x15, 0xc0, 0xb7, 0xbb, 0xfd, 0x11, 0x3f, 0x3b, 0x4e, 0x4f, 0x25, 0x99, 0xf1, 0xec, 0x2c, 0x3b,
- 0x99, 0x1e, 0xc4, 0x86, 0x45, 0x24, 0x90, 0xe1, 0xc0, 0xde, 0x20, 0xb1, 0x96, 0x35, 0x43, 0x46,
- 0xa1, 0x13, 0x2d, 0x12, 0x5a, 0xa9, 0xd5, 0xb1, 0x5f, 0xec, 0x52, 0x7f, 0x39, 0x55, 0xed, 0x49,
- 0x3c, 0x07, 0x24, 0xc4, 0x9c, 0x11, 0x1f, 0xe2, 0xc0, 0x8d, 0x3b, 0xec, 0x8a, 0x05, 0xf1, 0x3f,
- 0xf0, 0x29, 0x24, 0xfe, 0x0b, 0x38, 0x80, 0xc4, 0x2e, 0x07, 0x6e, 0xa8, 0xaa, 0xfa, 0xc3, 0x9d,
- 0xd8, 0x4e, 0xb4, 0xc9, 0x2c, 0xb3, 0xda, 0xb9, 0x75, 0x3d, 0x57, 0x57, 0xbf, 0xf7, 0x7b, 0x1f,
- 0x55, 0xf5, 0x0c, 0x84, 0x86, 0x31, 0xb2, 0xd0, 0xf5, 0x9d, 0x80, 0xf7, 0x37, 0x86, 0x2c, 0x8a,
- 0x23, 0xb2, 0x1a, 0x50, 0xff, 0xf1, 0x88, 0xab, 0xd1, 0x46, 0x3a, 0xe1, 0xe5, 0x46, 0x37, 0x0a,
- 0x82, 0x28, 0x54, 0xe2, 0x97, 0x6f, 0x70, 0x64, 0x8f, 0x69, 0x17, 0xf3, 0xf7, 0xac, 0x10, 0x6a,
- 0x9d, 0xb6, 0x8d, 0xc7, 0x23, 0xe4, 0x31, 0xb9, 0x09, 0x95, 0x21, 0x22, 0xeb, 0xb4, 0x5b, 0xda,
- 0x9a, 0xb6, 0x6e, 0xd8, 0xc9, 0x88, 0x3c, 0x80, 0x12, 0x8b, 0x7c, 0x6c, 0xe9, 0x6b, 0xda, 0x7a,
- 0x73, 0xeb, 0xee, 0xc6, 0xd4, 0x6f, 0x6d, 0xec, 0x21, 0x32, 0x3b, 0xf2, 0xd1, 0x96, 0x93, 0xc9,
- 0x0a, 0x94, 0xbb, 0xd1, 0x28, 0x8c, 0x5b, 0xc6, 0x9a, 0xb6, 0xbe, 0x68, 0xab, 0x81, 0xd5, 0x07,
- 0x10, 0xdf, 0xe3, 0xc3, 0x28, 0xe4, 0x48, 0x1e, 0x40, 0x85, 0xc7, 0x6e, 0x3c, 0xe2, 0xf2, 0x83,
- 0xf5, 0xad, 0x3b, 0xc5, 0xa5, 0x13, 0xe5, 0xf7, 0xe5, 0x14, 0x3b, 0x99, 0x4a, 0x9a, 0xa0, 0x77,
- 0xda, 0x52, 0x17, 0xc3, 0xd6, 0x3b, 0xed, 0x19, 0x1f, 0x8a, 0x00, 0x0e, 0x78, 0xf4, 0x31, 0x5a,
- 0xf6, 0x18, 0xea, 0xf2, 0x83, 0x57, 0x31, 0xed, 0x15, 0xa8, 0xc5, 0x34, 0x40, 0x1e, 0xbb, 0xc1,
- 0x50, 0xea, 0x54, 0xb2, 0x73, 0xc1, 0x8c, 0xef, 0x3e, 0xd5, 0xa0, 0xb1, 0x8f, 0xfd, 0xdc, 0x8b,
- 0xd9, 0x34, 0x6d, 0x62, 0x9a, 0x58, 0xba, 0x3b, 0x70, 0xc3, 0x10, 0xfd, 0x04, 0x5e, 0xd9, 0xce,
- 0x05, 0xe4, 0x0e, 0xd4, 0xba, 0x91, 0xef, 0x3b, 0xa1, 0x1b, 0xa0, 0x5c, 0xbe, 0x66, 0x2f, 0x08,
- 0xc1, 0x23, 0x37, 0x40, 0x72, 0x1f, 0x16, 0x87, 0x2e, 0x8b, 0x69, 0x4c, 0xa3, 0xd0, 0x89, 0xdd,
- 0x7e, 0xab, 0x24, 0x27, 0x34, 0x32, 0xe1, 0x81, 0xdb, 0xb7, 0xde, 0xd3, 0x80, 0x7c, 0x9d, 0x73,
- 0xda, 0x0f, 0x0b, 0xca, 0x5c, 0x2b, 0xf8, 0x87, 0xb0, 0x34, 0x44, 0xe6, 0x24, 0x6a, 0x3b, 0x0c,
- 0x8f, 0x5b, 0xc6, 0x9a, 0xb1, 0x5e, 0xdf, 0xba, 0x3f, 0xe3, 0xfd, 0x49, 0x55, 0xec, 0xc5, 0x21,
- 0xb2, 0x1d, 0xf5, 0xaa, 0x8d, 0xc7, 0xd6, 0x07, 0x1a, 0x2c, 0xc9, 0xdf, 0x95, 0xd6, 0x01, 0x86,
- 0x12, 0x1d, 0x17, 0xa2, 0x44, 0x59, 0x35, 0xb8, 0x00, 0xdd, 0x54, 0xaf, 0x14, 0x81, 0x96, 0x2e,
- 0x02, 0x5a, 0x3e, 0x0f, 0x94, 0xdc, 0x85, 0x3a, 0x9e, 0x0e, 0x29, 0x43, 0x47, 0x44, 0x40, 0xab,
- 0x22, 0xa3, 0x01, 0x94, 0xe8, 0x80, 0x06, 0x93, 0x11, 0x56, 0xbd, 0x74, 0x84, 0x59, 0x1c, 0x96,
- 0x0b, 0x5e, 0x4a, 0xa2, 0xf5, 0x1d, 0xb8, 0x39, 0x49, 0xd6, 0xcd, 0x90, 0xb4, 0x34, 0x09, 0xf8,
- 0x73, 0xf3, 0x00, 0xe7, 0x00, 0xed, 0x95, 0x9c, 0x71, 0x2e, 0xb5, 0xfe, 0xab, 0xc1, 0xad, 0x1d,
- 0x86, 0x6e, 0x8c, 0x3b, 0x91, 0xef, 0x63, 0x57, 0x98, 0x98, 0x06, 0xc8, 0x1b, 0xb0, 0x10, 0xf0,
- 0xbe, 0x13, 0x8f, 0x87, 0x28, 0xa9, 0x37, 0xb7, 0x5e, 0x9d, 0xf1, 0xad, 0x5d, 0xde, 0x3f, 0x18,
- 0x0f, 0xd1, 0xae, 0x06, 0xea, 0x81, 0x58, 0xd0, 0xe8, 0x66, 0xeb, 0x65, 0x25, 0xa1, 0x20, 0x13,
- 0xde, 0x61, 0x78, 0xdc, 0x69, 0x4b, 0xef, 0x18, 0xb6, 0x1a, 0x14, 0xf3, 0xac, 0x74, 0x36, 0xcf,
- 0x5a, 0x50, 0x1d, 0xb2, 0xe8, 0x74, 0xdc, 0x69, 0x4b, 0xc7, 0x18, 0x76, 0x3a, 0x24, 0x5f, 0x86,
- 0x0a, 0xef, 0x0e, 0x30, 0x70, 0xa5, 0x3b, 0xea, 0x5b, 0xb7, 0xa7, 0x22, 0xdf, 0xf6, 0xa3, 0x43,
- 0x3b, 0x99, 0x68, 0xfd, 0x54, 0x87, 0xd5, 0x36, 0x8b, 0x86, 0x9f, 0x70, 0xcb, 0x77, 0x61, 0x29,
- 0x5f, 0x5d, 0x45, 0xb5, 0x42, 0xf0, 0xd9, 0xa2, 0xce, 0xc9, 0x0e, 0xb3, 0x91, 0x9b, 0x2b, 0x22,
- 0xde, 0x6e, 0x76, 0x0b, 0x63, 0xeb, 0x9f, 0x1a, 0xac, 0xbc, 0xe5, 0xf2, 0x6b, 0x85, 0x92, 0x19,
- 0xac, 0xcf, 0x34, 0xd8, 0x98, 0x63, 0x70, 0xe9, 0x42, 0x83, 0xcb, 0x57, 0x30, 0xf8, 0x03, 0x0d,
- 0x6e, 0xb7, 0x91, 0x77, 0x19, 0x3d, 0xc4, 0x4f, 0x8f, 0xd5, 0xbf, 0xd0, 0x60, 0x75, 0x7f, 0x10,
- 0x9d, 0x3c, 0xbf, 0x16, 0x5b, 0xbf, 0xd5, 0xe1, 0xa6, 0xaa, 0x4d, 0x7b, 0x69, 0xf5, 0xfd, 0x98,
- 0x12, 0x74, 0x0d, 0xea, 0x59, 0xc1, 0xcf, 0xd2, 0x74, 0x52, 0x94, 0x5b, 0x5a, 0x9a, 0x69, 0x69,
- 0x79, 0x8e, 0xa5, 0x95, 0xa2, 0x6f, 0xbf, 0x09, 0xcd, 0x7c, 0xd7, 0x91, 0xae, 0x55, 0xfb, 0xc6,
- 0xfd, 0xe9, 0xae, 0xcd, 0x70, 0x48, 0xcf, 0xe6, 0x1b, 0x96, 0x74, 0xec, 0xfb, 0x3a, 0xac, 0x88,
- 0xaa, 0xf6, 0x82, 0xd9, 0xe5, 0x99, 0xfd, 0x43, 0x83, 0xe5, 0xb7, 0x5c, 0x7e, 0x9d, 0xc8, 0xae,
- 0x37, 0xf9, 0xcf, 0x1b, 0x5b, 0xfe, 0xc8, 0xc6, 0xfe, 0x4b, 0x83, 0x56, 0x5a, 0xef, 0x3e, 0x1d,
- 0x16, 0x8b, 0x2d, 0x4d, 0xd4, 0xba, 0xe7, 0xd7, 0xda, 0x6b, 0x2e, 0xee, 0xff, 0xd6, 0x61, 0xb1,
- 0x13, 0x72, 0x64, 0xf1, 0x33, 0xb3, 0xf4, 0xb5, 0xf3, 0x1a, 0xab, 0xcb, 0xc9, 0x19, 0x5d, 0x2e,
- 0x75, 0x45, 0x11, 0xdc, 0x38, 0xf6, 0xc5, 0x89, 0x34, 0x3b, 0xdf, 0xe4, 0x82, 0xe2, 0x29, 0x5f,
- 0x95, 0x81, 0x89, 0x53, 0xfe, 0x04, 0xd5, 0x6a, 0x91, 0xea, 0xab, 0x00, 0x19, 0x7c, 0xde, 0x5a,
- 0x58, 0x33, 0xc4, 0x31, 0x3d, 0x97, 0x88, 0x1b, 0x10, 0x8b, 0x4e, 0x3a, 0x6d, 0xde, 0xaa, 0xad,
- 0x19, 0xe2, 0x06, 0xa4, 0x46, 0xe4, 0x2b, 0xb0, 0xc0, 0xa2, 0x13, 0xa7, 0xe7, 0xc6, 0x6e, 0x0b,
- 0xe4, 0x21, 0x7b, 0xce, 0x69, 0xb2, 0xca, 0xa2, 0x93, 0xb6, 0x1b, 0xbb, 0xd6, 0x53, 0x1d, 0x16,
- 0xdb, 0xe8, 0x63, 0x8c, 0xff, 0x7f, 0xe8, 0x05, 0x62, 0xa5, 0x39, 0xc4, 0xca, 0xf3, 0x88, 0x55,
- 0xce, 0x11, 0xbb, 0x07, 0x8d, 0x21, 0xa3, 0x81, 0xcb, 0xc6, 0x8e, 0x87, 0x63, 0x71, 0xbd, 0x31,
- 0x64, 0x95, 0x57, 0xb2, 0x87, 0x38, 0xe6, 0xd6, 0x87, 0x1a, 0x2c, 0xee, 0xa3, 0xcb, 0xba, 0x83,
- 0x67, 0x86, 0x61, 0x42, 0x7f, 0xa3, 0xa8, 0xff, 0xfc, 0x33, 0xf4, 0xe7, 0xc1, 0x64, 0xc8, 0x47,
- 0x7e, 0xec, 0xe4, 0x70, 0x14, 0x80, 0x25, 0x25, 0xdf, 0xc9, 0x10, 0x6d, 0x42, 0xf9, 0x78, 0x84,
- 0x6c, 0x7c, 0xf1, 0x6d, 0x42, 0xcd, 0xb3, 0xfe, 0xa6, 0x81, 0xb9, 0x3f, 0xe6, 0x3b, 0x51, 0x78,
- 0x44, 0xfb, 0xcf, 0x9d, 0xe5, 0x04, 0x4a, 0xd2, 0x5f, 0xe5, 0x35, 0x63, 0xbd, 0x66, 0xcb, 0x67,
- 0xe1, 0x4b, 0x0f, 0xc7, 0xce, 0x90, 0xe1, 0x11, 0x3d, 0x45, 0xe5, 0xed, 0x9a, 0x5d, 0xf7, 0x70,
- 0xbc, 0x97, 0x88, 0xac, 0xbf, 0xea, 0xd0, 0x48, 0x7d, 0x29, 0xf8, 0x5c, 0xc5, 0xa0, 0xfc, 0x4e,
- 0xac, 0x5f, 0xbe, 0xeb, 0x32, 0xfd, 0xa6, 0x34, 0xbb, 0x8e, 0xde, 0x83, 0x86, 0x74, 0x87, 0x13,
- 0x46, 0x3d, 0xcc, 0xbc, 0x5b, 0x97, 0xb2, 0x47, 0x52, 0x54, 0x04, 0x55, 0xb9, 0x4c, 0x88, 0x54,
- 0xa7, 0x87, 0x08, 0x81, 0xd2, 0x80, 0xc6, 0xaa, 0xae, 0x34, 0x6c, 0xf9, 0x4c, 0xee, 0x42, 0x3d,
- 0xc0, 0x98, 0xd1, 0xae, 0x42, 0x54, 0x93, 0xc9, 0x09, 0x4a, 0x24, 0x28, 0x58, 0xdf, 0x83, 0xfa,
- 0x01, 0x0d, 0xf0, 0x80, 0x76, 0xbd, 0x5d, 0xde, 0xbf, 0x0a, 0xcf, 0xbc, 0x7d, 0xa3, 0x17, 0xda,
- 0x37, 0x73, 0xb7, 0x20, 0xeb, 0xfb, 0x1a, 0x2c, 0xbc, 0xe9, 0x8f, 0xf8, 0xe0, 0x8a, 0x5f, 0x2f,
- 0x14, 0x6c, 0x7d, 0x4a, 0xc1, 0x9e, 0xa3, 0xc3, 0xcf, 0x35, 0xa8, 0x3e, 0xc4, 0xf1, 0xd6, 0x3e,
- 0xf6, 0xa5, 0x83, 0x45, 0xd1, 0x4d, 0xdb, 0x3a, 0x72, 0x20, 0x30, 0x4e, 0x94, 0x99, 0x64, 0x7d,
- 0xc8, 0xab, 0xcc, 0x05, 0xfb, 0xec, 0x6d, 0x58, 0xa0, 0xdc, 0x79, 0xec, 0xfa, 0xb4, 0x27, 0x03,
- 0x64, 0xc1, 0xae, 0x52, 0xfe, 0xb6, 0x18, 0x8a, 0x02, 0x97, 0xa9, 0xa9, 0xd2, 0xc1, 0xb0, 0x27,
- 0x24, 0xd6, 0x3b, 0x00, 0x89, 0x6a, 0x02, 0x50, 0x16, 0x7e, 0xda, 0x64, 0xf8, 0x7d, 0x15, 0xaa,
- 0x1e, 0x8e, 0xb7, 0x38, 0xf6, 0x5b, 0xba, 0xdc, 0x1d, 0x66, 0x51, 0x4b, 0x56, 0xb2, 0xd3, 0xe9,
- 0xd6, 0x0f, 0x74, 0xa8, 0x7d, 0x2b, 0x72, 0x7b, 0x9d, 0xb0, 0x87, 0xa7, 0xcf, 0x14, 0xff, 0x11,
- 0x45, 0xbf, 0xf7, 0x28, 0xdf, 0x20, 0x72, 0x81, 0xc8, 0x1e, 0x39, 0xc8, 0xb3, 0x27, 0x19, 0x0a,
- 0xec, 0x54, 0x68, 0xe6, 0x0c, 0xdd, 0x78, 0x90, 0x16, 0x0b, 0x90, 0xa2, 0x3d, 0x21, 0x21, 0x6d,
- 0x68, 0xa4, 0x13, 0x98, 0x1b, 0xa8, 0x92, 0x51, 0xdf, 0xba, 0x37, 0x35, 0x93, 0x1f, 0xe2, 0xf8,
- 0x6d, 0xd7, 0x1f, 0xe1, 0x9e, 0x4b, 0x99, 0x5d, 0x4f, 0x16, 0x11, 0x6f, 0x59, 0x4f, 0x35, 0x00,
- 0x49, 0x40, 0x24, 0xfb, 0xf9, 0x45, 0xb5, 0x8f, 0xb2, 0x28, 0xf9, 0x12, 0xac, 0x84, 0xa3, 0xc0,
- 0x61, 0xe8, 0xbb, 0x31, 0xf6, 0x9c, 0x04, 0x06, 0x4f, 0xe0, 0x90, 0x70, 0x14, 0xd8, 0xea, 0xa7,
- 0xfd, 0xe4, 0x17, 0xeb, 0x87, 0x1a, 0xc0, 0x9b, 0xc2, 0x72, 0xa5, 0xc6, 0xd9, 0x3b, 0x8e, 0x36,
- 0xe5, 0x8e, 0x33, 0x81, 0x4e, 0x2f, 0xa2, 0xdb, 0x4e, 0xd1, 0x89, 0xc2, 0xc5, 0x93, 0xde, 0xe7,
- 0xbd, 0x19, 0xee, 0xcc, 0x8d, 0x4f, 0xe8, 0xca, 0x67, 0xeb, 0x67, 0xaa, 0x5d, 0x2c, 0xb4, 0x53,
- 0x2a, 0x15, 0xbc, 0xac, 0x9d, 0xf5, 0xb2, 0xac, 0x35, 0x41, 0xc4, 0xc6, 0x0e, 0xa7, 0x4f, 0x30,
- 0x4d, 0x12, 0x25, 0xda, 0xa7, 0x4f, 0x50, 0xa4, 0x81, 0x44, 0x12, 0x9d, 0xf0, 0x74, 0xb7, 0x10,
- 0x18, 0xa2, 0x13, 0x4e, 0xbe, 0x00, 0x37, 0x18, 0x76, 0x31, 0x8c, 0xfd, 0xb1, 0x13, 0x44, 0x3d,
- 0x7a, 0x44, 0x31, 0x4d, 0x15, 0x33, 0xfd, 0x61, 0x37, 0x91, 0x5b, 0x7f, 0xd7, 0xa0, 0xf9, 0xed,
- 0xb4, 0x82, 0x2a, 0xcd, 0x9e, 0x41, 0xdd, 0xfa, 0x9a, 0x34, 0xb6, 0xc0, 0x6f, 0x4e, 0xef, 0x38,
- 0x83, 0x64, 0x2f, 0x70, 0xec, 0x2b, 0xa5, 0xb6, 0xa1, 0x2e, 0xdd, 0x91, 0xac, 0x51, 0x9a, 0xeb,
- 0x83, 0xdc, 0xf3, 0x36, 0x1c, 0x65, 0xcf, 0xd6, 0x2f, 0x75, 0x20, 0xaa, 0xe7, 0x20, 0x9d, 0xf4,
- 0xdc, 0x5d, 0x14, 0x5e, 0x9b, 0x7e, 0x51, 0x38, 0x7f, 0x02, 0xfc, 0x0c, 0x28, 0xb3, 0xf2, 0x86,
- 0x60, 0xa1, 0x08, 0xb4, 0xa1, 0x81, 0xa7, 0x31, 0x73, 0xd3, 0xa4, 0xab, 0x5e, 0x3a, 0xe9, 0xe4,
- 0x6b, 0x49, 0x26, 0xbf, 0xab, 0xc3, 0x4a, 0x7a, 0x95, 0x7c, 0xc1, 0xeb, 0x62, 0x5e, 0xbf, 0xd3,
- 0xe1, 0x95, 0x02, 0xaf, 0x3d, 0x16, 0xf5, 0x19, 0x72, 0xfe, 0x82, 0xdb, 0x3c, 0x6e, 0xaf, 0xff,
+ // 1867 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x6f, 0x24, 0x47,
+ 0x15, 0x4f, 0x77, 0xcf, 0x87, 0xe7, 0xcd, 0x78, 0xdc, 0x5b, 0xb6, 0x77, 0x67, 0xb3, 0x21, 0xeb,
+ 0xed, 0x45, 0xc4, 0x04, 0x61, 0x83, 0x97, 0x03, 0xb9, 0x81, 0x3d, 0x0a, 0x19, 0x16, 0xaf, 0x4c,
+ 0xdb, 0x0a, 0x12, 0x8a, 0xd4, 0x6a, 0xcf, 0x3c, 0xcf, 0x94, 0xfa, 0x6b, 0x5c, 0xd5, 0xb3, 0xf6,
+ 0xec, 0x81, 0x0b, 0x7b, 0x46, 0x7c, 0x88, 0x03, 0x37, 0xee, 0x90, 0x88, 0x80, 0xf8, 0x1f, 0x08,
+ 0x1f, 0x42, 0xe2, 0xbf, 0x80, 0x03, 0x48, 0x24, 0x1c, 0xb8, 0xa1, 0xaa, 0xea, 0x8f, 0x69, 0x7b,
+ 0x66, 0x6c, 0xc5, 0xde, 0xb0, 0x51, 0xf6, 0xd6, 0xf5, 0xba, 0xba, 0xea, 0xbd, 0xdf, 0x7b, 0xef,
+ 0x57, 0xf5, 0x5e, 0x03, 0xa1, 0x61, 0x8c, 0x2c, 0x74, 0x7d, 0x27, 0xe0, 0xfd, 0x8d, 0x21, 0x8b,
+ 0xe2, 0x88, 0xac, 0x06, 0xd4, 0x7f, 0x3c, 0xe2, 0x6a, 0xb4, 0x91, 0x4e, 0x78, 0xb9, 0xd1, 0x8d,
+ 0x82, 0x20, 0x0a, 0x95, 0xf8, 0xe5, 0x1b, 0x1c, 0xd9, 0x63, 0xda, 0xc5, 0xfc, 0x3b, 0x2b, 0x84,
+ 0x5a, 0xa7, 0x6d, 0xe3, 0xf1, 0x08, 0x79, 0x4c, 0x6e, 0x42, 0x65, 0x88, 0xc8, 0x3a, 0xed, 0x96,
+ 0xb6, 0xa6, 0xad, 0x1b, 0x76, 0x32, 0x22, 0x0f, 0xa0, 0xc4, 0x22, 0x1f, 0x5b, 0xfa, 0x9a, 0xb6,
+ 0xde, 0xdc, 0xba, 0xbb, 0x31, 0x75, 0xaf, 0x8d, 0x3d, 0x44, 0x66, 0x47, 0x3e, 0xda, 0x72, 0x32,
+ 0x59, 0x81, 0x72, 0x37, 0x1a, 0x85, 0x71, 0xcb, 0x58, 0xd3, 0xd6, 0x17, 0x6d, 0x35, 0xb0, 0xfa,
+ 0x00, 0x62, 0x3f, 0x3e, 0x8c, 0x42, 0x8e, 0xe4, 0x01, 0x54, 0x78, 0xec, 0xc6, 0x23, 0x2e, 0x37,
+ 0xac, 0x6f, 0xdd, 0x29, 0x2e, 0x9d, 0x28, 0xbf, 0x2f, 0xa7, 0xd8, 0xc9, 0x54, 0xd2, 0x04, 0xbd,
+ 0xd3, 0x96, 0xba, 0x18, 0xb6, 0xde, 0x69, 0xcf, 0xd8, 0x28, 0x02, 0x38, 0xe0, 0xd1, 0x27, 0x68,
+ 0xd9, 0x63, 0xa8, 0xcb, 0x0d, 0xaf, 0x62, 0xda, 0x2b, 0x50, 0x8b, 0x69, 0x80, 0x3c, 0x76, 0x83,
+ 0xa1, 0xd4, 0xa9, 0x64, 0xe7, 0x82, 0x19, 0xfb, 0x3e, 0xd5, 0xa0, 0xb1, 0x8f, 0xfd, 0xdc, 0x8b,
+ 0xd9, 0x34, 0x6d, 0x62, 0x9a, 0x58, 0xba, 0x3b, 0x70, 0xc3, 0x10, 0xfd, 0x04, 0xbc, 0xb2, 0x9d,
+ 0x0b, 0xc8, 0x1d, 0xa8, 0x75, 0x23, 0xdf, 0x77, 0x42, 0x37, 0x40, 0xb9, 0x7c, 0xcd, 0x5e, 0x10,
+ 0x82, 0x47, 0x6e, 0x80, 0xe4, 0x3e, 0x2c, 0x0e, 0x5d, 0x16, 0xd3, 0x98, 0x46, 0xa1, 0x13, 0xbb,
+ 0xfd, 0x56, 0x49, 0x4e, 0x68, 0x64, 0xc2, 0x03, 0xb7, 0x6f, 0xbd, 0xa7, 0x01, 0xf9, 0x26, 0xe7,
+ 0xb4, 0x1f, 0x16, 0x94, 0xb9, 0x56, 0xe0, 0x1f, 0xc2, 0xd2, 0x10, 0x99, 0x93, 0xa8, 0xed, 0x30,
+ 0x3c, 0x6e, 0x19, 0x6b, 0xc6, 0x7a, 0x7d, 0xeb, 0xfe, 0x8c, 0xef, 0x27, 0x55, 0xb1, 0x17, 0x87,
+ 0xc8, 0x76, 0xd4, 0xa7, 0x36, 0x1e, 0x5b, 0x1f, 0x6a, 0xb0, 0x24, 0xdf, 0x2b, 0xad, 0x03, 0x0c,
+ 0x25, 0x74, 0x5c, 0x88, 0x12, 0x65, 0xd5, 0xe0, 0x02, 0xe8, 0xa6, 0x7a, 0xa5, 0x08, 0x68, 0xe9,
+ 0x22, 0x40, 0xcb, 0xe7, 0x01, 0x25, 0x77, 0xa1, 0x8e, 0xa7, 0x43, 0xca, 0xd0, 0x11, 0x11, 0xd0,
+ 0xaa, 0xc8, 0x68, 0x00, 0x25, 0x3a, 0xa0, 0xc1, 0x64, 0x84, 0x55, 0x2f, 0x1d, 0x61, 0x16, 0x87,
+ 0xe5, 0x82, 0x97, 0x92, 0x68, 0x7d, 0x07, 0x6e, 0x4e, 0x22, 0xeb, 0x66, 0x90, 0xb4, 0x34, 0x09,
+ 0xf0, 0x17, 0xe6, 0x01, 0x9c, 0x03, 0x68, 0xaf, 0xe4, 0x18, 0xe7, 0x52, 0xeb, 0xbf, 0x1a, 0xdc,
+ 0xda, 0x61, 0xe8, 0xc6, 0xb8, 0x13, 0xf9, 0x3e, 0x76, 0x85, 0x89, 0x69, 0x80, 0xbc, 0x01, 0x0b,
+ 0x01, 0xef, 0x3b, 0xf1, 0x78, 0x88, 0x12, 0xf5, 0xe6, 0xd6, 0xab, 0x33, 0xf6, 0xda, 0xe5, 0xfd,
+ 0x83, 0xf1, 0x10, 0xed, 0x6a, 0xa0, 0x1e, 0x88, 0x05, 0x8d, 0x6e, 0xb6, 0x5e, 0x46, 0x09, 0x05,
+ 0x99, 0xf0, 0x0e, 0xc3, 0xe3, 0x4e, 0x5b, 0x7a, 0xc7, 0xb0, 0xd5, 0xa0, 0x98, 0x67, 0xa5, 0xb3,
+ 0x79, 0xd6, 0x82, 0xea, 0x90, 0x45, 0xa7, 0xe3, 0x4e, 0x5b, 0x3a, 0xc6, 0xb0, 0xd3, 0x21, 0xf9,
+ 0x2a, 0x54, 0x78, 0x77, 0x80, 0x81, 0x2b, 0xdd, 0x51, 0xdf, 0xba, 0x3d, 0x15, 0xf2, 0x6d, 0x3f,
+ 0x3a, 0xb4, 0x93, 0x89, 0xd6, 0xcf, 0x74, 0x58, 0x6d, 0xb3, 0x68, 0xf8, 0x29, 0xb7, 0x7c, 0x17,
+ 0x96, 0xf2, 0xd5, 0x55, 0x54, 0x2b, 0x08, 0x3e, 0x5f, 0xd4, 0x39, 0x39, 0x61, 0x36, 0x72, 0x73,
+ 0x45, 0xc4, 0xdb, 0xcd, 0x6e, 0x61, 0x6c, 0xfd, 0x53, 0x83, 0x95, 0xb7, 0x5c, 0x7e, 0xad, 0xa0,
+ 0x64, 0x06, 0xeb, 0x33, 0x0d, 0x36, 0xe6, 0x18, 0x5c, 0xba, 0xd0, 0xe0, 0xf2, 0x15, 0x0c, 0xfe,
+ 0x50, 0x83, 0xdb, 0x6d, 0xe4, 0x5d, 0x46, 0x0f, 0xf1, 0xb3, 0x63, 0xf5, 0x2f, 0x35, 0x58, 0xdd,
+ 0x1f, 0x44, 0x27, 0xcf, 0xaf, 0xc5, 0xd6, 0xef, 0x74, 0xb8, 0xa9, 0xb8, 0x69, 0x2f, 0x65, 0xdf,
+ 0x4f, 0x28, 0x41, 0xd7, 0xa0, 0x9e, 0x11, 0x7e, 0x96, 0xa6, 0x93, 0xa2, 0xdc, 0xd2, 0xd2, 0x4c,
+ 0x4b, 0xcb, 0x73, 0x2c, 0xad, 0x14, 0x7d, 0xfb, 0x6d, 0x68, 0xe6, 0xa7, 0x8e, 0x74, 0xad, 0x3a,
+ 0x37, 0xee, 0x4f, 0x77, 0x6d, 0x06, 0x87, 0xf4, 0x6c, 0x7e, 0x60, 0x49, 0xc7, 0xbe, 0xaf, 0xc3,
+ 0x8a, 0x60, 0xb5, 0x17, 0x98, 0x5d, 0x1e, 0xb3, 0x7f, 0x68, 0xb0, 0xfc, 0x96, 0xcb, 0xaf, 0x13,
+ 0xb2, 0xeb, 0x4d, 0xfe, 0xf3, 0xc6, 0x96, 0x3f, 0xb6, 0xb1, 0xff, 0xd2, 0xa0, 0x95, 0xf2, 0xdd,
+ 0x67, 0xc3, 0x62, 0x71, 0xa4, 0x09, 0xae, 0x7b, 0x7e, 0xad, 0xbd, 0x66, 0x72, 0xff, 0xb7, 0x0e,
+ 0x8b, 0x9d, 0x90, 0x23, 0x8b, 0x9f, 0x99, 0xa5, 0xaf, 0x9d, 0xd7, 0x58, 0x15, 0x27, 0x67, 0x74,
+ 0xb9, 0x54, 0x89, 0x22, 0x70, 0xe3, 0xd8, 0x17, 0x37, 0xd2, 0xec, 0x7e, 0x93, 0x0b, 0x8a, 0xb7,
+ 0x7c, 0x45, 0x03, 0x13, 0xb7, 0xfc, 0x09, 0x54, 0xab, 0x45, 0x54, 0x5f, 0x05, 0xc8, 0xc0, 0xe7,
+ 0xad, 0x85, 0x35, 0x43, 0x5c, 0xd3, 0x73, 0x89, 0xa8, 0x80, 0x58, 0x74, 0xd2, 0x69, 0xf3, 0x56,
+ 0x6d, 0xcd, 0x10, 0x15, 0x90, 0x1a, 0x91, 0xaf, 0xc1, 0x02, 0x8b, 0x4e, 0x9c, 0x9e, 0x1b, 0xbb,
+ 0x2d, 0x90, 0x97, 0xec, 0x39, 0xb7, 0xc9, 0x2a, 0x8b, 0x4e, 0xda, 0x6e, 0xec, 0x5a, 0x4f, 0x75,
+ 0x58, 0x6c, 0xa3, 0x8f, 0x31, 0xfe, 0xff, 0x41, 0x2f, 0x20, 0x56, 0x9a, 0x83, 0x58, 0x79, 0x1e,
+ 0x62, 0x95, 0x73, 0x88, 0xdd, 0x83, 0xc6, 0x90, 0xd1, 0xc0, 0x65, 0x63, 0xc7, 0xc3, 0xb1, 0x28,
+ 0x6f, 0x0c, 0xc9, 0xf2, 0x4a, 0xf6, 0x10, 0xc7, 0xdc, 0xfa, 0x48, 0x83, 0xc5, 0x7d, 0x74, 0x59,
+ 0x77, 0xf0, 0xcc, 0x60, 0x98, 0xd0, 0xdf, 0x28, 0xea, 0x3f, 0xff, 0x0e, 0xfd, 0x45, 0x30, 0x19,
+ 0xf2, 0x91, 0x1f, 0x3b, 0x39, 0x38, 0x0a, 0x80, 0x25, 0x25, 0xdf, 0xc9, 0x20, 0xda, 0x84, 0xf2,
+ 0xf1, 0x08, 0xd9, 0xf8, 0xe2, 0x6a, 0x42, 0xcd, 0xb3, 0xfe, 0xa6, 0x81, 0xb9, 0x3f, 0xe6, 0x3b,
+ 0x51, 0x78, 0x44, 0xfb, 0xcf, 0x9d, 0xe5, 0x04, 0x4a, 0xd2, 0x5f, 0xe5, 0x35, 0x63, 0xbd, 0x66,
+ 0xcb, 0x67, 0xe1, 0x4b, 0x0f, 0xc7, 0xce, 0x90, 0xe1, 0x11, 0x3d, 0x45, 0xe5, 0xed, 0x9a, 0x5d,
+ 0xf7, 0x70, 0xbc, 0x97, 0x88, 0xac, 0xbf, 0xea, 0xd0, 0x48, 0x7d, 0x29, 0xf0, 0xb9, 0x8a, 0x41,
+ 0x79, 0x4d, 0xac, 0x5f, 0xbe, 0xeb, 0x32, 0xbd, 0x52, 0x9a, 0xcd, 0xa3, 0xf7, 0xa0, 0x21, 0xdd,
+ 0xe1, 0x84, 0x51, 0x0f, 0x33, 0xef, 0xd6, 0xa5, 0xec, 0x91, 0x14, 0x15, 0x81, 0xaa, 0x5c, 0x26,
+ 0x44, 0xaa, 0xd3, 0x43, 0x84, 0x40, 0x69, 0x40, 0x63, 0xc5, 0x2b, 0x0d, 0x5b, 0x3e, 0x93, 0xbb,
+ 0x50, 0x0f, 0x30, 0x66, 0xb4, 0xab, 0x20, 0xaa, 0xc9, 0xe4, 0x04, 0x25, 0x12, 0x28, 0x58, 0x3f,
+ 0x80, 0xfa, 0x01, 0x0d, 0xf0, 0x80, 0x76, 0xbd, 0x5d, 0xde, 0xbf, 0x0a, 0x9e, 0x79, 0xfb, 0x46,
+ 0x2f, 0xb4, 0x6f, 0xe6, 0x1e, 0x41, 0xd6, 0x07, 0x1a, 0x2c, 0xbc, 0xe9, 0x8f, 0xf8, 0xe0, 0x8a,
+ 0xbb, 0x17, 0x08, 0x5b, 0x9f, 0x42, 0xd8, 0x73, 0x8e, 0xc1, 0xb3, 0xb7, 0xc9, 0xd2, 0x94, 0xdb,
+ 0xa4, 0x05, 0x85, 0x03, 0x62, 0x5a, 0x1b, 0xc6, 0xfa, 0x85, 0x06, 0xd5, 0x87, 0x38, 0xde, 0xda,
+ 0xc7, 0xbe, 0x0c, 0x14, 0x41, 0xde, 0x69, 0x7b, 0x48, 0x0e, 0x84, 0x3b, 0x26, 0xe8, 0x2a, 0xd1,
+ 0x13, 0x72, 0xb6, 0xba, 0x40, 0xd1, 0xdb, 0xb0, 0x40, 0xb9, 0xf3, 0xd8, 0xf5, 0x69, 0x4f, 0x2a,
+ 0xb9, 0x60, 0x57, 0x29, 0x7f, 0x5b, 0x0c, 0x05, 0x51, 0x66, 0xe6, 0xaa, 0xb4, 0x32, 0xec, 0x09,
+ 0x89, 0xf5, 0x0e, 0x40, 0xa2, 0x9a, 0x00, 0x3a, 0x0b, 0x63, 0x6d, 0x32, 0x8c, 0xbf, 0x0e, 0x55,
+ 0x0f, 0xc7, 0x5b, 0x1c, 0xfb, 0x2d, 0x5d, 0x9e, 0x32, 0xb3, 0xd0, 0x4f, 0x56, 0xb2, 0xd3, 0xe9,
+ 0xd6, 0x0f, 0x75, 0xa8, 0x7d, 0x27, 0x72, 0x7b, 0x9d, 0xb0, 0x87, 0xa7, 0xcf, 0xd4, 0x8d, 0x47,
+ 0x14, 0xfd, 0xde, 0xa3, 0xfc, 0xa0, 0xc9, 0x05, 0x22, 0x0b, 0xe5, 0x20, 0xcf, 0xc2, 0x64, 0x28,
+ 0x60, 0xa7, 0x42, 0x33, 0x67, 0xe8, 0xc6, 0x83, 0x94, 0x74, 0x40, 0x8a, 0xf6, 0x84, 0x84, 0xb4,
+ 0xa1, 0x91, 0x4e, 0x60, 0x6e, 0xa0, 0xa8, 0xa7, 0xbe, 0x75, 0x6f, 0x2a, 0x23, 0x3c, 0xc4, 0xf1,
+ 0xdb, 0xae, 0x3f, 0xc2, 0x3d, 0x97, 0x32, 0xbb, 0x9e, 0x2c, 0x22, 0xbe, 0xb2, 0x9e, 0x6a, 0x00,
+ 0x12, 0x01, 0x41, 0x1a, 0xe7, 0x17, 0xd5, 0x3e, 0xce, 0xa2, 0xe4, 0x2b, 0xb0, 0x12, 0x8e, 0x02,
+ 0x87, 0xa1, 0xef, 0xc6, 0xd8, 0x73, 0x12, 0x30, 0x78, 0x02, 0x0e, 0x09, 0x47, 0x81, 0xad, 0x5e,
+ 0xed, 0x27, 0x6f, 0xac, 0x1f, 0x69, 0x00, 0x6f, 0x0a, 0xcb, 0x95, 0x1a, 0x67, 0xa3, 0x5b, 0x9b,
+ 0x12, 0xdd, 0x13, 0xd0, 0xe9, 0x45, 0xe8, 0xb6, 0x53, 0xe8, 0x04, 0x01, 0xf2, 0xa4, 0x87, 0x7a,
+ 0x6f, 0x86, 0x3b, 0x73, 0xe3, 0x13, 0x74, 0xe5, 0xb3, 0xf5, 0x73, 0xd5, 0x76, 0x16, 0xda, 0x29,
+ 0x95, 0x0a, 0x5e, 0xd6, 0xce, 0x7a, 0x59, 0x72, 0x56, 0x10, 0xb1, 0xb1, 0xc3, 0xe9, 0x13, 0x4c,
+ 0x93, 0x44, 0x89, 0xf6, 0xe9, 0x13, 0x14, 0x69, 0x20, 0x21, 0x89, 0x4e, 0x78, 0x7a, 0xea, 0x08,
+ 0x18, 0xa2, 0x13, 0x4e, 0xbe, 0x04, 0x37, 0x18, 0x76, 0x31, 0x8c, 0xfd, 0xb1, 0x13, 0x44, 0x3d,
+ 0x7a, 0x44, 0x31, 0x4d, 0x15, 0x33, 0x7d, 0xb1, 0x9b, 0xc8, 0xad, 0xbf, 0x6b, 0xd0, 0xfc, 0x6e,
+ 0xca, 0xc4, 0x4a, 0xb3, 0x67, 0xc0, 0x7f, 0xdf, 0x90, 0xc6, 0x16, 0xf0, 0x9b, 0xd3, 0x83, 0xce,
+ 0x40, 0xb2, 0x17, 0x38, 0xf6, 0x95, 0x52, 0xdb, 0x50, 0x97, 0xee, 0x48, 0xd6, 0x28, 0xcd, 0xf5,
+ 0x41, 0xee, 0x79, 0x1b, 0x8e, 0xb2, 0x67, 0xeb, 0x57, 0x3a, 0x10, 0xd5, 0xbb, 0x90, 0x4e, 0x7a,
+ 0xee, 0x0a, 0x8e, 0xd7, 0xa6, 0x17, 0x1c, 0xe7, 0x6f, 0x92, 0x9f, 0x03, 0x65, 0x56, 0xde, 0x58,
+ 0x2c, 0x90, 0x40, 0x1b, 0x1a, 0x78, 0x1a, 0x33, 0x37, 0x4d, 0xba, 0xea, 0xa5, 0x93, 0x4e, 0x7e,
+ 0x96, 0x64, 0xf2, 0xbb, 0x3a, 0xac, 0xa4, 0x25, 0xe9, 0x0b, 0xbc, 0x2e, 0xc6, 0xeb, 0xf7, 0x3a,
+ 0xbc, 0x52, 0xc0, 0x6b, 0x8f, 0x45, 0x7d, 0x86, 0x9c, 0xbf, 0xc0, 0x6d, 0x1e, 0x6e, 0xaf, 0xff,
0xc5, 0x80, 0x6a, 0x62, 0x30, 0xa9, 0x41, 0xd9, 0x7b, 0x14, 0x85, 0x68, 0xbe, 0x44, 0x56, 0xe1,
- 0x86, 0x77, 0xf6, 0xcf, 0x0b, 0xb3, 0x47, 0x96, 0x61, 0xc9, 0x2b, 0xf6, 0xf5, 0x4d, 0x24, 0x04,
- 0x9a, 0x5e, 0xa1, 0xad, 0x6d, 0x1e, 0x91, 0x5b, 0xb0, 0xec, 0x9d, 0xef, 0xfc, 0x9a, 0x62, 0xdf,
- 0x37, 0xbd, 0x62, 0x73, 0x94, 0x9b, 0x03, 0xb9, 0xc4, 0x37, 0x30, 0xce, 0x4e, 0xf9, 0xdc, 0xa4,
- 0x64, 0x15, 0x4c, 0xef, 0x4c, 0x8f, 0xd2, 0xfc, 0xbd, 0x46, 0x96, 0xa1, 0xe9, 0x15, 0x9a, 0x70,
- 0xe6, 0x1f, 0x34, 0x42, 0x60, 0xd1, 0x9b, 0xec, 0x32, 0x99, 0x7f, 0xd4, 0xc8, 0x2d, 0x20, 0xde,
- 0xb9, 0x66, 0x8c, 0xf9, 0x27, 0x8d, 0xac, 0xc0, 0x92, 0x57, 0xe8, 0x59, 0x70, 0xf3, 0xcf, 0x1a,
- 0xb9, 0x01, 0x0d, 0x6f, 0xa2, 0x3c, 0x99, 0xbf, 0xd2, 0xd5, 0xa7, 0x26, 0x63, 0xca, 0x7c, 0x57,
- 0x27, 0x77, 0xe0, 0xa6, 0x37, 0x35, 0xd0, 0xcc, 0xf7, 0x74, 0xd2, 0x80, 0xaa, 0xa7, 0xba, 0x03,
- 0xe6, 0x8f, 0x0c, 0x39, 0x52, 0xd7, 0x56, 0xf3, 0xc7, 0x06, 0xa9, 0x43, 0xc5, 0x93, 0xe7, 0x43,
- 0xf3, 0x27, 0xea, 0x27, 0x75, 0xfc, 0x37, 0x3f, 0x34, 0xa4, 0xfa, 0x93, 0x97, 0x01, 0xf3, 0x3f,
- 0x06, 0x69, 0x42, 0xcd, 0x4b, 0xcf, 0xb3, 0xe6, 0xaf, 0x6b, 0x52, 0xeb, 0xe2, 0x56, 0x61, 0xbe,
- 0x5f, 0x23, 0x4b, 0x00, 0x5e, 0x76, 0xec, 0x31, 0x7f, 0x53, 0x7b, 0xfd, 0x0d, 0x58, 0x48, 0xff,
- 0x3f, 0x24, 0x00, 0x95, 0x5d, 0x97, 0xc7, 0xc8, 0xcc, 0x97, 0xc4, 0xb3, 0x8d, 0x6e, 0x0f, 0x99,
- 0xa9, 0x89, 0xe7, 0xef, 0x30, 0x2a, 0xe4, 0xba, 0xf0, 0xf9, 0x9e, 0x08, 0x4c, 0xd3, 0xd8, 0x6e,
- 0x7f, 0x77, 0xbb, 0x4f, 0xe3, 0xc1, 0xe8, 0x50, 0x44, 0xcd, 0xe6, 0x13, 0xea, 0xfb, 0xf4, 0x49,
- 0x8c, 0xdd, 0xc1, 0xa6, 0x8a, 0xa8, 0x2f, 0xf6, 0x28, 0x8f, 0x19, 0x3d, 0x1c, 0xc5, 0xd8, 0xdb,
- 0x4c, 0x93, 0x65, 0x53, 0x86, 0x59, 0x36, 0x1c, 0x1e, 0x1e, 0x56, 0xa4, 0xe4, 0xc1, 0xff, 0x02,
- 0x00, 0x00, 0xff, 0xff, 0xc1, 0x89, 0xef, 0x28, 0xb0, 0x1f, 0x00, 0x00,
+ 0x86, 0x77, 0xf6, 0x27, 0x88, 0xd9, 0x23, 0xcb, 0xb0, 0xe4, 0x15, 0xff, 0x0f, 0x98, 0x48, 0x08,
+ 0x34, 0xbd, 0x42, 0x7b, 0xdc, 0x3c, 0x22, 0xb7, 0x60, 0xd9, 0x3b, 0xdf, 0x41, 0x36, 0xc5, 0xb9,
+ 0x6f, 0x7a, 0xc5, 0x26, 0x2b, 0x37, 0x07, 0x72, 0x89, 0x6f, 0x61, 0x9c, 0x55, 0x0b, 0xdc, 0xa4,
+ 0x64, 0x15, 0x4c, 0xef, 0x4c, 0xaf, 0xd3, 0xfc, 0x83, 0x46, 0x96, 0xa1, 0xe9, 0x15, 0x9a, 0x79,
+ 0xe6, 0x07, 0x1a, 0x21, 0xb0, 0xe8, 0x4d, 0x76, 0xab, 0xcc, 0x3f, 0x6a, 0xe4, 0x16, 0x10, 0xef,
+ 0x5c, 0x53, 0xc7, 0xfc, 0x93, 0x46, 0x56, 0x60, 0xc9, 0x2b, 0xf4, 0x3e, 0xb8, 0xf9, 0x67, 0x8d,
+ 0xdc, 0x80, 0x86, 0x37, 0x41, 0x4f, 0xe6, 0xaf, 0x75, 0xb5, 0xd5, 0x64, 0x4c, 0x99, 0xef, 0xea,
+ 0xe4, 0x0e, 0xdc, 0xf4, 0xa6, 0x06, 0x9a, 0xf9, 0x9e, 0x4e, 0x1a, 0x50, 0xf5, 0x54, 0x97, 0xc1,
+ 0xfc, 0xb1, 0x21, 0x47, 0xaa, 0xfc, 0x35, 0x7f, 0x62, 0x90, 0x3a, 0x54, 0x3c, 0x79, 0xcf, 0x34,
+ 0x7f, 0xaa, 0x5e, 0xa9, 0x32, 0xc2, 0xfc, 0xc8, 0x90, 0xea, 0x4f, 0x16, 0x15, 0xe6, 0x7f, 0x0c,
+ 0xd2, 0x84, 0x9a, 0x97, 0xde, 0x8b, 0xcd, 0xdf, 0xd4, 0xa4, 0xd6, 0xc5, 0xa3, 0xc2, 0x7c, 0xbf,
+ 0x46, 0x96, 0x00, 0xbc, 0xec, 0xda, 0x63, 0xfe, 0xb6, 0xf6, 0xfa, 0x1b, 0xb0, 0x90, 0xfe, 0x87,
+ 0x24, 0x00, 0x95, 0x5d, 0x97, 0xc7, 0xc8, 0xcc, 0x97, 0xc4, 0xb3, 0x8d, 0x6e, 0x0f, 0x99, 0xa9,
+ 0x89, 0xe7, 0xef, 0x31, 0x2a, 0xe4, 0xba, 0xf0, 0xf9, 0x9e, 0x08, 0x4c, 0xd3, 0xd8, 0x6e, 0x7f,
+ 0x7f, 0xbb, 0x4f, 0xe3, 0xc1, 0xe8, 0x50, 0x44, 0xcd, 0xe6, 0x13, 0xea, 0xfb, 0xf4, 0x49, 0x8c,
+ 0xdd, 0xc1, 0xa6, 0x8a, 0xa8, 0x2f, 0xf7, 0x28, 0x8f, 0x19, 0x3d, 0x1c, 0xc5, 0xd8, 0xdb, 0x4c,
+ 0x93, 0x65, 0x53, 0x86, 0x59, 0x36, 0x1c, 0x1e, 0x1e, 0x56, 0xa4, 0xe4, 0xc1, 0xff, 0x02, 0x00,
+ 0x00, 0xff, 0xff, 0xc4, 0x5d, 0xb1, 0xba, 0xf8, 0x1f, 0x00, 0x00,
}
diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go
index f4232bc82234f07d54d713772e89d70f77b5586f..6a312c2bc04169e53915820fd400c177c7f48811 100644
--- a/internal/proxy/proxy.go
+++ b/internal/proxy/proxy.go
@@ -2,6 +2,8 @@ package proxy
import (
"context"
+ "fmt"
+ "io"
"log"
"math/rand"
"net"
@@ -9,6 +11,10 @@ import (
"sync"
"time"
+ "github.com/opentracing/opentracing-go"
+ "github.com/uber/jaeger-client-go"
+ "github.com/uber/jaeger-client-go/config"
+
"google.golang.org/grpc"
"github.com/zilliztech/milvus-distributed/internal/allocator"
@@ -39,6 +45,9 @@ type Proxy struct {
manipulationMsgStream *msgstream.PulsarMsgStream
queryMsgStream *msgstream.PulsarMsgStream
+ tracer opentracing.Tracer
+ closer io.Closer
+
// Add callback functions at different stages
startCallbacks []func()
closeCallbacks []func()
@@ -51,11 +60,28 @@ func Init() {
func CreateProxy(ctx context.Context) (*Proxy, error) {
rand.Seed(time.Now().UnixNano())
ctx1, cancel := context.WithCancel(ctx)
+ var err error
p := &Proxy{
proxyLoopCtx: ctx1,
proxyLoopCancel: cancel,
}
+ cfg := &config.Configuration{
+ ServiceName: "tracing",
+ Sampler: &config.SamplerConfig{
+ Type: "const",
+ Param: 1,
+ },
+ Reporter: &config.ReporterConfig{
+ LogSpans: true,
+ },
+ }
+ p.tracer, p.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
+ if err != nil {
+ panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
+ }
+ opentracing.SetGlobalTracer(p.tracer)
+
pulsarAddress := Params.PulsarAddress()
p.queryMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, Params.MsgStreamSearchBufSize())
@@ -198,6 +224,8 @@ func (p *Proxy) stopProxyLoop() {
p.tick.Close()
p.proxyLoopWg.Wait()
+
+ p.closer.Close()
}
// Close closes the server.
diff --git a/internal/proxy/repack_func.go b/internal/proxy/repack_func.go
index 44139999e0403719ca9eaf141f110980b808b6e1..f8873fe12f27bcae72473bd9b5eb252a9eac1aee 100644
--- a/internal/proxy/repack_func.go
+++ b/internal/proxy/repack_func.go
@@ -182,6 +182,7 @@ func insertRepackFunc(tsMsgs []msgstream.TsMsg,
insertMsg := &msgstream.InsertMsg{
InsertRequest: sliceRequest,
}
+ insertMsg.SetContext(request.GetContext())
if together { // all rows with same hash value are accumulated to only one message
if len(result[key].Msgs) <= 0 {
result[key].Msgs = append(result[key].Msgs, insertMsg)
diff --git a/internal/proxy/task.go b/internal/proxy/task.go
index 425cae75cfb3e4de24b55cf4a24cf3cc5aa55dbe..d01c45f0632545feca63a477eec4df35de3cf85c 100644
--- a/internal/proxy/task.go
+++ b/internal/proxy/task.go
@@ -7,6 +7,9 @@ import (
"math"
"strconv"
+ "github.com/opentracing/opentracing-go"
+ oplog "github.com/opentracing/opentracing-go/log"
+
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/allocator"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
@@ -74,12 +77,21 @@ func (it *InsertTask) Type() internalpb.MsgType {
}
func (it *InsertTask) PreExecute() error {
+ span := opentracing.StartSpan("InsertTask preExecute")
+ defer span.Finish()
+ it.ctx = opentracing.ContextWithSpan(it.ctx, span)
+ span.SetTag("hash keys", it.ReqID)
+ span.SetTag("start time", it.BeginTs())
collectionName := it.BaseInsertTask.CollectionName
if err := ValidateCollectionName(collectionName); err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
partitionTag := it.BaseInsertTask.PartitionTag
if err := ValidatePartitionTag(partitionTag, true); err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
@@ -87,22 +99,36 @@ func (it *InsertTask) PreExecute() error {
}
func (it *InsertTask) Execute() error {
+ span, ctx := opentracing.StartSpanFromContext(it.ctx, "InsertTask Execute")
+ defer span.Finish()
+ it.ctx = ctx
+ span.SetTag("hash keys", it.ReqID)
+ span.SetTag("start time", it.BeginTs())
collectionName := it.BaseInsertTask.CollectionName
+ span.LogFields(oplog.String("collection_name", collectionName))
if !globalMetaCache.Hit(collectionName) {
err := globalMetaCache.Sync(collectionName)
if err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
}
description, err := globalMetaCache.Get(collectionName)
if err != nil || description == nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
autoID := description.Schema.AutoID
+ span.LogFields(oplog.Bool("auto_id", autoID))
var rowIDBegin UniqueID
var rowIDEnd UniqueID
rowNums := len(it.BaseInsertTask.RowData)
rowIDBegin, rowIDEnd, _ = it.rowIDAllocator.Alloc(uint32(rowNums))
+ span.LogFields(oplog.Int("rowNums", rowNums),
+ oplog.Int("rowIDBegin", int(rowIDBegin)),
+ oplog.Int("rowIDEnd", int(rowIDEnd)))
it.BaseInsertTask.RowIDs = make([]UniqueID, rowNums)
for i := rowIDBegin; i < rowIDEnd; i++ {
offset := i - rowIDBegin
@@ -125,6 +151,8 @@ func (it *InsertTask) Execute() error {
EndTs: it.EndTs(),
Msgs: make([]msgstream.TsMsg, 1),
}
+ tsMsg.SetContext(it.ctx)
+ span.LogFields(oplog.String("send msg", "send msg"))
msgPack.Msgs[0] = tsMsg
err = it.manipulationMsgStream.Produce(msgPack)
@@ -138,11 +166,14 @@ func (it *InsertTask) Execute() error {
if err != nil {
it.result.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
it.result.Status.Reason = err.Error()
+ span.LogFields(oplog.Error(err))
}
return nil
}
func (it *InsertTask) PostExecute() error {
+ span, _ := opentracing.StartSpanFromContext(it.ctx, "InsertTask postExecute")
+ defer span.Finish()
return nil
}
@@ -352,24 +383,38 @@ func (qt *QueryTask) SetTs(ts Timestamp) {
}
func (qt *QueryTask) PreExecute() error {
+ span := opentracing.StartSpan("InsertTask preExecute")
+ defer span.Finish()
+ qt.ctx = opentracing.ContextWithSpan(qt.ctx, span)
+ span.SetTag("hash keys", qt.ReqID)
+ span.SetTag("start time", qt.BeginTs())
+
collectionName := qt.query.CollectionName
if !globalMetaCache.Hit(collectionName) {
err := globalMetaCache.Sync(collectionName)
if err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
}
_, err := globalMetaCache.Get(collectionName)
if err != nil { // err is not nil if collection not exists
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
if err := ValidateCollectionName(qt.query.CollectionName); err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
for _, tag := range qt.query.PartitionTags {
if err := ValidatePartitionTag(tag, false); err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
}
@@ -379,6 +424,8 @@ func (qt *QueryTask) PreExecute() error {
}
queryBytes, err := proto.Marshal(qt.query)
if err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
qt.Query = &commonpb.Blob{
@@ -388,6 +435,10 @@ func (qt *QueryTask) PreExecute() error {
}
func (qt *QueryTask) Execute() error {
+ span, ctx := opentracing.StartSpanFromContext(qt.ctx, "InsertTask Execute")
+ defer span.Finish()
+ span.SetTag("hash keys", qt.ReqID)
+ span.SetTag("start time", qt.BeginTs())
var tsMsg msgstream.TsMsg = &msgstream.SearchMsg{
SearchRequest: qt.SearchRequest,
BaseMsg: msgstream.BaseMsg{
@@ -401,20 +452,28 @@ func (qt *QueryTask) Execute() error {
EndTs: qt.Timestamp,
Msgs: make([]msgstream.TsMsg, 1),
}
+ tsMsg.SetContext(ctx)
msgPack.Msgs[0] = tsMsg
err := qt.queryMsgStream.Produce(msgPack)
log.Printf("[Proxy] length of searchMsg: %v", len(msgPack.Msgs))
if err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
log.Printf("[Proxy] send search request failed: %v", err)
}
return err
}
func (qt *QueryTask) PostExecute() error {
+ span, _ := opentracing.StartSpanFromContext(qt.ctx, "InsertTask postExecute")
+ span.SetTag("hash keys", qt.ReqID)
+ span.SetTag("start time", qt.BeginTs())
for {
select {
case <-qt.ctx.Done():
log.Print("wait to finish failed, timeout!")
+ span.LogFields(oplog.String("wait to finish failed, timeout", "wait to finish failed, timeout"))
+ span.Finish()
return errors.New("wait to finish failed, timeout")
case searchResults := <-qt.resultBuf:
filterSearchResult := make([]*internalpb.SearchResult, 0)
@@ -435,6 +494,8 @@ func (qt *QueryTask) PostExecute() error {
Reason: filterReason,
},
}
+ span.LogFields(oplog.Error(errors.New(filterReason)))
+ span.Finish()
return errors.New(filterReason)
}
@@ -465,6 +526,7 @@ func (qt *QueryTask) PostExecute() error {
Reason: filterReason,
},
}
+ span.Finish()
return nil
}
@@ -476,6 +538,7 @@ func (qt *QueryTask) PostExecute() error {
Reason: filterReason,
},
}
+ span.Finish()
return nil
}
@@ -526,10 +589,13 @@ func (qt *QueryTask) PostExecute() error {
reducedHitsBs, err := proto.Marshal(reducedHits)
if err != nil {
log.Println("marshal error")
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
qt.result.Hits = append(qt.result.Hits, reducedHitsBs)
}
+ span.Finish()
return nil
}
}
@@ -637,7 +703,10 @@ func (dct *DescribeCollectionTask) PreExecute() error {
func (dct *DescribeCollectionTask) Execute() error {
var err error
dct.result, err = dct.masterClient.DescribeCollection(dct.ctx, &dct.DescribeCollectionRequest)
- globalMetaCache.Update(dct.CollectionName.CollectionName, dct.result)
+ if err != nil {
+ return err
+ }
+ err = globalMetaCache.Update(dct.CollectionName.CollectionName, dct.result)
return err
}
diff --git a/internal/querynode/data_sync_service.go b/internal/querynode/data_sync_service.go
index 5ed2caa0675b53c4aa9a895a6983cd3f98dc0057..44a595caf5679969e4455b592e70cdb1f9f34012 100644
--- a/internal/querynode/data_sync_service.go
+++ b/internal/querynode/data_sync_service.go
@@ -48,7 +48,6 @@ func (dsService *dataSyncService) initNodes() {
var insertNode Node = newInsertNode(dsService.replica)
var serviceTimeNode Node = newServiceTimeNode(dsService.replica)
- var gcNode Node = newGCNode(dsService.replica)
dsService.fg.AddNode(&dmStreamNode)
dsService.fg.AddNode(&ddStreamNode)
@@ -58,7 +57,6 @@ func (dsService *dataSyncService) initNodes() {
dsService.fg.AddNode(&insertNode)
dsService.fg.AddNode(&serviceTimeNode)
- dsService.fg.AddNode(&gcNode)
// dmStreamNode
var err = dsService.fg.SetEdges(dmStreamNode.Name(),
@@ -108,17 +106,9 @@ func (dsService *dataSyncService) initNodes() {
// serviceTimeNode
err = dsService.fg.SetEdges(serviceTimeNode.Name(),
[]string{insertNode.Name()},
- []string{gcNode.Name()},
+ []string{},
)
if err != nil {
log.Fatal("set edges failed in node:", serviceTimeNode.Name())
}
-
- // gcNode
- err = dsService.fg.SetEdges(gcNode.Name(),
- []string{serviceTimeNode.Name()},
- []string{})
- if err != nil {
- log.Fatal("set edges failed in node:", gcNode.Name())
- }
}
diff --git a/internal/querynode/flow_graph_dd_node.go b/internal/querynode/flow_graph_dd_node.go
index a7a2ac73201ff48439090ad0ef5f58d9c1b38e1d..f4a5e1136946450de3a25f206f64a2b6ef7e0ec1 100644
--- a/internal/querynode/flow_graph_dd_node.go
+++ b/internal/querynode/flow_graph_dd_node.go
@@ -44,11 +44,6 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg {
},
}
ddNode.ddMsg = &ddMsg
- gcRecord := gcRecord{
- collections: make([]UniqueID, 0),
- partitions: make([]partitionWithID, 0),
- }
- ddNode.ddMsg.gcRecord = &gcRecord
// sort tsMessages
tsMessages := msMsg.TsMessages()
@@ -120,11 +115,11 @@ func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
func (ddNode *ddNode) dropCollection(msg *msgstream.DropCollectionMsg) {
collectionID := msg.CollectionID
- //err := ddNode.replica.removeCollection(collectionID)
- //if err != nil {
- // log.Println(err)
- // return
- //}
+ err := ddNode.replica.removeCollection(collectionID)
+ if err != nil {
+ log.Println(err)
+ return
+ }
collectionName := msg.CollectionName.CollectionName
ddNode.ddMsg.collectionRecords[collectionName] = append(ddNode.ddMsg.collectionRecords[collectionName],
@@ -132,8 +127,6 @@ func (ddNode *ddNode) dropCollection(msg *msgstream.DropCollectionMsg) {
createOrDrop: false,
timestamp: msg.Timestamp,
})
-
- ddNode.ddMsg.gcRecord.collections = append(ddNode.ddMsg.gcRecord.collections, collectionID)
}
func (ddNode *ddNode) createPartition(msg *msgstream.CreatePartitionMsg) {
@@ -157,22 +150,17 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
collectionID := msg.CollectionID
partitionTag := msg.PartitionName.Tag
- //err := ddNode.replica.removePartition(collectionID, partitionTag)
- //if err != nil {
- // log.Println(err)
- // return
- //}
+ err := ddNode.replica.removePartition(collectionID, partitionTag)
+ if err != nil {
+ log.Println(err)
+ return
+ }
ddNode.ddMsg.partitionRecords[partitionTag] = append(ddNode.ddMsg.partitionRecords[partitionTag],
metaOperateRecord{
createOrDrop: false,
timestamp: msg.Timestamp,
})
-
- ddNode.ddMsg.gcRecord.partitions = append(ddNode.ddMsg.gcRecord.partitions, partitionWithID{
- partitionTag: partitionTag,
- collectionID: collectionID,
- })
}
func newDDNode(replica collectionReplica) *ddNode {
diff --git a/internal/querynode/flow_graph_filter_dm_node.go b/internal/querynode/flow_graph_filter_dm_node.go
index fbc8eedb5c82b00e868561b6e5971a9af3f78468..ceddaeab0b95a1a1b1880def7689ee3278a20e3a 100644
--- a/internal/querynode/flow_graph_filter_dm_node.go
+++ b/internal/querynode/flow_graph_filter_dm_node.go
@@ -2,7 +2,6 @@ package querynode
import (
"log"
- "math"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
@@ -60,7 +59,6 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
}
}
- iMsg.gcRecord = ddMsg.gcRecord
var res Msg = &iMsg
return []*Msg{&res}
}
@@ -83,35 +81,17 @@ func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg
log.Println("Error, misaligned messages detected")
return nil
}
-
tmpTimestamps := make([]Timestamp, 0)
tmpRowIDs := make([]int64, 0)
tmpRowData := make([]*commonpb.Blob, 0)
-
- // calculate valid time range
- timeBegin := Timestamp(0)
- timeEnd := Timestamp(math.MaxUint64)
- for _, record := range records {
- if record.createOrDrop && timeBegin < record.timestamp {
- timeBegin = record.timestamp
- }
- if !record.createOrDrop && timeEnd > record.timestamp {
- timeEnd = record.timestamp
- }
- }
-
+ targetTimestamp := records[len(records)-1].timestamp
for i, t := range msg.Timestamps {
- if t >= timeBegin && t <= timeEnd {
+ if t >= targetTimestamp {
tmpTimestamps = append(tmpTimestamps, t)
tmpRowIDs = append(tmpRowIDs, msg.RowIDs[i])
tmpRowData = append(tmpRowData, msg.RowData[i])
}
}
-
- if len(tmpRowIDs) <= 0 {
- return nil
- }
-
msg.Timestamps = tmpTimestamps
msg.RowIDs = tmpRowIDs
msg.RowData = tmpRowData
diff --git a/internal/querynode/flow_graph_gc_node.go b/internal/querynode/flow_graph_gc_node.go
deleted file mode 100644
index cd0a9b984e7cf991436f3a3195935133e45c4c9a..0000000000000000000000000000000000000000
--- a/internal/querynode/flow_graph_gc_node.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package querynode
-
-import (
- "log"
-)
-
-type gcNode struct {
- BaseNode
- replica collectionReplica
-}
-
-func (gcNode *gcNode) Name() string {
- return "gcNode"
-}
-
-func (gcNode *gcNode) Operate(in []*Msg) []*Msg {
- //fmt.Println("Do gcNode operation")
-
- if len(in) != 1 {
- log.Println("Invalid operate message input in gcNode, input length = ", len(in))
- // TODO: add error handling
- }
-
- gcMsg, ok := (*in[0]).(*gcMsg)
- if !ok {
- log.Println("type assertion failed for gcMsg")
- // TODO: add error handling
- }
-
- // drop collections
- for _, collectionID := range gcMsg.gcRecord.collections {
- err := gcNode.replica.removeCollection(collectionID)
- if err != nil {
- log.Println(err)
- }
- }
-
- // drop partitions
- for _, partition := range gcMsg.gcRecord.partitions {
- err := gcNode.replica.removePartition(partition.collectionID, partition.partitionTag)
- if err != nil {
- log.Println(err)
- }
- }
-
- return nil
-}
-
-func newGCNode(replica collectionReplica) *gcNode {
- maxQueueLength := Params.FlowGraphMaxQueueLength
- maxParallelism := Params.FlowGraphMaxParallelism
-
- baseNode := BaseNode{}
- baseNode.SetMaxQueueLength(maxQueueLength)
- baseNode.SetMaxParallelism(maxParallelism)
-
- return &gcNode{
- BaseNode: baseNode,
- replica: replica,
- }
-}
diff --git a/internal/querynode/flow_graph_insert_node.go b/internal/querynode/flow_graph_insert_node.go
index 9a2c8ca1f11e34738dfbfae93eaeb8e715b70ef3..f60369521967aea4714a6099959fdcbadecb5883 100644
--- a/internal/querynode/flow_graph_insert_node.go
+++ b/internal/querynode/flow_graph_insert_node.go
@@ -90,7 +90,6 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
wg.Wait()
var res Msg = &serviceTimeMsg{
- gcRecord: iMsg.gcRecord,
timeRange: iMsg.timeRange,
}
return []*Msg{&res}
diff --git a/internal/querynode/flow_graph_message.go b/internal/querynode/flow_graph_message.go
index 451f9b6952ad003a3f687ee44550808cce6bc481..88a133fab34a84d21da9f635103c1d8f7466f5d0 100644
--- a/internal/querynode/flow_graph_message.go
+++ b/internal/querynode/flow_graph_message.go
@@ -16,7 +16,6 @@ type key2SegMsg struct {
type ddMsg struct {
collectionRecords map[string][]metaOperateRecord
partitionRecords map[string][]metaOperateRecord
- gcRecord *gcRecord
timeRange TimeRange
}
@@ -27,7 +26,6 @@ type metaOperateRecord struct {
type insertMsg struct {
insertMessages []*msgstream.InsertMsg
- gcRecord *gcRecord
timeRange TimeRange
}
@@ -37,12 +35,6 @@ type deleteMsg struct {
}
type serviceTimeMsg struct {
- gcRecord *gcRecord
- timeRange TimeRange
-}
-
-type gcMsg struct {
- gcRecord *gcRecord
timeRange TimeRange
}
@@ -63,39 +55,42 @@ type DeletePreprocessData struct {
count int32
}
-// TODO: replace partitionWithID by partition id
-type partitionWithID struct {
- partitionTag string
- collectionID UniqueID
-}
-
-type gcRecord struct {
- // collections and partitions to be dropped
- collections []UniqueID
- // TODO: use partition id
- partitions []partitionWithID
-}
-
func (ksMsg *key2SegMsg) TimeTick() Timestamp {
return ksMsg.timeRange.timestampMax
}
+func (ksMsg *key2SegMsg) DownStreamNodeIdx() int {
+ return 0
+}
+
func (suMsg *ddMsg) TimeTick() Timestamp {
return suMsg.timeRange.timestampMax
}
+func (suMsg *ddMsg) DownStreamNodeIdx() int {
+ return 0
+}
+
func (iMsg *insertMsg) TimeTick() Timestamp {
return iMsg.timeRange.timestampMax
}
+func (iMsg *insertMsg) DownStreamNodeIdx() int {
+ return 0
+}
+
func (dMsg *deleteMsg) TimeTick() Timestamp {
return dMsg.timeRange.timestampMax
}
+func (dMsg *deleteMsg) DownStreamNodeIdx() int {
+ return 0
+}
+
func (stMsg *serviceTimeMsg) TimeTick() Timestamp {
return stMsg.timeRange.timestampMax
}
-func (gcMsg *gcMsg) TimeTick() Timestamp {
- return gcMsg.timeRange.timestampMax
+func (stMsg *serviceTimeMsg) DownStreamNodeIdx() int {
+ return 0
}
diff --git a/internal/querynode/flow_graph_service_time_node.go b/internal/querynode/flow_graph_service_time_node.go
index 275666560d589c95f57b1f13ddefa90b5eb76ee6..38feb029c8af67745dbedca59af89ee25b9b9023 100644
--- a/internal/querynode/flow_graph_service_time_node.go
+++ b/internal/querynode/flow_graph_service_time_node.go
@@ -30,12 +30,7 @@ func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg {
// update service time
stNode.replica.getTSafe().set(serviceTimeMsg.timeRange.timestampMax)
//fmt.Println("update tSafe to:", getPhysicalTime(serviceTimeMsg.timeRange.timestampMax))
-
- var res Msg = &gcMsg{
- gcRecord: serviceTimeMsg.gcRecord,
- timeRange: serviceTimeMsg.timeRange,
- }
- return []*Msg{&res}
+ return nil
}
func newServiceTimeNode(replica collectionReplica) *serviceTimeNode {
diff --git a/internal/querynode/load_index_service.go b/internal/querynode/load_index_service.go
index cedbd50bb0447e0c535e926eacf208ca1d1e1b29..d8ae759b6731512e1e07df7bf03e63c0b421a0e2 100644
--- a/internal/querynode/load_index_service.go
+++ b/internal/querynode/load_index_service.go
@@ -11,9 +11,6 @@ import (
"strings"
"time"
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
-
minioKV "github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
@@ -38,16 +35,17 @@ type loadIndexService struct {
func newLoadIndexService(ctx context.Context, replica collectionReplica) *loadIndexService {
ctx1, cancel := context.WithCancel(ctx)
- // init minio
- minioClient, err := minio.New(Params.MinioEndPoint, &minio.Options{
- Creds: credentials.NewStaticV4(Params.MinioAccessKeyID, Params.MinioSecretAccessKey, ""),
- Secure: Params.MinioUseSSLStr,
- })
- if err != nil {
- panic(err)
+ option := &minioKV.Option{
+ Address: Params.MinioEndPoint,
+ AccessKeyID: Params.MinioAccessKeyID,
+ SecretAccessKeyID: Params.MinioSecretAccessKey,
+ UseSSL: Params.MinioUseSSLStr,
+ CreateBucket: true,
+ BucketName: Params.MinioBucketName,
}
- MinioKV, err := minioKV.NewMinIOKV(ctx1, minioClient, Params.MinioBucketName)
+ // TODO: load bucketName from config
+ MinioKV, err := minioKV.NewMinIOKV(ctx1, option)
if err != nil {
panic(err)
}
diff --git a/internal/querynode/load_index_service_test.go b/internal/querynode/load_index_service_test.go
index cb2fb8504e190345567ee170ecce71846fe2ce45..000edb49df2bf53fddef89b3261c9ceb15f4c5de 100644
--- a/internal/querynode/load_index_service_test.go
+++ b/internal/querynode/load_index_service_test.go
@@ -5,8 +5,6 @@ import (
"sort"
"testing"
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
"github.com/stretchr/testify/assert"
"github.com/zilliztech/milvus-distributed/internal/indexbuilder"
@@ -68,13 +66,16 @@ func TestLoadIndexService(t *testing.T) {
binarySet, err := index.Serialize()
assert.Equal(t, err, nil)
- //save index to minio
- minioClient, err := minio.New(Params.MinioEndPoint, &minio.Options{
- Creds: credentials.NewStaticV4(Params.MinioAccessKeyID, Params.MinioSecretAccessKey, ""),
- Secure: Params.MinioUseSSLStr,
- })
- assert.Equal(t, err, nil)
- minioKV, err := minioKV.NewMinIOKV(node.queryNodeLoopCtx, minioClient, Params.MinioBucketName)
+ option := &minioKV.Option{
+ Address: Params.MinioEndPoint,
+ AccessKeyID: Params.MinioAccessKeyID,
+ SecretAccessKeyID: Params.MinioSecretAccessKey,
+ UseSSL: Params.MinioUseSSLStr,
+ BucketName: Params.MinioBucketName,
+ CreateBucket: true,
+ }
+
+ minioKV, err := minioKV.NewMinIOKV(node.queryNodeLoopCtx, option)
assert.Equal(t, err, nil)
indexPaths := make([]string, 0)
for _, index := range binarySet {
diff --git a/internal/storage/internal/S3/S3_test.go b/internal/storage/internal/S3/S3_test.go
deleted file mode 100644
index c565f4a5a15cf25d109f5dcff642a0a21f2f94d1..0000000000000000000000000000000000000000
--- a/internal/storage/internal/S3/S3_test.go
+++ /dev/null
@@ -1,134 +0,0 @@
-package s3driver
-
-import (
- "context"
- "testing"
-
- "github.com/stretchr/testify/assert"
- storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-var option = storagetype.Option{BucketName: "zilliz-hz"}
-var ctx = context.Background()
-var client, err = NewS3Driver(ctx, option)
-
-func TestS3Driver_PutRowAndGetRow(t *testing.T) {
- err = client.PutRow(ctx, []byte("bar"), []byte("abcdefghijklmnoopqrstuvwxyz"), "SegmentA", 1)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("bar"), []byte("djhfkjsbdfbsdughorsgsdjhgoisdgh"), "SegmentA", 2)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("bar"), []byte("123854676ershdgfsgdfk,sdhfg;sdi8"), "SegmentB", 3)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("bar1"), []byte("testkeybarorbar_1"), "SegmentC", 3)
- assert.Nil(t, err)
- object, _ := client.GetRow(ctx, []byte("bar"), 1)
- assert.Equal(t, "abcdefghijklmnoopqrstuvwxyz", string(object))
- object, _ = client.GetRow(ctx, []byte("bar"), 2)
- assert.Equal(t, "djhfkjsbdfbsdughorsgsdjhgoisdgh", string(object))
- object, _ = client.GetRow(ctx, []byte("bar"), 5)
- assert.Equal(t, "123854676ershdgfsgdfk,sdhfg;sdi8", string(object))
- object, _ = client.GetRow(ctx, []byte("bar1"), 5)
- assert.Equal(t, "testkeybarorbar_1", string(object))
-}
-
-func TestS3Driver_DeleteRow(t *testing.T) {
- err = client.DeleteRow(ctx, []byte("bar"), 5)
- assert.Nil(t, err)
- object, _ := client.GetRow(ctx, []byte("bar"), 6)
- assert.Nil(t, object)
- err = client.DeleteRow(ctx, []byte("bar1"), 5)
- assert.Nil(t, err)
- object2, _ := client.GetRow(ctx, []byte("bar1"), 6)
- assert.Nil(t, object2)
-}
-
-func TestS3Driver_GetSegments(t *testing.T) {
- err = client.PutRow(ctx, []byte("seg"), []byte("abcdefghijklmnoopqrstuvwxyz"), "SegmentA", 1)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("seg"), []byte("djhfkjsbdfbsdughorsgsdjhgoisdgh"), "SegmentA", 2)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("seg"), []byte("123854676ershdgfsgdfk,sdhfg;sdi8"), "SegmentB", 3)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("seg2"), []byte("testkeybarorbar_1"), "SegmentC", 1)
- assert.Nil(t, err)
-
- segements, err := client.GetSegments(ctx, []byte("seg"), 4)
- assert.Nil(t, err)
- assert.Equal(t, 2, len(segements))
- if segements[0] == "SegmentA" {
- assert.Equal(t, "SegmentA", segements[0])
- assert.Equal(t, "SegmentB", segements[1])
- } else {
- assert.Equal(t, "SegmentB", segements[0])
- assert.Equal(t, "SegmentA", segements[1])
- }
-}
-
-func TestS3Driver_PutRowsAndGetRows(t *testing.T) {
- keys := [][]byte{[]byte("foo"), []byte("bar")}
- values := [][]byte{[]byte("The key is foo!"), []byte("The key is bar!")}
- segments := []string{"segmentA", "segmentB"}
- timestamps := []uint64{1, 2}
- err = client.PutRows(ctx, keys, values, segments, timestamps)
- assert.Nil(t, err)
-
- objects, err := client.GetRows(ctx, keys, timestamps)
- assert.Nil(t, err)
- assert.Equal(t, "The key is foo!", string(objects[0]))
- assert.Equal(t, "The key is bar!", string(objects[1]))
-}
-
-func TestS3Driver_DeleteRows(t *testing.T) {
- keys := [][]byte{[]byte("foo"), []byte("bar")}
- timestamps := []uint64{3, 3}
- err := client.DeleteRows(ctx, keys, timestamps)
- assert.Nil(t, err)
-
- objects, err := client.GetRows(ctx, keys, timestamps)
- assert.Nil(t, err)
- assert.Nil(t, objects[0])
- assert.Nil(t, objects[1])
-}
-
-func TestS3Driver_PutLogAndGetLog(t *testing.T) {
- err = client.PutLog(ctx, []byte("insert"), []byte("This is insert log!"), 1, 11)
- assert.Nil(t, err)
- err = client.PutLog(ctx, []byte("delete"), []byte("This is delete log!"), 2, 10)
- assert.Nil(t, err)
- err = client.PutLog(ctx, []byte("update"), []byte("This is update log!"), 3, 9)
- assert.Nil(t, err)
- err = client.PutLog(ctx, []byte("select"), []byte("This is select log!"), 4, 8)
- assert.Nil(t, err)
-
- channels := []int{5, 8, 9, 10, 11, 12, 13}
- logValues, err := client.GetLog(ctx, 0, 5, channels)
- assert.Nil(t, err)
- assert.Equal(t, "This is select log!", string(logValues[0]))
- assert.Equal(t, "This is update log!", string(logValues[1]))
- assert.Equal(t, "This is delete log!", string(logValues[2]))
- assert.Equal(t, "This is insert log!", string(logValues[3]))
-}
-
-func TestS3Driver_Segment(t *testing.T) {
- err := client.PutSegmentIndex(ctx, "segmentA", []byte("This is segmentA's index!"))
- assert.Nil(t, err)
-
- segmentIndex, err := client.GetSegmentIndex(ctx, "segmentA")
- assert.Equal(t, "This is segmentA's index!", string(segmentIndex))
- assert.Nil(t, err)
-
- err = client.DeleteSegmentIndex(ctx, "segmentA")
- assert.Nil(t, err)
-}
-
-func TestS3Driver_SegmentDL(t *testing.T) {
- err := client.PutSegmentDL(ctx, "segmentB", []byte("This is segmentB's delete log!"))
- assert.Nil(t, err)
-
- segmentDL, err := client.GetSegmentDL(ctx, "segmentB")
- assert.Nil(t, err)
- assert.Equal(t, "This is segmentB's delete log!", string(segmentDL))
-
- err = client.DeleteSegmentDL(ctx, "segmentB")
- assert.Nil(t, err)
-}
diff --git a/internal/storage/internal/S3/s3_engine.go b/internal/storage/internal/S3/s3_engine.go
deleted file mode 100644
index 8034d679e7e01fb45867b7424b6e9e31bc35c294..0000000000000000000000000000000000000000
--- a/internal/storage/internal/S3/s3_engine.go
+++ /dev/null
@@ -1,173 +0,0 @@
-package s3driver
-
-import (
- "bytes"
- "context"
- "io"
-
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/session"
- "github.com/aws/aws-sdk-go/service/s3"
- . "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-type S3Store struct {
- client *s3.S3
-}
-
-func NewS3Store(config aws.Config) (*S3Store, error) {
- sess := session.Must(session.NewSession(&config))
- service := s3.New(sess)
-
- return &S3Store{
- client: service,
- }, nil
-}
-
-func (s *S3Store) Put(ctx context.Context, key Key, value Value) error {
- _, err := s.client.PutObjectWithContext(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(string(key)),
- Body: bytes.NewReader(value),
- })
-
- //sess := session.Must(session.NewSessionWithOptions(session.Options{
- // SharedConfigState: session.SharedConfigEnable,
- //}))
- //uploader := s3manager.NewUploader(sess)
- //
- //_, err := uploader.Upload(&s3manager.UploadInput{
- // Bucket: aws.String(bucketName),
- // Key: aws.String(string(key)),
- // Body: bytes.NewReader(value),
- //})
-
- return err
-}
-
-func (s *S3Store) Get(ctx context.Context, key Key) (Value, error) {
- object, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(string(key)),
- })
- if err != nil {
- return nil, err
- }
-
- //TODO: get size
- size := 256 * 1024
- buf := make([]byte, size)
- n, err := object.Body.Read(buf)
- if err != nil && err != io.EOF {
- return nil, err
- }
- return buf[:n], nil
-}
-
-func (s *S3Store) GetByPrefix(ctx context.Context, prefix Key, keyOnly bool) ([]Key, []Value, error) {
- objectsOutput, err := s.client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
- Bucket: aws.String(bucketName),
- Prefix: aws.String(string(prefix)),
- })
-
- var objectsKeys []Key
- var objectsValues []Value
-
- if objectsOutput != nil && err == nil {
- for _, object := range objectsOutput.Contents {
- objectsKeys = append(objectsKeys, []byte(*object.Key))
- if !keyOnly {
- value, err := s.Get(ctx, []byte(*object.Key))
- if err != nil {
- return nil, nil, err
- }
- objectsValues = append(objectsValues, value)
- }
- }
- } else {
- return nil, nil, err
- }
-
- return objectsKeys, objectsValues, nil
-
-}
-
-func (s *S3Store) Scan(ctx context.Context, keyStart Key, keyEnd Key, limit int, keyOnly bool) ([]Key, []Value, error) {
- var keys []Key
- var values []Value
- limitCount := uint(limit)
- objects, err := s.client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
- Bucket: aws.String(bucketName),
- Prefix: aws.String(string(keyStart)),
- })
- if err == nil && objects != nil {
- for _, object := range objects.Contents {
- if *object.Key >= string(keyEnd) {
- keys = append(keys, []byte(*object.Key))
- if !keyOnly {
- value, err := s.Get(ctx, []byte(*object.Key))
- if err != nil {
- return nil, nil, err
- }
- values = append(values, value)
- }
- limitCount--
- if limitCount <= 0 {
- break
- }
- }
- }
- }
-
- return keys, values, err
-}
-
-func (s *S3Store) Delete(ctx context.Context, key Key) error {
- _, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(string(key)),
- })
- return err
-}
-
-func (s *S3Store) DeleteByPrefix(ctx context.Context, prefix Key) error {
-
- objects, err := s.client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
- Bucket: aws.String(bucketName),
- Prefix: aws.String(string(prefix)),
- })
-
- if objects != nil && err == nil {
- for _, object := range objects.Contents {
- _, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
- Bucket: aws.String(bucketName),
- Key: object.Key,
- })
- return err
- }
- }
-
- return nil
-}
-
-func (s *S3Store) DeleteRange(ctx context.Context, keyStart Key, keyEnd Key) error {
-
- objects, err := s.client.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
- Bucket: aws.String(bucketName),
- Prefix: aws.String(string(keyStart)),
- })
-
- if objects != nil && err == nil {
- for _, object := range objects.Contents {
- if *object.Key > string(keyEnd) {
- _, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
- Bucket: aws.String(bucketName),
- Key: object.Key,
- })
- return err
- }
- }
- }
-
- return nil
-}
diff --git a/internal/storage/internal/S3/s3_store.go b/internal/storage/internal/S3/s3_store.go
deleted file mode 100644
index 19199baa54dcdc50d6ae947f899be1068c383642..0000000000000000000000000000000000000000
--- a/internal/storage/internal/S3/s3_store.go
+++ /dev/null
@@ -1,339 +0,0 @@
-package s3driver
-
-import (
- "context"
-
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/endpoints"
- "github.com/zilliztech/milvus-distributed/internal/storage/internal/minio/codec"
- . "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-type S3Driver struct {
- driver *S3Store
-}
-
-var bucketName string
-
-func NewS3Driver(ctx context.Context, option Option) (*S3Driver, error) {
- // to-do read conf
-
- bucketName = option.BucketName
-
- S3Client, err := NewS3Store(aws.Config{
- Region: aws.String(endpoints.CnNorthwest1RegionID)})
-
- if err != nil {
- return nil, err
- }
-
- return &S3Driver{
- S3Client,
- }, nil
-}
-
-func (s *S3Driver) put(ctx context.Context, key Key, value Value, timestamp Timestamp, suffix string) error {
- minioKey, err := codec.MvccEncode(key, timestamp, suffix)
- if err != nil {
- return err
- }
-
- err = s.driver.Put(ctx, minioKey, value)
- return err
-}
-
-func (s *S3Driver) scanLE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
- keyEnd, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- return nil, nil, nil, err
- }
-
- keys, values, err := s.driver.Scan(ctx, key, []byte(keyEnd), -1, keyOnly)
- if err != nil {
- return nil, nil, nil, err
- }
-
- var timestamps []Timestamp
- for _, key := range keys {
- _, timestamp, _, _ := codec.MvccDecode(key)
- timestamps = append(timestamps, timestamp)
- }
-
- return timestamps, keys, values, nil
-}
-
-func (s *S3Driver) scanGE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
- keyStart, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- return nil, nil, nil, err
- }
-
- keys, values, err := s.driver.Scan(ctx, key, keyStart, -1, keyOnly)
- if err != nil {
- return nil, nil, nil, err
- }
-
- var timestamps []Timestamp
- for _, key := range keys {
- _, timestamp, _, _ := codec.MvccDecode(key)
- timestamps = append(timestamps, timestamp)
- }
-
- return timestamps, keys, values, nil
-}
-
-//scan(ctx context.Context, key Key, start Timestamp, end Timestamp, withValue bool) ([]Timestamp, []Key, []Value, error)
-func (s *S3Driver) deleteLE(ctx context.Context, key Key, timestamp Timestamp) error {
- keyEnd, err := codec.MvccEncode(key, timestamp, "delete")
- if err != nil {
- return err
- }
- err = s.driver.DeleteRange(ctx, key, keyEnd)
- return err
-}
-func (s *S3Driver) deleteGE(ctx context.Context, key Key, timestamp Timestamp) error {
- keys, _, err := s.driver.GetByPrefix(ctx, key, true)
- if err != nil {
- return err
- }
- keyStart, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- panic(err)
- }
- err = s.driver.DeleteRange(ctx, []byte(keyStart), keys[len(keys)-1])
- return err
-}
-func (s *S3Driver) deleteRange(ctx context.Context, key Key, start Timestamp, end Timestamp) error {
- keyStart, err := codec.MvccEncode(key, start, "")
- if err != nil {
- return err
- }
- keyEnd, err := codec.MvccEncode(key, end, "")
- if err != nil {
- return err
- }
- err = s.driver.DeleteRange(ctx, keyStart, keyEnd)
- return err
-}
-
-func (s *S3Driver) GetRow(ctx context.Context, key Key, timestamp Timestamp) (Value, error) {
- minioKey, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- return nil, err
- }
-
- keys, values, err := s.driver.Scan(ctx, append(key, byte('_')), minioKey, 1, false)
- if values == nil || keys == nil {
- return nil, err
- }
-
- _, _, suffix, err := codec.MvccDecode(keys[0])
- if err != nil {
- return nil, err
- }
- if suffix == "delete" {
- return nil, nil
- }
-
- return values[0], err
-}
-func (s *S3Driver) GetRows(ctx context.Context, keys []Key, timestamps []Timestamp) ([]Value, error) {
- var values []Value
- for i, key := range keys {
- value, err := s.GetRow(ctx, key, timestamps[i])
- if err != nil {
- return nil, err
- }
- values = append(values, value)
- }
- return values, nil
-}
-
-func (s *S3Driver) PutRow(ctx context.Context, key Key, value Value, segment string, timestamp Timestamp) error {
- minioKey, err := codec.MvccEncode(key, timestamp, segment)
- if err != nil {
- return err
- }
- err = s.driver.Put(ctx, minioKey, value)
- return err
-}
-func (s *S3Driver) PutRows(ctx context.Context, keys []Key, values []Value, segments []string, timestamps []Timestamp) error {
- maxThread := 100
- batchSize := 1
- keysLength := len(keys)
-
- if keysLength/batchSize > maxThread {
- batchSize = keysLength / maxThread
- }
-
- batchNums := keysLength / batchSize
-
- if keysLength%batchSize != 0 {
- batchNums = keysLength/batchSize + 1
- }
-
- errCh := make(chan error)
- f := func(ctx2 context.Context, keys2 []Key, values2 []Value, segments2 []string, timestamps2 []Timestamp) {
- for i := 0; i < len(keys2); i++ {
- err := s.PutRow(ctx2, keys2[i], values2[i], segments2[i], timestamps2[i])
- errCh <- err
- }
- }
- for i := 0; i < batchNums; i++ {
- j := i
- go func() {
- start, end := j*batchSize, (j+1)*batchSize
- if len(keys) < end {
- end = len(keys)
- }
- f(ctx, keys[start:end], values[start:end], segments[start:end], timestamps[start:end])
- }()
- }
-
- for i := 0; i < len(keys); i++ {
- if err := <-errCh; err != nil {
- return err
- }
- }
- return nil
-}
-
-func (s *S3Driver) GetSegments(ctx context.Context, key Key, timestamp Timestamp) ([]string, error) {
- keyEnd, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- return nil, err
- }
- keys, _, err := s.driver.Scan(ctx, append(key, byte('_')), keyEnd, -1, true)
- if err != nil {
- return nil, err
- }
- segmentsSet := map[string]bool{}
- for _, key := range keys {
- _, _, segment, err := codec.MvccDecode(key)
- if err != nil {
- panic("must no error")
- }
- if segment != "delete" {
- segmentsSet[segment] = true
- }
- }
-
- var segments []string
- for k, v := range segmentsSet {
- if v {
- segments = append(segments, k)
- }
- }
- return segments, err
-}
-
-func (s *S3Driver) DeleteRow(ctx context.Context, key Key, timestamp Timestamp) error {
- minioKey, err := codec.MvccEncode(key, timestamp, "delete")
- if err != nil {
- return err
- }
- value := []byte("0")
- err = s.driver.Put(ctx, minioKey, value)
- return err
-}
-
-func (s *S3Driver) DeleteRows(ctx context.Context, keys []Key, timestamps []Timestamp) error {
- maxThread := 100
- batchSize := 1
- keysLength := len(keys)
-
- if keysLength/batchSize > maxThread {
- batchSize = keysLength / maxThread
- }
-
- batchNums := keysLength / batchSize
-
- if keysLength%batchSize != 0 {
- batchNums = keysLength/batchSize + 1
- }
-
- errCh := make(chan error)
- f := func(ctx2 context.Context, keys2 []Key, timestamps2 []Timestamp) {
- for i := 0; i < len(keys2); i++ {
- err := s.DeleteRow(ctx2, keys2[i], timestamps2[i])
- errCh <- err
- }
- }
- for i := 0; i < batchNums; i++ {
- j := i
- go func() {
- start, end := j*batchSize, (j+1)*batchSize
- if len(keys) < end {
- end = len(keys)
- }
- f(ctx, keys[start:end], timestamps[start:end])
- }()
- }
-
- for i := 0; i < len(keys); i++ {
- if err := <-errCh; err != nil {
- return err
- }
- }
- return nil
-}
-
-func (s *S3Driver) PutLog(ctx context.Context, key Key, value Value, timestamp Timestamp, channel int) error {
- logKey := codec.LogEncode(key, timestamp, channel)
- err := s.driver.Put(ctx, logKey, value)
- return err
-}
-
-func (s *S3Driver) GetLog(ctx context.Context, start Timestamp, end Timestamp, channels []int) ([]Value, error) {
- keys, values, err := s.driver.GetByPrefix(ctx, []byte("log_"), false)
- if err != nil {
- return nil, err
- }
-
- var resultValues []Value
- for i, key := range keys {
- _, ts, channel, err := codec.LogDecode(string(key))
- if err != nil {
- return nil, err
- }
- if ts >= start && ts <= end {
- for j := 0; j < len(channels); j++ {
- if channel == channels[j] {
- resultValues = append(resultValues, values[i])
- }
- }
- }
- }
-
- return resultValues, nil
-}
-
-func (s *S3Driver) GetSegmentIndex(ctx context.Context, segment string) (SegmentIndex, error) {
-
- return s.driver.Get(ctx, codec.SegmentEncode(segment, "index"))
-}
-
-func (s *S3Driver) PutSegmentIndex(ctx context.Context, segment string, index SegmentIndex) error {
-
- return s.driver.Put(ctx, codec.SegmentEncode(segment, "index"), index)
-}
-
-func (s *S3Driver) DeleteSegmentIndex(ctx context.Context, segment string) error {
-
- return s.driver.Delete(ctx, codec.SegmentEncode(segment, "index"))
-}
-
-func (s *S3Driver) GetSegmentDL(ctx context.Context, segment string) (SegmentDL, error) {
-
- return s.driver.Get(ctx, codec.SegmentEncode(segment, "DL"))
-}
-
-func (s *S3Driver) PutSegmentDL(ctx context.Context, segment string, log SegmentDL) error {
-
- return s.driver.Put(ctx, codec.SegmentEncode(segment, "DL"), log)
-}
-
-func (s *S3Driver) DeleteSegmentDL(ctx context.Context, segment string) error {
-
- return s.driver.Delete(ctx, codec.SegmentEncode(segment, "DL"))
-}
diff --git a/internal/storage/internal/minio/codec/codec.go b/internal/storage/internal/minio/codec/codec.go
deleted file mode 100644
index 4d2b76ee2f939b9f221b47d2ef3f38ce8fe0c71e..0000000000000000000000000000000000000000
--- a/internal/storage/internal/minio/codec/codec.go
+++ /dev/null
@@ -1,101 +0,0 @@
-package codec
-
-import (
- "errors"
- "fmt"
-)
-
-func MvccEncode(key []byte, ts uint64, suffix string) ([]byte, error) {
- return []byte(string(key) + "_" + fmt.Sprintf("%016x", ^ts) + "_" + suffix), nil
-}
-
-func MvccDecode(key []byte) (string, uint64, string, error) {
- if len(key) < 16 {
- return "", 0, "", errors.New("insufficient bytes to decode value")
- }
-
- suffixIndex := 0
- TSIndex := 0
- undersCount := 0
- for i := len(key) - 1; i > 0; i-- {
- if key[i] == byte('_') {
- undersCount++
- if undersCount == 1 {
- suffixIndex = i + 1
- }
- if undersCount == 2 {
- TSIndex = i + 1
- break
- }
- }
- }
- if suffixIndex == 0 || TSIndex == 0 {
- return "", 0, "", errors.New("key is wrong formatted")
- }
-
- var TS uint64
- _, err := fmt.Sscanf(string(key[TSIndex:suffixIndex-1]), "%x", &TS)
- TS = ^TS
- if err != nil {
- return "", 0, "", err
- }
-
- return string(key[0 : TSIndex-1]), TS, string(key[suffixIndex:]), nil
-}
-
-func LogEncode(key []byte, ts uint64, channel int) []byte {
- suffix := string(key) + "_" + fmt.Sprintf("%d", channel)
- logKey, err := MvccEncode([]byte("log"), ts, suffix)
- if err != nil {
- return nil
- }
- return logKey
-}
-
-func LogDecode(logKey string) (string, uint64, int, error) {
- if len(logKey) < 16 {
- return "", 0, 0, errors.New("insufficient bytes to decode value")
- }
-
- channelIndex := 0
- keyIndex := 0
- TSIndex := 0
- undersCount := 0
-
- for i := len(logKey) - 1; i > 0; i-- {
- if logKey[i] == '_' {
- undersCount++
- if undersCount == 1 {
- channelIndex = i + 1
- }
- if undersCount == 2 {
- keyIndex = i + 1
- }
- if undersCount == 3 {
- TSIndex = i + 1
- break
- }
- }
- }
- if channelIndex == 0 || TSIndex == 0 || keyIndex == 0 || logKey[:TSIndex-1] != "log" {
- return "", 0, 0, errors.New("key is wrong formatted")
- }
-
- var TS uint64
- var channel int
- _, err := fmt.Sscanf(logKey[TSIndex:keyIndex-1], "%x", &TS)
- if err != nil {
- return "", 0, 0, err
- }
- TS = ^TS
-
- _, err = fmt.Sscanf(logKey[channelIndex:], "%d", &channel)
- if err != nil {
- return "", 0, 0, err
- }
- return logKey[keyIndex : channelIndex-1], TS, channel, nil
-}
-
-func SegmentEncode(segment string, suffix string) []byte {
- return []byte(segment + "_" + suffix)
-}
diff --git a/internal/storage/internal/minio/minio_store.go b/internal/storage/internal/minio/minio_store.go
deleted file mode 100644
index 18e2512401ac94093cd55eb89b016a9be4d92dcf..0000000000000000000000000000000000000000
--- a/internal/storage/internal/minio/minio_store.go
+++ /dev/null
@@ -1,361 +0,0 @@
-package miniodriver
-
-import (
- "context"
-
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
- "github.com/zilliztech/milvus-distributed/internal/storage/internal/minio/codec"
- storageType "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-type MinioDriver struct {
- driver *minioStore
-}
-
-var bucketName string
-
-func NewMinioDriver(ctx context.Context, option storageType.Option) (*MinioDriver, error) {
- // to-do read conf
- var endPoint = "localhost:9000"
- var accessKeyID = "testminio"
- var secretAccessKey = "testminio"
- var useSSL = false
-
- bucketName := option.BucketName
-
- minioClient, err := minio.New(endPoint, &minio.Options{
- Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
- Secure: useSSL,
- })
-
- if err != nil {
- return nil, err
- }
-
- bucketExists, err := minioClient.BucketExists(ctx, bucketName)
- if err != nil {
- return nil, err
- }
-
- if !bucketExists {
- err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{})
- if err != nil {
- return nil, err
- }
- }
- return &MinioDriver{
- &minioStore{
- client: minioClient,
- },
- }, nil
-}
-
-func (s *MinioDriver) put(ctx context.Context, key storageType.Key, value storageType.Value, timestamp storageType.Timestamp, suffix string) error {
- minioKey, err := codec.MvccEncode(key, timestamp, suffix)
- if err != nil {
- return err
- }
-
- err = s.driver.Put(ctx, minioKey, value)
- return err
-}
-
-func (s *MinioDriver) scanLE(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp, keyOnly bool) ([]storageType.Timestamp, []storageType.Key, []storageType.Value, error) {
- keyEnd, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- return nil, nil, nil, err
- }
-
- keys, values, err := s.driver.Scan(ctx, key, []byte(keyEnd), -1, keyOnly)
- if err != nil {
- return nil, nil, nil, err
- }
-
- var timestamps []storageType.Timestamp
- for _, key := range keys {
- _, timestamp, _, _ := codec.MvccDecode(key)
- timestamps = append(timestamps, timestamp)
- }
-
- return timestamps, keys, values, nil
-}
-
-func (s *MinioDriver) scanGE(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp, keyOnly bool) ([]storageType.Timestamp, []storageType.Key, []storageType.Value, error) {
- keyStart, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- return nil, nil, nil, err
- }
-
- keys, values, err := s.driver.Scan(ctx, key, keyStart, -1, keyOnly)
- if err != nil {
- return nil, nil, nil, err
- }
-
- var timestamps []storageType.Timestamp
- for _, key := range keys {
- _, timestamp, _, _ := codec.MvccDecode(key)
- timestamps = append(timestamps, timestamp)
- }
-
- return timestamps, keys, values, nil
-}
-
-//scan(ctx context.Context, key storageType.Key, start storageType.Timestamp, end storageType.Timestamp, withValue bool) ([]storageType.Timestamp, []storageType.Key, []storageType.Value, error)
-func (s *MinioDriver) deleteLE(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) error {
- keyEnd, err := codec.MvccEncode(key, timestamp, "delete")
- if err != nil {
- return err
- }
- err = s.driver.DeleteRange(ctx, key, keyEnd)
- return err
-}
-func (s *MinioDriver) deleteGE(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) error {
- keys, _, err := s.driver.GetByPrefix(ctx, key, true)
- if err != nil {
- return err
- }
- keyStart, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- panic(err)
- }
- err = s.driver.DeleteRange(ctx, keyStart, keys[len(keys)-1])
- if err != nil {
- panic(err)
- }
- return nil
-}
-func (s *MinioDriver) deleteRange(ctx context.Context, key storageType.Key, start storageType.Timestamp, end storageType.Timestamp) error {
- keyStart, err := codec.MvccEncode(key, start, "")
- if err != nil {
- return err
- }
- keyEnd, err := codec.MvccEncode(key, end, "")
- if err != nil {
- return err
- }
- err = s.driver.DeleteRange(ctx, keyStart, keyEnd)
- return err
-}
-
-func (s *MinioDriver) GetRow(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) (storageType.Value, error) {
- minioKey, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- return nil, err
- }
-
- keys, values, err := s.driver.Scan(ctx, append(key, byte('_')), minioKey, 1, false)
- if values == nil || keys == nil {
- return nil, err
- }
-
- _, _, suffix, err := codec.MvccDecode(keys[0])
- if err != nil {
- return nil, err
- }
- if suffix == "delete" {
- return nil, nil
- }
-
- return values[0], err
-}
-func (s *MinioDriver) GetRows(ctx context.Context, keys []storageType.Key, timestamps []storageType.Timestamp) ([]storageType.Value, error) {
- var values []storageType.Value
- for i, key := range keys {
- value, err := s.GetRow(ctx, key, timestamps[i])
- if err != nil {
- return nil, err
- }
- values = append(values, value)
- }
- return values, nil
-}
-
-func (s *MinioDriver) PutRow(ctx context.Context, key storageType.Key, value storageType.Value, segment string, timestamp storageType.Timestamp) error {
- minioKey, err := codec.MvccEncode(key, timestamp, segment)
- if err != nil {
- return err
- }
- err = s.driver.Put(ctx, minioKey, value)
- return err
-}
-func (s *MinioDriver) PutRows(ctx context.Context, keys []storageType.Key, values []storageType.Value, segments []string, timestamps []storageType.Timestamp) error {
- maxThread := 100
- batchSize := 1
- keysLength := len(keys)
-
- if keysLength/batchSize > maxThread {
- batchSize = keysLength / maxThread
- }
-
- batchNums := keysLength / batchSize
-
- if keysLength%batchSize != 0 {
- batchNums = keysLength/batchSize + 1
- }
-
- errCh := make(chan error)
- f := func(ctx2 context.Context, keys2 []storageType.Key, values2 []storageType.Value, segments2 []string, timestamps2 []storageType.Timestamp) {
- for i := 0; i < len(keys2); i++ {
- err := s.PutRow(ctx2, keys2[i], values2[i], segments2[i], timestamps2[i])
- errCh <- err
- }
- }
- for i := 0; i < batchNums; i++ {
- j := i
- go func() {
- start, end := j*batchSize, (j+1)*batchSize
- if len(keys) < end {
- end = len(keys)
- }
- f(ctx, keys[start:end], values[start:end], segments[start:end], timestamps[start:end])
- }()
- }
-
- for i := 0; i < len(keys); i++ {
- if err := <-errCh; err != nil {
- return err
- }
- }
- return nil
-}
-
-func (s *MinioDriver) GetSegments(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) ([]string, error) {
- keyEnd, err := codec.MvccEncode(key, timestamp, "")
- if err != nil {
- return nil, err
- }
- keys, _, err := s.driver.Scan(ctx, append(key, byte('_')), keyEnd, -1, true)
- if err != nil {
- return nil, err
- }
- segmentsSet := map[string]bool{}
- for _, key := range keys {
- _, _, segment, err := codec.MvccDecode(key)
- if err != nil {
- panic("must no error")
- }
- if segment != "delete" {
- segmentsSet[segment] = true
- }
- }
-
- var segments []string
- for k, v := range segmentsSet {
- if v {
- segments = append(segments, k)
- }
- }
- return segments, err
-}
-
-func (s *MinioDriver) DeleteRow(ctx context.Context, key storageType.Key, timestamp storageType.Timestamp) error {
- minioKey, err := codec.MvccEncode(key, timestamp, "delete")
- if err != nil {
- return err
- }
- value := []byte("0")
- err = s.driver.Put(ctx, minioKey, value)
- return err
-}
-
-func (s *MinioDriver) DeleteRows(ctx context.Context, keys []storageType.Key, timestamps []storageType.Timestamp) error {
- maxThread := 100
- batchSize := 1
- keysLength := len(keys)
-
- if keysLength/batchSize > maxThread {
- batchSize = keysLength / maxThread
- }
-
- batchNums := keysLength / batchSize
-
- if keysLength%batchSize != 0 {
- batchNums = keysLength/batchSize + 1
- }
-
- errCh := make(chan error)
- f := func(ctx2 context.Context, keys2 []storageType.Key, timestamps2 []storageType.Timestamp) {
- for i := 0; i < len(keys2); i++ {
- err := s.DeleteRow(ctx2, keys2[i], timestamps2[i])
- errCh <- err
- }
- }
- for i := 0; i < batchNums; i++ {
- j := i
- go func() {
- start, end := j*batchSize, (j+1)*batchSize
- if len(keys) < end {
- end = len(keys)
- }
- f(ctx, keys[start:end], timestamps[start:end])
- }()
- }
-
- for i := 0; i < len(keys); i++ {
- if err := <-errCh; err != nil {
- return err
- }
- }
- return nil
-}
-
-func (s *MinioDriver) PutLog(ctx context.Context, key storageType.Key, value storageType.Value, timestamp storageType.Timestamp, channel int) error {
- logKey := codec.LogEncode(key, timestamp, channel)
- err := s.driver.Put(ctx, logKey, value)
- return err
-}
-
-func (s *MinioDriver) GetLog(ctx context.Context, start storageType.Timestamp, end storageType.Timestamp, channels []int) ([]storageType.Value, error) {
- keys, values, err := s.driver.GetByPrefix(ctx, []byte("log_"), false)
- if err != nil {
- return nil, err
- }
-
- var resultValues []storageType.Value
- for i, key := range keys {
- _, ts, channel, err := codec.LogDecode(string(key))
- if err != nil {
- return nil, err
- }
- if ts >= start && ts <= end {
- for j := 0; j < len(channels); j++ {
- if channel == channels[j] {
- resultValues = append(resultValues, values[i])
- }
- }
- }
- }
-
- return resultValues, nil
-}
-
-func (s *MinioDriver) GetSegmentIndex(ctx context.Context, segment string) (storageType.SegmentIndex, error) {
-
- return s.driver.Get(ctx, codec.SegmentEncode(segment, "index"))
-}
-
-func (s *MinioDriver) PutSegmentIndex(ctx context.Context, segment string, index storageType.SegmentIndex) error {
-
- return s.driver.Put(ctx, codec.SegmentEncode(segment, "index"), index)
-}
-
-func (s *MinioDriver) DeleteSegmentIndex(ctx context.Context, segment string) error {
-
- return s.driver.Delete(ctx, codec.SegmentEncode(segment, "index"))
-}
-
-func (s *MinioDriver) GetSegmentDL(ctx context.Context, segment string) (storageType.SegmentDL, error) {
-
- return s.driver.Get(ctx, codec.SegmentEncode(segment, "DL"))
-}
-
-func (s *MinioDriver) PutSegmentDL(ctx context.Context, segment string, log storageType.SegmentDL) error {
-
- return s.driver.Put(ctx, codec.SegmentEncode(segment, "DL"), log)
-}
-
-func (s *MinioDriver) DeleteSegmentDL(ctx context.Context, segment string) error {
-
- return s.driver.Delete(ctx, codec.SegmentEncode(segment, "DL"))
-}
diff --git a/internal/storage/internal/minio/minio_storeEngine.go b/internal/storage/internal/minio/minio_storeEngine.go
deleted file mode 100644
index 64d74e859032b0306c9e9cce655d48d3df52a8a0..0000000000000000000000000000000000000000
--- a/internal/storage/internal/minio/minio_storeEngine.go
+++ /dev/null
@@ -1,130 +0,0 @@
-package miniodriver
-
-import (
- "bytes"
- "context"
- "io"
-
- "github.com/minio/minio-go/v7"
- . "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-type minioStore struct {
- client *minio.Client
-}
-
-func (s *minioStore) Put(ctx context.Context, key Key, value Value) error {
- reader := bytes.NewReader(value)
- _, err := s.client.PutObject(ctx, bucketName, string(key), reader, int64(len(value)), minio.PutObjectOptions{})
-
- if err != nil {
- return err
- }
-
- return err
-}
-
-func (s *minioStore) Get(ctx context.Context, key Key) (Value, error) {
- object, err := s.client.GetObject(ctx, bucketName, string(key), minio.GetObjectOptions{})
- if err != nil {
- return nil, err
- }
-
- size := 256 * 1024
- buf := make([]byte, size)
- n, err := object.Read(buf)
- if err != nil && err != io.EOF {
- return nil, err
- }
- return buf[:n], nil
-}
-
-func (s *minioStore) GetByPrefix(ctx context.Context, prefix Key, keyOnly bool) ([]Key, []Value, error) {
- objects := s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: string(prefix)})
-
- var objectsKeys []Key
- var objectsValues []Value
-
- for object := range objects {
- objectsKeys = append(objectsKeys, []byte(object.Key))
- if !keyOnly {
- value, err := s.Get(ctx, []byte(object.Key))
- if err != nil {
- return nil, nil, err
- }
- objectsValues = append(objectsValues, value)
- }
- }
-
- return objectsKeys, objectsValues, nil
-
-}
-
-func (s *minioStore) Scan(ctx context.Context, keyStart Key, keyEnd Key, limit int, keyOnly bool) ([]Key, []Value, error) {
- var keys []Key
- var values []Value
- limitCount := uint(limit)
- for object := range s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: string(keyStart)}) {
- if object.Key >= string(keyEnd) {
- keys = append(keys, []byte(object.Key))
- if !keyOnly {
- value, err := s.Get(ctx, []byte(object.Key))
- if err != nil {
- return nil, nil, err
- }
- values = append(values, value)
- }
- limitCount--
- if limitCount <= 0 {
- break
- }
- }
- }
-
- return keys, values, nil
-}
-
-func (s *minioStore) Delete(ctx context.Context, key Key) error {
- err := s.client.RemoveObject(ctx, bucketName, string(key), minio.RemoveObjectOptions{})
- return err
-}
-
-func (s *minioStore) DeleteByPrefix(ctx context.Context, prefix Key) error {
- objectsCh := make(chan minio.ObjectInfo)
-
- go func() {
- defer close(objectsCh)
-
- for object := range s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: string(prefix)}) {
- objectsCh <- object
- }
- }()
-
- for rErr := range s.client.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{GovernanceBypass: true}) {
- if rErr.Err != nil {
- return rErr.Err
- }
- }
- return nil
-}
-
-func (s *minioStore) DeleteRange(ctx context.Context, keyStart Key, keyEnd Key) error {
- objectsCh := make(chan minio.ObjectInfo)
-
- go func() {
- defer close(objectsCh)
-
- for object := range s.client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: string(keyStart)}) {
- if object.Key <= string(keyEnd) {
- objectsCh <- object
- }
- }
- }()
-
- for rErr := range s.client.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{GovernanceBypass: true}) {
- if rErr.Err != nil {
- return rErr.Err
- }
- }
- return nil
-}
diff --git a/internal/storage/internal/minio/minio_test.go b/internal/storage/internal/minio/minio_test.go
deleted file mode 100644
index d98a98cfeaac2a4d77b8cf999940a04c1a61c71d..0000000000000000000000000000000000000000
--- a/internal/storage/internal/minio/minio_test.go
+++ /dev/null
@@ -1,134 +0,0 @@
-package miniodriver
-
-import (
- "context"
- "testing"
-
- "github.com/stretchr/testify/assert"
- storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-var option = storagetype.Option{BucketName: "zilliz-hz"}
-var ctx = context.Background()
-var client, err = NewMinioDriver(ctx, option)
-
-func TestMinioDriver_PutRowAndGetRow(t *testing.T) {
- err = client.PutRow(ctx, []byte("bar"), []byte("abcdefghijklmnoopqrstuvwxyz"), "SegmentA", 1)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("bar"), []byte("djhfkjsbdfbsdughorsgsdjhgoisdgh"), "SegmentA", 2)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("bar"), []byte("123854676ershdgfsgdfk,sdhfg;sdi8"), "SegmentB", 3)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("bar1"), []byte("testkeybarorbar_1"), "SegmentC", 3)
- assert.Nil(t, err)
- object, _ := client.GetRow(ctx, []byte("bar"), 5)
- assert.Equal(t, "abcdefghijklmnoopqrstuvwxyz", string(object))
- object, _ = client.GetRow(ctx, []byte("bar"), 2)
- assert.Equal(t, "djhfkjsbdfbsdughorsgsdjhgoisdgh", string(object))
- object, _ = client.GetRow(ctx, []byte("bar"), 5)
- assert.Equal(t, "123854676ershdgfsgdfk,sdhfg;sdi8", string(object))
- object, _ = client.GetRow(ctx, []byte("bar1"), 5)
- assert.Equal(t, "testkeybarorbar_1", string(object))
-}
-
-func TestMinioDriver_DeleteRow(t *testing.T) {
- err = client.DeleteRow(ctx, []byte("bar"), 5)
- assert.Nil(t, err)
- object, _ := client.GetRow(ctx, []byte("bar"), 6)
- assert.Nil(t, object)
- err = client.DeleteRow(ctx, []byte("bar1"), 5)
- assert.Nil(t, err)
- object2, _ := client.GetRow(ctx, []byte("bar1"), 6)
- assert.Nil(t, object2)
-}
-
-func TestMinioDriver_GetSegments(t *testing.T) {
- err = client.PutRow(ctx, []byte("seg"), []byte("abcdefghijklmnoopqrstuvwxyz"), "SegmentA", 1)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("seg"), []byte("djhfkjsbdfbsdughorsgsdjhgoisdgh"), "SegmentA", 2)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("seg"), []byte("123854676ershdgfsgdfk,sdhfg;sdi8"), "SegmentB", 3)
- assert.Nil(t, err)
- err = client.PutRow(ctx, []byte("seg2"), []byte("testkeybarorbar_1"), "SegmentC", 1)
- assert.Nil(t, err)
-
- segements, err := client.GetSegments(ctx, []byte("seg"), 4)
- assert.Nil(t, err)
- assert.Equal(t, 2, len(segements))
- if segements[0] == "SegmentA" {
- assert.Equal(t, "SegmentA", segements[0])
- assert.Equal(t, "SegmentB", segements[1])
- } else {
- assert.Equal(t, "SegmentB", segements[0])
- assert.Equal(t, "SegmentA", segements[1])
- }
-}
-
-func TestMinioDriver_PutRowsAndGetRows(t *testing.T) {
- keys := [][]byte{[]byte("foo"), []byte("bar")}
- values := [][]byte{[]byte("The key is foo!"), []byte("The key is bar!")}
- segments := []string{"segmentA", "segmentB"}
- timestamps := []uint64{1, 2}
- err = client.PutRows(ctx, keys, values, segments, timestamps)
- assert.Nil(t, err)
-
- objects, err := client.GetRows(ctx, keys, timestamps)
- assert.Nil(t, err)
- assert.Equal(t, "The key is foo!", string(objects[0]))
- assert.Equal(t, "The key is bar!", string(objects[1]))
-}
-
-func TestMinioDriver_DeleteRows(t *testing.T) {
- keys := [][]byte{[]byte("foo"), []byte("bar")}
- timestamps := []uint64{3, 3}
- err := client.DeleteRows(ctx, keys, timestamps)
- assert.Nil(t, err)
-
- objects, err := client.GetRows(ctx, keys, timestamps)
- assert.Nil(t, err)
- assert.Nil(t, objects[0])
- assert.Nil(t, objects[1])
-}
-
-func TestMinioDriver_PutLogAndGetLog(t *testing.T) {
- err = client.PutLog(ctx, []byte("insert"), []byte("This is insert log!"), 1, 11)
- assert.Nil(t, err)
- err = client.PutLog(ctx, []byte("delete"), []byte("This is delete log!"), 2, 10)
- assert.Nil(t, err)
- err = client.PutLog(ctx, []byte("update"), []byte("This is update log!"), 3, 9)
- assert.Nil(t, err)
- err = client.PutLog(ctx, []byte("select"), []byte("This is select log!"), 4, 8)
- assert.Nil(t, err)
-
- channels := []int{5, 8, 9, 10, 11, 12, 13}
- logValues, err := client.GetLog(ctx, 0, 5, channels)
- assert.Nil(t, err)
- assert.Equal(t, "This is select log!", string(logValues[0]))
- assert.Equal(t, "This is update log!", string(logValues[1]))
- assert.Equal(t, "This is delete log!", string(logValues[2]))
- assert.Equal(t, "This is insert log!", string(logValues[3]))
-}
-
-func TestMinioDriver_Segment(t *testing.T) {
- err := client.PutSegmentIndex(ctx, "segmentA", []byte("This is segmentA's index!"))
- assert.Nil(t, err)
-
- segmentIndex, err := client.GetSegmentIndex(ctx, "segmentA")
- assert.Equal(t, "This is segmentA's index!", string(segmentIndex))
- assert.Nil(t, err)
-
- err = client.DeleteSegmentIndex(ctx, "segmentA")
- assert.Nil(t, err)
-}
-
-func TestMinioDriver_SegmentDL(t *testing.T) {
- err := client.PutSegmentDL(ctx, "segmentB", []byte("This is segmentB's delete log!"))
- assert.Nil(t, err)
-
- segmentDL, err := client.GetSegmentDL(ctx, "segmentB")
- assert.Nil(t, err)
- assert.Equal(t, "This is segmentB's delete log!", string(segmentDL))
-
- err = client.DeleteSegmentDL(ctx, "segmentB")
- assert.Nil(t, err)
-}
diff --git a/internal/storage/internal/tikv/codec/codec.go b/internal/storage/internal/tikv/codec/codec.go
deleted file mode 100644
index ca09296097e8778ca1023c3395801e7cb0e99669..0000000000000000000000000000000000000000
--- a/internal/storage/internal/tikv/codec/codec.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package codec
-
-import (
- "encoding/binary"
- "errors"
-
- "github.com/tikv/client-go/codec"
-)
-
-var (
- Delimiter = byte('_')
- DelimiterPlusOne = Delimiter + 0x01
- DeleteMark = byte('d')
- SegmentIndexMark = byte('i')
- SegmentDLMark = byte('d')
-)
-
-// EncodeKey append timestamp, delimiter, and suffix string
-// to one slice key.
-// Note: suffix string should not contains Delimiter
-func EncodeKey(key []byte, timestamp uint64, suffix string) []byte {
- //TODO: should we encode key to memory comparable
- ret := EncodeDelimiter(key, Delimiter)
- ret = codec.EncodeUintDesc(ret, timestamp)
- return append(ret, suffix...)
-}
-
-func DecodeKey(key []byte) ([]byte, uint64, string, error) {
- if len(key) < 8 {
- return nil, 0, "", errors.New("insufficient bytes to decode value")
- }
-
- lenDeKey := 0
- for i := len(key) - 1; i > 0; i-- {
- if key[i] == Delimiter {
- lenDeKey = i
- break
- }
- }
-
- if lenDeKey == 0 || lenDeKey+8 > len(key) {
- return nil, 0, "", errors.New("insufficient bytes to decode value")
- }
-
- tsBytes := key[lenDeKey+1 : lenDeKey+9]
- ts := binary.BigEndian.Uint64(tsBytes)
- suffix := string(key[lenDeKey+9:])
- key = key[:lenDeKey-1]
- return key, ^ts, suffix, nil
-}
-
-// EncodeDelimiter append a delimiter byte to slice b, and return the appended slice.
-func EncodeDelimiter(b []byte, delimiter byte) []byte {
- return append(b, delimiter)
-}
-
-func EncodeSegment(segName []byte, segType byte) []byte {
- segmentKey := []byte("segment")
- segmentKey = append(segmentKey, Delimiter)
- segmentKey = append(segmentKey, segName...)
- return append(segmentKey, Delimiter, segType)
-}
diff --git a/internal/storage/internal/tikv/tikv_store.go b/internal/storage/internal/tikv/tikv_store.go
deleted file mode 100644
index 5ecf8936f675f574f262219013001db4519f77cd..0000000000000000000000000000000000000000
--- a/internal/storage/internal/tikv/tikv_store.go
+++ /dev/null
@@ -1,389 +0,0 @@
-package tikvdriver
-
-import (
- "context"
- "errors"
- "strconv"
- "strings"
-
- "github.com/tikv/client-go/config"
- "github.com/tikv/client-go/rawkv"
- . "github.com/zilliztech/milvus-distributed/internal/storage/internal/tikv/codec"
- . "github.com/zilliztech/milvus-distributed/internal/storage/type"
- storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-func keyAddOne(key Key) Key {
- if key == nil {
- return nil
- }
- lenKey := len(key)
- ret := make(Key, lenKey)
- copy(ret, key)
- ret[lenKey-1] += 0x01
- return ret
-}
-
-type tikvEngine struct {
- client *rawkv.Client
- conf config.Config
-}
-
-func (e tikvEngine) Put(ctx context.Context, key Key, value Value) error {
- return e.client.Put(ctx, key, value)
-}
-
-func (e tikvEngine) BatchPut(ctx context.Context, keys []Key, values []Value) error {
- return e.client.BatchPut(ctx, keys, values)
-}
-
-func (e tikvEngine) Get(ctx context.Context, key Key) (Value, error) {
- return e.client.Get(ctx, key)
-}
-
-func (e tikvEngine) GetByPrefix(ctx context.Context, prefix Key, keyOnly bool) (keys []Key, values []Value, err error) {
- startKey := prefix
- endKey := keyAddOne(prefix)
- limit := e.conf.Raw.MaxScanLimit
- for {
- ks, vs, err := e.Scan(ctx, startKey, endKey, limit, keyOnly)
- if err != nil {
- return keys, values, err
- }
- keys = append(keys, ks...)
- values = append(values, vs...)
- if len(ks) < limit {
- break
- }
- // update the start key, and exclude the start key
- startKey = append(ks[len(ks)-1], '\000')
- }
- return
-}
-
-func (e tikvEngine) Scan(ctx context.Context, startKey Key, endKey Key, limit int, keyOnly bool) ([]Key, []Value, error) {
- return e.client.Scan(ctx, startKey, endKey, limit, rawkv.ScanOption{KeyOnly: keyOnly})
-}
-
-func (e tikvEngine) Delete(ctx context.Context, key Key) error {
- return e.client.Delete(ctx, key)
-}
-
-func (e tikvEngine) DeleteByPrefix(ctx context.Context, prefix Key) error {
- startKey := prefix
- endKey := keyAddOne(prefix)
- return e.client.DeleteRange(ctx, startKey, endKey)
-}
-
-func (e tikvEngine) DeleteRange(ctx context.Context, startKey Key, endKey Key) error {
- return e.client.DeleteRange(ctx, startKey, endKey)
-}
-
-func (e tikvEngine) Close() error {
- return e.client.Close()
-}
-
-type TikvStore struct {
- engine *tikvEngine
-}
-
-func NewTikvStore(ctx context.Context, option storagetype.Option) (*TikvStore, error) {
-
- conf := config.Default()
- client, err := rawkv.NewClient(ctx, []string{option.TikvAddress}, conf)
- if err != nil {
- return nil, err
- }
- return &TikvStore{
- &tikvEngine{
- client: client,
- conf: conf,
- },
- }, nil
-}
-
-func (s *TikvStore) Name() string {
- return "TiKV storage"
-}
-
-func (s *TikvStore) put(ctx context.Context, key Key, value Value, timestamp Timestamp, suffix string) error {
- return s.engine.Put(ctx, EncodeKey(key, timestamp, suffix), value)
-}
-
-func (s *TikvStore) scanLE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
- panic("implement me")
-}
-
-func (s *TikvStore) scanGE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
- panic("implement me")
-}
-
-func (s *TikvStore) scan(ctx context.Context, key Key, start Timestamp, end Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error) {
- //startKey := EncodeKey(key, start, "")
- //endKey := EncodeKey(EncodeDelimiter(key, DelimiterPlusOne), end, "")
- //return s.engine.Scan(ctx, startKey, endKey, -1, keyOnly)
- panic("implement me")
-}
-
-func (s *TikvStore) deleteLE(ctx context.Context, key Key, timestamp Timestamp) error {
- panic("implement me")
-}
-
-func (s *TikvStore) deleteGE(ctx context.Context, key Key, timestamp Timestamp) error {
- panic("implement me")
-}
-
-func (s *TikvStore) deleteRange(ctx context.Context, key Key, start Timestamp, end Timestamp) error {
- panic("implement me")
-}
-
-func (s *TikvStore) GetRow(ctx context.Context, key Key, timestamp Timestamp) (Value, error) {
- startKey := EncodeKey(key, timestamp, "")
- endKey := EncodeDelimiter(key, DelimiterPlusOne)
- keys, values, err := s.engine.Scan(ctx, startKey, endKey, 1, false)
- if err != nil || keys == nil {
- return nil, err
- }
- _, _, suffix, err := DecodeKey(keys[0])
- if err != nil {
- return nil, err
- }
- // key is marked deleted
- if suffix == string(DeleteMark) {
- return nil, nil
- }
- return values[0], nil
-}
-
-// TODO: how to spilt keys to some batches
-var batchSize = 100
-
-type kvPair struct {
- key Key
- value Value
- err error
-}
-
-func batchKeys(keys []Key) [][]Key {
- keysLen := len(keys)
- numBatch := (keysLen-1)/batchSize + 1
- batches := make([][]Key, numBatch)
-
- for i := 0; i < numBatch; i++ {
- batchStart := i * batchSize
- batchEnd := batchStart + batchSize
- // the last batch
- if i == numBatch-1 {
- batchEnd = keysLen
- }
- batches[i] = keys[batchStart:batchEnd]
- }
- return batches
-}
-
-func (s *TikvStore) GetRows(ctx context.Context, keys []Key, timestamps []Timestamp) ([]Value, error) {
- if len(keys) != len(timestamps) {
- return nil, errors.New("the len of keys is not equal to the len of timestamps")
- }
-
- batches := batchKeys(keys)
- ch := make(chan kvPair, len(keys))
- ctx, cancel := context.WithCancel(ctx)
- defer cancel()
-
- for n, b := range batches {
- batch := b
- numBatch := n
- go func() {
- for i, key := range batch {
- select {
- case <-ctx.Done():
- return
- default:
- v, err := s.GetRow(ctx, key, timestamps[numBatch*batchSize+i])
- ch <- kvPair{
- key: key,
- value: v,
- err: err,
- }
- }
- }
- }()
- }
-
- var err error
- var values []Value
- kvMap := make(map[string]Value)
- for i := 0; i < len(keys); i++ {
- kv := <-ch
- if kv.err != nil {
- cancel()
- if err == nil {
- err = kv.err
- }
- }
- kvMap[string(kv.key)] = kv.value
- }
- for _, key := range keys {
- values = append(values, kvMap[string(key)])
- }
- return values, err
-}
-
-func (s *TikvStore) PutRow(ctx context.Context, key Key, value Value, segment string, timestamp Timestamp) error {
- return s.put(ctx, key, value, timestamp, segment)
-}
-
-func (s *TikvStore) PutRows(ctx context.Context, keys []Key, values []Value, segments []string, timestamps []Timestamp) error {
- if len(keys) != len(values) {
- return errors.New("the len of keys is not equal to the len of values")
- }
- if len(keys) != len(timestamps) {
- return errors.New("the len of keys is not equal to the len of timestamps")
- }
-
- encodedKeys := make([]Key, len(keys))
- for i, key := range keys {
- encodedKeys[i] = EncodeKey(key, timestamps[i], segments[i])
- }
- return s.engine.BatchPut(ctx, encodedKeys, values)
-}
-
-func (s *TikvStore) DeleteRow(ctx context.Context, key Key, timestamp Timestamp) error {
- return s.put(ctx, key, Value{0x00}, timestamp, string(DeleteMark))
-}
-
-func (s *TikvStore) DeleteRows(ctx context.Context, keys []Key, timestamps []Timestamp) error {
- encodeKeys := make([]Key, len(keys))
- values := make([]Value, len(keys))
- for i, key := range keys {
- encodeKeys[i] = EncodeKey(key, timestamps[i], string(DeleteMark))
- values[i] = Value{0x00}
- }
- return s.engine.BatchPut(ctx, encodeKeys, values)
-}
-
-//func (s *TikvStore) DeleteRows(ctx context.Context, keys []Key, timestamp Timestamp) error {
-// batches := batchKeys(keys)
-// ch := make(chan error, len(batches))
-// ctx, cancel := context.WithCancel(ctx)
-//
-// for _, b := range batches {
-// batch := b
-// go func() {
-// for _, key := range batch {
-// select {
-// case <-ctx.Done():
-// return
-// default:
-// ch <- s.DeleteRow(ctx, key, timestamp)
-// }
-// }
-// }()
-// }
-//
-// var err error
-// for i := 0; i < len(keys); i++ {
-// if e := <-ch; e != nil {
-// cancel()
-// if err == nil {
-// err = e
-// }
-// }
-// }
-// return err
-//}
-
-func (s *TikvStore) PutLog(ctx context.Context, key Key, value Value, timestamp Timestamp, channel int) error {
- suffix := string(EncodeDelimiter(key, DelimiterPlusOne)) + strconv.Itoa(channel)
- return s.put(ctx, Key("log"), value, timestamp, suffix)
-}
-
-func (s *TikvStore) GetLog(ctx context.Context, start Timestamp, end Timestamp, channels []int) (logs []Value, err error) {
- key := Key("log")
- startKey := EncodeKey(key, end, "")
- endKey := EncodeKey(key, start, "")
- // TODO: use for loop to ensure get all keys
- keys, values, err := s.engine.Scan(ctx, startKey, endKey, s.engine.conf.Raw.MaxScanLimit, false)
- if err != nil || keys == nil {
- return nil, err
- }
-
- for i, key := range keys {
- _, _, suffix, err := DecodeKey(key)
- log := values[i]
- if err != nil {
- return logs, err
- }
-
- // no channels filter
- if len(channels) == 0 {
- logs = append(logs, log)
- }
- slice := strings.Split(suffix, string(DelimiterPlusOne))
- channel, err := strconv.Atoi(slice[len(slice)-1])
- if err != nil {
- panic(err)
- }
- for _, item := range channels {
- if item == channel {
- logs = append(logs, log)
- break
- }
- }
- }
- return
-}
-
-func (s *TikvStore) GetSegmentIndex(ctx context.Context, segment string) (SegmentIndex, error) {
- return s.engine.Get(ctx, EncodeSegment([]byte(segment), SegmentIndexMark))
-}
-
-func (s *TikvStore) PutSegmentIndex(ctx context.Context, segment string, index SegmentIndex) error {
- return s.engine.Put(ctx, EncodeSegment([]byte(segment), SegmentIndexMark), index)
-}
-
-func (s *TikvStore) DeleteSegmentIndex(ctx context.Context, segment string) error {
- return s.engine.Delete(ctx, EncodeSegment([]byte(segment), SegmentIndexMark))
-}
-
-func (s *TikvStore) GetSegmentDL(ctx context.Context, segment string) (SegmentDL, error) {
- return s.engine.Get(ctx, EncodeSegment([]byte(segment), SegmentDLMark))
-}
-
-func (s *TikvStore) PutSegmentDL(ctx context.Context, segment string, log SegmentDL) error {
- return s.engine.Put(ctx, EncodeSegment([]byte(segment), SegmentDLMark), log)
-}
-
-func (s *TikvStore) DeleteSegmentDL(ctx context.Context, segment string) error {
- return s.engine.Delete(ctx, EncodeSegment([]byte(segment), SegmentDLMark))
-}
-
-func (s *TikvStore) GetSegments(ctx context.Context, key Key, timestamp Timestamp) ([]string, error) {
- keys, _, err := s.engine.GetByPrefix(ctx, EncodeDelimiter(key, Delimiter), true)
- if err != nil {
- return nil, err
- }
- segmentsSet := map[string]bool{}
- for _, key := range keys {
- _, ts, segment, err := DecodeKey(key)
- if err != nil {
- panic("must no error")
- }
- if ts <= timestamp && segment != string(DeleteMark) {
- segmentsSet[segment] = true
- }
- }
-
- var segments []string
- for k, v := range segmentsSet {
- if v {
- segments = append(segments, k)
- }
- }
- return segments, err
-}
-
-func (s *TikvStore) Close() error {
- return s.engine.Close()
-}
diff --git a/internal/storage/internal/tikv/tikv_test.go b/internal/storage/internal/tikv/tikv_test.go
deleted file mode 100644
index 4e69d14d2c3079f7251123666881206a9af03d03..0000000000000000000000000000000000000000
--- a/internal/storage/internal/tikv/tikv_test.go
+++ /dev/null
@@ -1,293 +0,0 @@
-package tikvdriver
-
-import (
- "bytes"
- "context"
- "fmt"
- "math"
- "os"
- "sort"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- . "github.com/zilliztech/milvus-distributed/internal/storage/internal/tikv/codec"
- . "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-//var store TikvStore
-var store *TikvStore
-var option = Option{TikvAddress: "localhost:2379"}
-
-func TestMain(m *testing.M) {
- store, _ = NewTikvStore(context.Background(), option)
- exitCode := m.Run()
- _ = store.Close()
- os.Exit(exitCode)
-}
-
-func TestTikvEngine_Prefix(t *testing.T) {
- ctx := context.Background()
- prefix := Key("key")
- engine := store.engine
- value := Value("value")
-
- // Put some key with same prefix
- key := prefix
- err := engine.Put(ctx, key, value)
- require.Nil(t, err)
- key = EncodeKey(prefix, 0, "")
- err = engine.Put(ctx, key, value)
- assert.Nil(t, err)
-
- // Get by prefix
- ks, _, err := engine.GetByPrefix(ctx, prefix, true)
- assert.Equal(t, 2, len(ks))
- assert.Nil(t, err)
-
- // Delete by prefix
- err = engine.DeleteByPrefix(ctx, prefix)
- assert.Nil(t, err)
- ks, _, err = engine.GetByPrefix(ctx, prefix, true)
- assert.Equal(t, 0, len(ks))
- assert.Nil(t, err)
-
- //Test large amount keys
- num := engine.conf.Raw.MaxScanLimit + 1
- keys := make([]Key, num)
- values := make([]Value, num)
- for i := 0; i < num; i++ {
- key = EncodeKey(prefix, uint64(i), "")
- keys[i] = key
- values[i] = value
- }
- err = engine.BatchPut(ctx, keys, values)
- assert.Nil(t, err)
-
- ks, _, err = engine.GetByPrefix(ctx, prefix, true)
- assert.Nil(t, err)
- assert.Equal(t, num, len(ks))
- err = engine.DeleteByPrefix(ctx, prefix)
- assert.Nil(t, err)
-}
-
-func TestTikvStore_Row(t *testing.T) {
- ctx := context.Background()
- key := Key("key")
-
- // Add same row with different timestamp
- err := store.PutRow(ctx, key, Value("value0"), "segment0", 0)
- assert.Nil(t, err)
- err = store.PutRow(ctx, key, Value("value1"), "segment0", 2)
- assert.Nil(t, err)
-
- // Get most recent row using key and timestamp
- v, err := store.GetRow(ctx, key, 3)
- assert.Nil(t, err)
- assert.Equal(t, Value("value1"), v)
- v, err = store.GetRow(ctx, key, 2)
- assert.Nil(t, err)
- assert.Equal(t, Value("value1"), v)
- v, err = store.GetRow(ctx, key, 1)
- assert.Nil(t, err)
- assert.Equal(t, Value("value0"), v)
-
- // Add a different row, but with same prefix
- key1 := Key("key_y")
- err = store.PutRow(ctx, key1, Value("valuey"), "segment0", 2)
- assert.Nil(t, err)
-
- // Get most recent row using key and timestamp
- v, err = store.GetRow(ctx, key, 3)
- assert.Nil(t, err)
- assert.Equal(t, Value("value1"), v)
- v, err = store.GetRow(ctx, key1, 3)
- assert.Nil(t, err)
- assert.Equal(t, Value("valuey"), v)
-
- // Delete a row
- err = store.DeleteRow(ctx, key, 4)
- assert.Nil(t, err)
- v, err = store.GetRow(ctx, key, 5)
- assert.Nil(t, err)
- assert.Nil(t, v)
-
- // Clear test data
- err = store.engine.DeleteByPrefix(ctx, key)
- assert.Nil(t, err)
- k, va, err := store.engine.GetByPrefix(ctx, key, false)
- assert.Nil(t, err)
- assert.Nil(t, k)
- assert.Nil(t, va)
-}
-
-func TestTikvStore_BatchRow(t *testing.T) {
- ctx := context.Background()
-
- // Prepare test data
- size := 0
- var testKeys []Key
- var testValues []Value
- var segments []string
- var timestamps []Timestamp
- for i := 0; size/store.engine.conf.Raw.MaxBatchPutSize < 1; i++ {
- key := fmt.Sprint("key", i)
- size += len(key)
- testKeys = append(testKeys, []byte(key))
- value := fmt.Sprint("value", i)
- size += len(value)
- testValues = append(testValues, []byte(value))
- segments = append(segments, "test")
- v, err := store.GetRow(ctx, Key(key), math.MaxUint64)
- assert.Nil(t, v)
- assert.Nil(t, err)
- }
-
- // Batch put rows
- for range testKeys {
- timestamps = append(timestamps, 1)
- }
- err := store.PutRows(ctx, testKeys, testValues, segments, timestamps)
- assert.Nil(t, err)
-
- // Batch get rows
- for i := range timestamps {
- timestamps[i] = 2
- }
- checkValues, err := store.GetRows(ctx, testKeys, timestamps)
- assert.NotNil(t, checkValues)
- assert.Nil(t, err)
- assert.Equal(t, len(checkValues), len(testValues))
- for i := range testKeys {
- assert.Equal(t, testValues[i], checkValues[i])
- }
-
- // Delete all test rows
- for i := range timestamps {
- timestamps[i] = math.MaxUint64
- }
- err = store.DeleteRows(ctx, testKeys, timestamps)
- assert.Nil(t, err)
- // Ensure all test row is deleted
- for i := range timestamps {
- timestamps[i] = math.MaxUint64
- }
- checkValues, err = store.GetRows(ctx, testKeys, timestamps)
- assert.Nil(t, err)
- for _, value := range checkValues {
- assert.Nil(t, value)
- }
-
- // Clean test data
- err = store.engine.DeleteByPrefix(ctx, Key("key"))
- assert.Nil(t, err)
-}
-
-func TestTikvStore_GetSegments(t *testing.T) {
- ctx := context.Background()
- key := Key("key")
-
- // Put rows
- err := store.PutRow(ctx, key, Value{0}, "a", 1)
- assert.Nil(t, err)
- err = store.PutRow(ctx, key, Value{0}, "a", 2)
- assert.Nil(t, err)
- err = store.PutRow(ctx, key, Value{0}, "c", 3)
- assert.Nil(t, err)
-
- // Get segments
- segs, err := store.GetSegments(ctx, key, 2)
- assert.Nil(t, err)
- assert.Equal(t, 1, len(segs))
- assert.Equal(t, "a", segs[0])
-
- segs, err = store.GetSegments(ctx, key, 3)
- assert.Nil(t, err)
- assert.Equal(t, 2, len(segs))
-
- // Clean test data
- err = store.engine.DeleteByPrefix(ctx, key)
- assert.Nil(t, err)
-}
-
-func TestTikvStore_Log(t *testing.T) {
- ctx := context.Background()
-
- // Put some log
- err := store.PutLog(ctx, Key("key1"), Value("value1"), 1, 1)
- assert.Nil(t, err)
- err = store.PutLog(ctx, Key("key1"), Value("value1_1"), 1, 2)
- assert.Nil(t, err)
- err = store.PutLog(ctx, Key("key2"), Value("value2"), 2, 1)
- assert.Nil(t, err)
-
- // Check log
- log, err := store.GetLog(ctx, 0, 2, []int{1, 2})
- if err != nil {
- panic(err)
- }
- sort.Slice(log, func(i, j int) bool {
- return bytes.Compare(log[i], log[j]) == -1
- })
- assert.Equal(t, log[0], Value("value1"))
- assert.Equal(t, log[1], Value("value1_1"))
- assert.Equal(t, log[2], Value("value2"))
-
- // Delete test data
- err = store.engine.DeleteByPrefix(ctx, Key("log"))
- assert.Nil(t, err)
-}
-
-func TestTikvStore_SegmentIndex(t *testing.T) {
- ctx := context.Background()
-
- // Put segment index
- err := store.PutSegmentIndex(ctx, "segment0", []byte("index0"))
- assert.Nil(t, err)
- err = store.PutSegmentIndex(ctx, "segment1", []byte("index1"))
- assert.Nil(t, err)
-
- // Get segment index
- index, err := store.GetSegmentIndex(ctx, "segment0")
- assert.Nil(t, err)
- assert.Equal(t, []byte("index0"), index)
- index, err = store.GetSegmentIndex(ctx, "segment1")
- assert.Nil(t, err)
- assert.Equal(t, []byte("index1"), index)
-
- // Delete segment index
- err = store.DeleteSegmentIndex(ctx, "segment0")
- assert.Nil(t, err)
- err = store.DeleteSegmentIndex(ctx, "segment1")
- assert.Nil(t, err)
- index, err = store.GetSegmentIndex(ctx, "segment0")
- assert.Nil(t, err)
- assert.Nil(t, index)
-}
-
-func TestTikvStore_DeleteSegmentDL(t *testing.T) {
- ctx := context.Background()
-
- // Put segment delete log
- err := store.PutSegmentDL(ctx, "segment0", []byte("index0"))
- assert.Nil(t, err)
- err = store.PutSegmentDL(ctx, "segment1", []byte("index1"))
- assert.Nil(t, err)
-
- // Get segment delete log
- index, err := store.GetSegmentDL(ctx, "segment0")
- assert.Nil(t, err)
- assert.Equal(t, []byte("index0"), index)
- index, err = store.GetSegmentDL(ctx, "segment1")
- assert.Nil(t, err)
- assert.Equal(t, []byte("index1"), index)
-
- // Delete segment delete log
- err = store.DeleteSegmentDL(ctx, "segment0")
- assert.Nil(t, err)
- err = store.DeleteSegmentDL(ctx, "segment1")
- assert.Nil(t, err)
- index, err = store.GetSegmentDL(ctx, "segment0")
- assert.Nil(t, err)
- assert.Nil(t, index)
-}
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
deleted file mode 100644
index 67e9e44e8939075caaaa8aefa17cccc12e786bbd..0000000000000000000000000000000000000000
--- a/internal/storage/storage.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package storage
-
-import (
- "context"
- "errors"
-
- S3Driver "github.com/zilliztech/milvus-distributed/internal/storage/internal/S3"
- minIODriver "github.com/zilliztech/milvus-distributed/internal/storage/internal/minio"
- tikvDriver "github.com/zilliztech/milvus-distributed/internal/storage/internal/tikv"
- storagetype "github.com/zilliztech/milvus-distributed/internal/storage/type"
-)
-
-func NewStore(ctx context.Context, option storagetype.Option) (storagetype.Store, error) {
- var err error
- var store storagetype.Store
- switch option.Type {
- case storagetype.TIKVDriver:
- store, err = tikvDriver.NewTikvStore(ctx, option)
- if err != nil {
- panic(err.Error())
- }
- return store, nil
- case storagetype.MinIODriver:
- store, err = minIODriver.NewMinioDriver(ctx, option)
- if err != nil {
- //panic(err.Error())
- return nil, err
- }
- return store, nil
- case storagetype.S3DRIVER:
- store, err = S3Driver.NewS3Driver(ctx, option)
- if err != nil {
- //panic(err.Error())
- return nil, err
- }
- return store, nil
- }
- return nil, errors.New("unsupported driver")
-}
diff --git a/internal/storage/type/storagetype.go b/internal/storage/type/storagetype.go
deleted file mode 100644
index 9549a106e573e96589ec9f415379a0a5b7144c2b..0000000000000000000000000000000000000000
--- a/internal/storage/type/storagetype.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package storagetype
-
-import (
- "context"
-
- "github.com/zilliztech/milvus-distributed/internal/util/typeutil"
-)
-
-type Key = []byte
-type Value = []byte
-type Timestamp = typeutil.Timestamp
-type DriverType = string
-type SegmentIndex = []byte
-type SegmentDL = []byte
-
-type Option struct {
- Type DriverType
- TikvAddress string
- BucketName string
-}
-
-const (
- MinIODriver DriverType = "MinIO"
- TIKVDriver DriverType = "TIKV"
- S3DRIVER DriverType = "S3"
-)
-
-/*
-type Store interface {
- Get(ctx context.Context, key Key, timestamp Timestamp) (Value, error)
- BatchGet(ctx context.Context, keys [] Key, timestamp Timestamp) ([]Value, error)
- Set(ctx context.Context, key Key, v Value, timestamp Timestamp) error
- BatchSet(ctx context.Context, keys []Key, v []Value, timestamp Timestamp) error
- Delete(ctx context.Context, key Key, timestamp Timestamp) error
- BatchDelete(ctx context.Context, keys []Key, timestamp Timestamp) error
- Close() error
-}
-*/
-
-type storeEngine interface {
- Put(ctx context.Context, key Key, value Value) error
- Get(ctx context.Context, key Key) (Value, error)
- GetByPrefix(ctx context.Context, prefix Key, keyOnly bool) ([]Key, []Value, error)
- Scan(ctx context.Context, startKey Key, endKey Key, limit int, keyOnly bool) ([]Key, []Value, error)
- Delete(ctx context.Context, key Key) error
- DeleteByPrefix(ctx context.Context, prefix Key) error
- DeleteRange(ctx context.Context, keyStart Key, keyEnd Key) error
-}
-
-type Store interface {
- //put(ctx context.Context, key Key, value Value, timestamp Timestamp, suffix string) error
- //scanLE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error)
- //scanGE(ctx context.Context, key Key, timestamp Timestamp, keyOnly bool) ([]Timestamp, []Key, []Value, error)
- //deleteLE(ctx context.Context, key Key, timestamp Timestamp) error
- //deleteGE(ctx context.Context, key Key, timestamp Timestamp) error
- //deleteRange(ctx context.Context, key Key, start Timestamp, end Timestamp) error
-
- GetRow(ctx context.Context, key Key, timestamp Timestamp) (Value, error)
- GetRows(ctx context.Context, keys []Key, timestamps []Timestamp) ([]Value, error)
-
- PutRow(ctx context.Context, key Key, value Value, segment string, timestamp Timestamp) error
- PutRows(ctx context.Context, keys []Key, values []Value, segments []string, timestamps []Timestamp) error
-
- GetSegments(ctx context.Context, key Key, timestamp Timestamp) ([]string, error)
-
- DeleteRow(ctx context.Context, key Key, timestamp Timestamp) error
- DeleteRows(ctx context.Context, keys []Key, timestamps []Timestamp) error
-
- PutLog(ctx context.Context, key Key, value Value, timestamp Timestamp, channel int) error
- GetLog(ctx context.Context, start Timestamp, end Timestamp, channels []int) ([]Value, error)
-
- GetSegmentIndex(ctx context.Context, segment string) (SegmentIndex, error)
- PutSegmentIndex(ctx context.Context, segment string, index SegmentIndex) error
- DeleteSegmentIndex(ctx context.Context, segment string) error
-
- GetSegmentDL(ctx context.Context, segment string) (SegmentDL, error)
- PutSegmentDL(ctx context.Context, segment string, log SegmentDL) error
- DeleteSegmentDL(ctx context.Context, segment string) error
-}
diff --git a/internal/util/flowgraph/input_node.go b/internal/util/flowgraph/input_node.go
index 7c4271b23be5e31373966c3b64acfc395285916f..b7891040e8208894b2c66f07582ddc26adc3d59e 100644
--- a/internal/util/flowgraph/input_node.go
+++ b/internal/util/flowgraph/input_node.go
@@ -1,8 +1,12 @@
package flowgraph
import (
+ "fmt"
"log"
+ "github.com/opentracing/opentracing-go"
+ "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
+
"github.com/zilliztech/milvus-distributed/internal/msgstream"
)
@@ -25,11 +29,32 @@ func (inNode *InputNode) InStream() *msgstream.MsgStream {
}
// empty input and return one *Msg
-func (inNode *InputNode) Operate(in []*Msg) []*Msg {
+func (inNode *InputNode) Operate([]*Msg) []*Msg {
//fmt.Println("Do InputNode operation")
-
msgPack := (*inNode.inStream).Consume()
+ var childs []opentracing.Span
+ tracer := opentracing.GlobalTracer()
+ if tracer != nil && msgPack != nil {
+ for _, msg := range msgPack.Msgs {
+ if msg.Type() == internalpb.MsgType_kInsert || msg.Type() == internalpb.MsgType_kSearch {
+ var child opentracing.Span
+ ctx := msg.GetContext()
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan(fmt.Sprintf("through msg input node, start time = %d", msg.BeginTs()),
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan(fmt.Sprintf("through msg input node, start time = %d", msg.BeginTs()))
+ }
+ child.SetTag("hash keys", msg.HashKeys())
+ child.SetTag("start time", msg.BeginTs())
+ child.SetTag("end time", msg.EndTs())
+ msg.SetContext(opentracing.ContextWithSpan(ctx, child))
+ childs = append(childs, child)
+ }
+ }
+ }
+
// TODO: add status
if msgPack == nil {
log.Println("null msg pack")
@@ -42,6 +67,10 @@ func (inNode *InputNode) Operate(in []*Msg) []*Msg {
timestampMax: msgPack.EndTs,
}
+ for _, child := range childs {
+ child.Finish()
+ }
+
return []*Msg{&msgStreamMsg}
}
diff --git a/internal/util/flowgraph/message.go b/internal/util/flowgraph/message.go
index f02d2604cbee807f791a9877cf969e3047fdfb61..e5c01d7d4ef92872f38abc45713cd68f06f5a6ee 100644
--- a/internal/util/flowgraph/message.go
+++ b/internal/util/flowgraph/message.go
@@ -4,6 +4,7 @@ import "github.com/zilliztech/milvus-distributed/internal/msgstream"
type Msg interface {
TimeTick() Timestamp
+ DownStreamNodeIdx() int
}
type MsgStreamMsg struct {
diff --git a/internal/writenode/client/client.go b/internal/writenode/client/client.go
index a966ff8862e21d5c0d701d84eb5507be1881ef76..6c922e2e1ca66740060e768c1dfcfadc8729e9d9 100644
--- a/internal/writenode/client/client.go
+++ b/internal/writenode/client/client.go
@@ -48,7 +48,7 @@ type SegmentDescription struct {
CloseTime Timestamp
}
-func (c *Client) FlushSegment(segmentID UniqueID) error {
+func (c *Client) FlushSegment(segmentID UniqueID, collectionID UniqueID, partitionTag string, timestamp Timestamp) error {
baseMsg := msgstream.BaseMsg{
BeginTimestamp: 0,
EndTimestamp: 0,
@@ -56,9 +56,11 @@ func (c *Client) FlushSegment(segmentID UniqueID) error {
}
flushMsg := internalPb.FlushMsg{
- MsgType: internalPb.MsgType_kFlush,
- SegmentID: segmentID,
- Timestamp: Timestamp(0),
+ MsgType: internalPb.MsgType_kFlush,
+ SegmentID: segmentID,
+ CollectionID: collectionID,
+ PartitionTag: partitionTag,
+ Timestamp: timestamp,
}
fMsg := &msgstream.FlushMsg{
diff --git a/internal/writenode/collection.go b/internal/writenode/collection.go
new file mode 100644
index 0000000000000000000000000000000000000000..21d411d110b6648266471781bb3218e472245378
--- /dev/null
+++ b/internal/writenode/collection.go
@@ -0,0 +1,37 @@
+package writenode
+
+import (
+ "log"
+
+ "github.com/golang/protobuf/proto"
+ "github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
+)
+
+type Collection struct {
+ schema *schemapb.CollectionSchema
+ id UniqueID
+}
+
+func (c *Collection) Name() string {
+ return c.schema.Name
+}
+
+func (c *Collection) ID() UniqueID {
+ return c.id
+}
+
+func newCollection(collectionID UniqueID, schemaStr string) *Collection {
+
+ var schema schemapb.CollectionSchema
+ err := proto.UnmarshalText(schemaStr, &schema)
+ if err != nil {
+ log.Println(err)
+ return nil
+ }
+
+ var newCollection = &Collection{
+ schema: &schema,
+ id: collectionID,
+ }
+ return newCollection
+}
diff --git a/internal/writenode/collection_replica.go b/internal/writenode/collection_replica.go
new file mode 100644
index 0000000000000000000000000000000000000000..2310802c8f3419bb1f1a58c8f23040f506f2e852
--- /dev/null
+++ b/internal/writenode/collection_replica.go
@@ -0,0 +1,94 @@
+package writenode
+
+import (
+ "strconv"
+ "sync"
+
+ "github.com/zilliztech/milvus-distributed/internal/errors"
+)
+
+type collectionReplica interface {
+
+ // collection
+ getCollectionNum() int
+ addCollection(collectionID UniqueID, schemaBlob string) error
+ removeCollection(collectionID UniqueID) error
+ getCollectionByID(collectionID UniqueID) (*Collection, error)
+ getCollectionByName(collectionName string) (*Collection, error)
+ hasCollection(collectionID UniqueID) bool
+}
+
+type collectionReplicaImpl struct {
+ mu sync.RWMutex
+ collections []*Collection
+}
+
+//----------------------------------------------------------------------------------------------------- collection
+func (colReplica *collectionReplicaImpl) getCollectionNum() int {
+ colReplica.mu.RLock()
+ defer colReplica.mu.RUnlock()
+
+ return len(colReplica.collections)
+}
+
+func (colReplica *collectionReplicaImpl) addCollection(collectionID UniqueID, schemaBlob string) error {
+ colReplica.mu.Lock()
+ defer colReplica.mu.Unlock()
+
+ var newCollection = newCollection(collectionID, schemaBlob)
+ colReplica.collections = append(colReplica.collections, newCollection)
+
+ return nil
+}
+
+func (colReplica *collectionReplicaImpl) removeCollection(collectionID UniqueID) error {
+ colReplica.mu.Lock()
+ defer colReplica.mu.Unlock()
+
+ tmpCollections := make([]*Collection, 0)
+ for _, col := range colReplica.collections {
+ if col.ID() != collectionID {
+ tmpCollections = append(tmpCollections, col)
+ }
+ }
+ colReplica.collections = tmpCollections
+ return nil
+}
+
+func (colReplica *collectionReplicaImpl) getCollectionByID(collectionID UniqueID) (*Collection, error) {
+ colReplica.mu.RLock()
+ defer colReplica.mu.RUnlock()
+
+ for _, collection := range colReplica.collections {
+ if collection.ID() == collectionID {
+ return collection, nil
+ }
+ }
+
+ return nil, errors.New("cannot find collection, id = " + strconv.FormatInt(collectionID, 10))
+}
+
+func (colReplica *collectionReplicaImpl) getCollectionByName(collectionName string) (*Collection, error) {
+ colReplica.mu.RLock()
+ defer colReplica.mu.RUnlock()
+
+ for _, collection := range colReplica.collections {
+ if collection.Name() == collectionName {
+ return collection, nil
+ }
+ }
+
+ return nil, errors.New("Cannot found collection: " + collectionName)
+}
+
+func (colReplica *collectionReplicaImpl) hasCollection(collectionID UniqueID) bool {
+ colReplica.mu.RLock()
+ defer colReplica.mu.RUnlock()
+
+ for _, col := range colReplica.collections {
+ if col.ID() == collectionID {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/writenode/collection_replica_test.go b/internal/writenode/collection_replica_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..be17d2b8808a6de428289a38ec0c7fee159dc316
--- /dev/null
+++ b/internal/writenode/collection_replica_test.go
@@ -0,0 +1,153 @@
+package writenode
+
+import (
+ "testing"
+
+ "github.com/golang/protobuf/proto"
+ "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+ "github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
+ "github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func newReplica() collectionReplica {
+ collections := make([]*Collection, 0)
+
+ var replica collectionReplica = &collectionReplicaImpl{
+ collections: collections,
+ }
+ return replica
+}
+
+func genTestCollectionMeta(collectionName string, collectionID UniqueID) *etcdpb.CollectionMeta {
+ fieldVec := schemapb.FieldSchema{
+ FieldID: UniqueID(100),
+ Name: "vec",
+ IsPrimaryKey: false,
+ DataType: schemapb.DataType_VECTOR_FLOAT,
+ TypeParams: []*commonpb.KeyValuePair{
+ {
+ Key: "dim",
+ Value: "16",
+ },
+ },
+ IndexParams: []*commonpb.KeyValuePair{
+ {
+ Key: "metric_type",
+ Value: "L2",
+ },
+ },
+ }
+
+ fieldInt := schemapb.FieldSchema{
+ FieldID: UniqueID(101),
+ Name: "age",
+ IsPrimaryKey: false,
+ DataType: schemapb.DataType_INT32,
+ }
+
+ schema := schemapb.CollectionSchema{
+ Name: collectionName,
+ AutoID: true,
+ Fields: []*schemapb.FieldSchema{
+ &fieldVec, &fieldInt,
+ },
+ }
+
+ collectionMeta := etcdpb.CollectionMeta{
+ ID: collectionID,
+ Schema: &schema,
+ CreateTime: Timestamp(0),
+ SegmentIDs: []UniqueID{0},
+ PartitionTags: []string{"default"},
+ }
+
+ return &collectionMeta
+}
+
+func initTestMeta(t *testing.T, replica collectionReplica, collectionName string, collectionID UniqueID, segmentID UniqueID) {
+ collectionMeta := genTestCollectionMeta(collectionName, collectionID)
+
+ schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
+ assert.NotEqual(t, "", schemaBlob)
+
+ var err = replica.addCollection(collectionMeta.ID, schemaBlob)
+ assert.NoError(t, err)
+
+ collection, err := replica.getCollectionByName(collectionName)
+ assert.NoError(t, err)
+ assert.Equal(t, collection.Name(), collectionName)
+ assert.Equal(t, collection.ID(), collectionID)
+ assert.Equal(t, replica.getCollectionNum(), 1)
+
+}
+
+//----------------------------------------------------------------------------------------------------- collection
+func TestCollectionReplica_getCollectionNum(t *testing.T) {
+ replica := newReplica()
+ initTestMeta(t, replica, "collection0", 0, 0)
+ assert.Equal(t, replica.getCollectionNum(), 1)
+}
+
+func TestCollectionReplica_addCollection(t *testing.T) {
+ replica := newReplica()
+ initTestMeta(t, replica, "collection0", 0, 0)
+}
+
+func TestCollectionReplica_removeCollection(t *testing.T) {
+ replica := newReplica()
+ initTestMeta(t, replica, "collection0", 0, 0)
+ assert.Equal(t, replica.getCollectionNum(), 1)
+
+ err := replica.removeCollection(0)
+ assert.NoError(t, err)
+ assert.Equal(t, replica.getCollectionNum(), 0)
+}
+
+func TestCollectionReplica_getCollectionByID(t *testing.T) {
+ replica := newReplica()
+ collectionName := "collection0"
+ collectionID := UniqueID(0)
+ initTestMeta(t, replica, collectionName, collectionID, 0)
+ targetCollection, err := replica.getCollectionByID(collectionID)
+ assert.NoError(t, err)
+ assert.NotNil(t, targetCollection)
+ assert.Equal(t, targetCollection.Name(), collectionName)
+ assert.Equal(t, targetCollection.ID(), collectionID)
+}
+
+func TestCollectionReplica_getCollectionByName(t *testing.T) {
+ replica := newReplica()
+ collectionName := "collection0"
+ collectionID := UniqueID(0)
+ initTestMeta(t, replica, collectionName, collectionID, 0)
+
+ targetCollection, err := replica.getCollectionByName(collectionName)
+ assert.NoError(t, err)
+ assert.NotNil(t, targetCollection)
+ assert.Equal(t, targetCollection.Name(), collectionName)
+ assert.Equal(t, targetCollection.ID(), collectionID)
+
+}
+
+func TestCollectionReplica_hasCollection(t *testing.T) {
+ replica := newReplica()
+ collectionName := "collection0"
+ collectionID := UniqueID(0)
+ initTestMeta(t, replica, collectionName, collectionID, 0)
+
+ hasCollection := replica.hasCollection(collectionID)
+ assert.Equal(t, hasCollection, true)
+ hasCollection = replica.hasCollection(UniqueID(1))
+ assert.Equal(t, hasCollection, false)
+
+}
+
+func TestCollectionReplica_freeAll(t *testing.T) {
+ replica := newReplica()
+ collectionName := "collection0"
+ collectionID := UniqueID(0)
+ initTestMeta(t, replica, collectionName, collectionID, 0)
+
+}
diff --git a/internal/writenode/collection_test.go b/internal/writenode/collection_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..6df2ad5d4910276c79d756ad40364ab131721bee
--- /dev/null
+++ b/internal/writenode/collection_test.go
@@ -0,0 +1,34 @@
+package writenode
+
+import (
+ "testing"
+
+ "github.com/golang/protobuf/proto"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestCollection_newCollection(t *testing.T) {
+ collectionName := "collection0"
+ collectionID := UniqueID(0)
+ collectionMeta := genTestCollectionMeta(collectionName, collectionID)
+
+ schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
+ assert.NotEqual(t, "", schemaBlob)
+
+ collection := newCollection(collectionMeta.ID, schemaBlob)
+ assert.Equal(t, collection.Name(), collectionName)
+ assert.Equal(t, collection.ID(), collectionID)
+}
+
+func TestCollection_deleteCollection(t *testing.T) {
+ collectionName := "collection0"
+ collectionID := UniqueID(0)
+ collectionMeta := genTestCollectionMeta(collectionName, collectionID)
+
+ schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
+ assert.NotEqual(t, "", schemaBlob)
+
+ collection := newCollection(collectionMeta.ID, schemaBlob)
+ assert.Equal(t, collection.Name(), collectionName)
+ assert.Equal(t, collection.ID(), collectionID)
+}
diff --git a/internal/writenode/data_sync_service.go b/internal/writenode/data_sync_service.go
index da91d06a7640a0175963bc05443257cb591cc662..efcd62b0365dce4b6ca4e37588268471340c697b 100644
--- a/internal/writenode/data_sync_service.go
+++ b/internal/writenode/data_sync_service.go
@@ -12,16 +12,18 @@ type dataSyncService struct {
fg *flowgraph.TimeTickedFlowGraph
ddChan chan *ddlFlushSyncMsg
insertChan chan *insertFlushSyncMsg
+ replica collectionReplica
}
func newDataSyncService(ctx context.Context,
- ddChan chan *ddlFlushSyncMsg, insertChan chan *insertFlushSyncMsg) *dataSyncService {
+ ddChan chan *ddlFlushSyncMsg, insertChan chan *insertFlushSyncMsg, replica collectionReplica) *dataSyncService {
return &dataSyncService{
ctx: ctx,
fg: nil,
ddChan: ddChan,
insertChan: insertChan,
+ replica: replica,
}
}
@@ -46,8 +48,8 @@ func (dsService *dataSyncService) initNodes() {
var filterDmNode Node = newFilteredDmNode()
- var ddNode Node = newDDNode(dsService.ctx, dsService.ddChan)
- var insertBufferNode Node = newInsertBufferNode(dsService.ctx, dsService.insertChan)
+ var ddNode Node = newDDNode(dsService.ctx, dsService.ddChan, dsService.replica)
+ var insertBufferNode Node = newInsertBufferNode(dsService.ctx, dsService.insertChan, dsService.replica)
dsService.fg.AddNode(&dmStreamNode)
dsService.fg.AddNode(&ddStreamNode)
diff --git a/internal/writenode/data_sync_service_test.go b/internal/writenode/data_sync_service_test.go
index a37425ec3e283af2627283e5a3c877ec8420ab6a..df82cec4d93fa212db9432a624a2d2e2cefd134b 100644
--- a/internal/writenode/data_sync_service_test.go
+++ b/internal/writenode/data_sync_service_test.go
@@ -197,7 +197,8 @@ func TestDataSyncService_Start(t *testing.T) {
assert.NoError(t, err)
// dataSync
- node.dataSyncService = newDataSyncService(node.ctx, nil, nil)
+ replica := newReplica()
+ node.dataSyncService = newDataSyncService(node.ctx, nil, nil, replica)
go node.dataSyncService.start()
node.Close()
diff --git a/internal/writenode/flow_graph_dd_node.go b/internal/writenode/flow_graph_dd_node.go
index 7dd8e1fd433fda223cab267bbae39bb07ce2122c..9d3b4db158ed89f920a865ddd966ebe382cfc067 100644
--- a/internal/writenode/flow_graph_dd_node.go
+++ b/internal/writenode/flow_graph_dd_node.go
@@ -9,9 +9,6 @@ import (
"strconv"
"github.com/golang/protobuf/proto"
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
-
"github.com/zilliztech/milvus-distributed/internal/allocator"
"github.com/zilliztech/milvus-distributed/internal/kv"
miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
@@ -30,6 +27,7 @@ type ddNode struct {
idAllocator *allocator.IDAllocator
kv kv.Base
+ replica collectionReplica
}
type ddData struct {
@@ -228,6 +226,15 @@ func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
log.Println(err)
return
}
+
+ schemaStr := proto.MarshalTextString(&schema)
+ // add collection
+ err = ddNode.replica.addCollection(collectionID, schemaStr)
+ if err != nil {
+ log.Println(err)
+ return
+ }
+
collectionName := schema.Name
ddNode.ddMsg.collectionRecords[collectionName] = append(ddNode.ddMsg.collectionRecords[collectionName],
metaOperateRecord{
@@ -252,6 +259,11 @@ func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
func (ddNode *ddNode) dropCollection(msg *msgstream.DropCollectionMsg) {
collectionID := msg.CollectionID
+ err := ddNode.replica.removeCollection(collectionID)
+ if err != nil {
+ log.Println(err)
+ }
+
// remove collection
if _, ok := ddNode.ddRecords.collectionRecords[collectionID]; !ok {
err := errors.New("cannot found collection " + strconv.FormatInt(collectionID, 10))
@@ -347,7 +359,7 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
ddNode.ddBuffer.ddData[collectionID].eventTypes = append(ddNode.ddBuffer.ddData[collectionID].eventTypes, storage.DropPartitionEventType)
}
-func newDDNode(ctx context.Context, outCh chan *ddlFlushSyncMsg) *ddNode {
+func newDDNode(ctx context.Context, outCh chan *ddlFlushSyncMsg, replica collectionReplica) *ddNode {
maxQueueLength := Params.FlowGraphMaxQueueLength
maxParallelism := Params.FlowGraphMaxParallelism
@@ -360,19 +372,16 @@ func newDDNode(ctx context.Context, outCh chan *ddlFlushSyncMsg) *ddNode {
partitionRecords: make(map[UniqueID]interface{}),
}
- minIOEndPoint := Params.MinioAddress
- minIOAccessKeyID := Params.MinioAccessKeyID
- minIOSecretAccessKey := Params.MinioSecretAccessKey
- minIOUseSSL := Params.MinioUseSSL
- minIOClient, err := minio.New(minIOEndPoint, &minio.Options{
- Creds: credentials.NewStaticV4(minIOAccessKeyID, minIOSecretAccessKey, ""),
- Secure: minIOUseSSL,
- })
- if err != nil {
- panic(err)
- }
bucketName := Params.MinioBucketName
- minioKV, err := miniokv.NewMinIOKV(ctx, minIOClient, bucketName)
+ option := &miniokv.Option{
+ Address: Params.MinioAddress,
+ AccessKeyID: Params.MinioAccessKeyID,
+ SecretAccessKeyID: Params.MinioSecretAccessKey,
+ UseSSL: Params.MinioUseSSL,
+ BucketName: bucketName,
+ CreateBucket: true,
+ }
+ minioKV, err := miniokv.NewMinIOKV(ctx, option)
if err != nil {
panic(err)
}
@@ -397,5 +406,6 @@ func newDDNode(ctx context.Context, outCh chan *ddlFlushSyncMsg) *ddNode {
idAllocator: idAllocator,
kv: minioKV,
+ replica: replica,
}
}
diff --git a/internal/writenode/flow_graph_dd_node_test.go b/internal/writenode/flow_graph_dd_node_test.go
index 25de3697eae6eea564feabea3a41469417029de7..9b5d71ffb90af4e2fbb02f0b31e888d1eea4f4de 100644
--- a/internal/writenode/flow_graph_dd_node_test.go
+++ b/internal/writenode/flow_graph_dd_node_test.go
@@ -45,8 +45,8 @@ func TestFlowGraphDDNode_Operate(t *testing.T) {
go fService.start()
Params.FlushDdBufSize = 4
-
- ddNode := newDDNode(ctx, ddChan)
+ replica := newReplica()
+ ddNode := newDDNode(ctx, ddChan, replica)
colID := UniqueID(0)
colName := "col-test-0"
diff --git a/internal/writenode/flow_graph_filter_dm_node.go b/internal/writenode/flow_graph_filter_dm_node.go
index 98e9ec0bc4cc5fb596c93a2725ef2bd521da911e..0bca67ebcb4ec4723454c083331822f617275990 100644
--- a/internal/writenode/flow_graph_filter_dm_node.go
+++ b/internal/writenode/flow_graph_filter_dm_node.go
@@ -1,8 +1,11 @@
package writenode
import (
+ "context"
"log"
+ "github.com/opentracing/opentracing-go"
+
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
@@ -31,11 +34,34 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
// TODO: add error handling
}
+ var childs []opentracing.Span
+ tracer := opentracing.GlobalTracer()
+ if tracer != nil {
+ for _, msg := range msgStreamMsg.TsMessages() {
+ if msg.Type() == internalPb.MsgType_kInsert {
+ var child opentracing.Span
+ ctx := msg.GetContext()
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan("pass filter node",
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan("pass filter node")
+ }
+ child.SetTag("hash keys", msg.HashKeys())
+ child.SetTag("start time", msg.BeginTs())
+ child.SetTag("end time", msg.EndTs())
+ msg.SetContext(opentracing.ContextWithSpan(ctx, child))
+ childs = append(childs, child)
+ }
+ }
+ }
+
ddMsg, ok := (*in[1]).(*ddMsg)
if !ok {
log.Println("type assertion failed for ddMsg")
// TODO: add error handling
}
+
fdmNode.ddMsg = ddMsg
var iMsg = insertMsg{
@@ -56,11 +82,20 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
}
}
- for _, msg := range msgStreamMsg.TsMessages() {
+ for key, msg := range msgStreamMsg.TsMessages() {
switch msg.Type() {
case internalPb.MsgType_kInsert:
+ var ctx2 context.Context
+ if childs != nil {
+ if childs[key] != nil {
+ ctx2 = opentracing.ContextWithSpan(msg.GetContext(), childs[key])
+ } else {
+ ctx2 = context.Background()
+ }
+ }
resMsg := fdmNode.filterInvalidInsertMessage(msg.(*msgstream.InsertMsg))
if resMsg != nil {
+ resMsg.SetContext(ctx2)
iMsg.insertMessages = append(iMsg.insertMessages, resMsg)
}
// case internalPb.MsgType_kDelete:
@@ -69,8 +104,11 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
log.Println("Non supporting message type:", msg.Type())
}
}
-
var res Msg = &iMsg
+
+ for _, child := range childs {
+ child.Finish()
+ }
return []*Msg{&res}
}
diff --git a/internal/writenode/flow_graph_insert_buffer_node.go b/internal/writenode/flow_graph_insert_buffer_node.go
index 0c001ee55d802fc3de1316de49679ce99a986e76..189a25bc60a077fd6acc88adb0f885f96ecbba82 100644
--- a/internal/writenode/flow_graph_insert_buffer_node.go
+++ b/internal/writenode/flow_graph_insert_buffer_node.go
@@ -4,18 +4,17 @@ import (
"bytes"
"context"
"encoding/binary"
+ "fmt"
"log"
"path"
"strconv"
- "time"
"unsafe"
- "github.com/golang/protobuf/proto"
- "github.com/minio/minio-go/v7"
- "github.com/minio/minio-go/v7/pkg/credentials"
+ "github.com/opentracing/opentracing-go"
+ oplog "github.com/opentracing/opentracing-go/log"
+
"github.com/zilliztech/milvus-distributed/internal/allocator"
"github.com/zilliztech/milvus-distributed/internal/kv"
- etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
@@ -23,7 +22,6 @@ import (
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
"github.com/zilliztech/milvus-distributed/internal/storage"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
- "go.etcd.io/etcd/clientv3"
)
const (
@@ -37,13 +35,13 @@ type (
insertBufferNode struct {
BaseNode
- kvClient *etcdkv.EtcdKV
insertBuffer *insertBuffer
minIOKV kv.Base
minioPrifex string
idAllocator *allocator.IDAllocator
outCh chan *insertFlushSyncMsg
pulsarWriteNodeTimeTickStream *msgstream.PulsarMsgStream
+ replica collectionReplica
}
insertBuffer struct {
@@ -102,11 +100,23 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
// iMsg is insertMsg
// 1. iMsg -> buffer
for _, msg := range iMsg.insertMessages {
+ ctx := msg.GetContext()
+ var span opentracing.Span
+ if ctx != nil {
+ span, _ = opentracing.StartSpanFromContext(ctx, fmt.Sprintf("insert buffer node, start time = %d", msg.BeginTs()))
+ } else {
+ span = opentracing.StartSpan(fmt.Sprintf("insert buffer node, start time = %d", msg.BeginTs()))
+ }
+ span.SetTag("hash keys", msg.HashKeys())
+ span.SetTag("start time", msg.BeginTs())
+ span.SetTag("end time", msg.EndTs())
if len(msg.RowIDs) != len(msg.Timestamps) || len(msg.RowIDs) != len(msg.RowData) {
log.Println("Error: misaligned messages detected")
continue
}
currentSegID := msg.GetSegmentID()
+ collectionName := msg.GetCollectionName()
+ span.LogFields(oplog.Int("segment id", int(currentSegID)))
idata, ok := ibNode.insertBuffer.insertData[currentSegID]
if !ok {
@@ -115,17 +125,35 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
}
}
+ // Timestamps
+ _, ok = idata.Data[1].(*storage.Int64FieldData)
+ if !ok {
+ idata.Data[1] = &storage.Int64FieldData{
+ Data: []int64{},
+ NumRows: 0,
+ }
+ }
+ tsData := idata.Data[1].(*storage.Int64FieldData)
+ for _, ts := range msg.Timestamps {
+ tsData.Data = append(tsData.Data, int64(ts))
+ }
+ tsData.NumRows += len(msg.Timestamps)
+ span.LogFields(oplog.Int("tsData numRows", tsData.NumRows))
+
// 1.1 Get CollectionMeta from etcd
- segMeta, collMeta, err := ibNode.getMeta(currentSegID)
+ collection, err := ibNode.replica.getCollectionByName(collectionName)
+ //collSchema, err := ibNode.getCollectionSchemaByName(collectionName)
if err != nil {
// GOOSE TODO add error handler
log.Println("Get meta wrong:", err)
continue
}
+ collectionID := collection.ID()
+ collSchema := collection.schema
// 1.2 Get Fields
var pos int = 0 // Record position of blob
- for _, field := range collMeta.Schema.Fields {
+ for _, field := range collSchema.Fields {
switch field.DataType {
case schemapb.DataType_VECTOR_FLOAT:
var dim int
@@ -360,9 +388,11 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
// 1.3 store in buffer
ibNode.insertBuffer.insertData[currentSegID] = idata
+ span.LogFields(oplog.String("store in buffer", "store in buffer"))
// 1.4 if full
// 1.4.1 generate binlogs
+ span.LogFields(oplog.String("generate binlogs", "generate binlogs"))
if ibNode.insertBuffer.full(currentSegID) {
log.Printf(". Insert Buffer full, auto flushing (%v) rows of data...", ibNode.insertBuffer.size(currentSegID))
// partitionTag -> partitionID
@@ -372,7 +402,10 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
log.Println("partitionTag to partitionID wrong")
// TODO GOOSE add error handler
}
-
+ collMeta := &etcdpb.CollectionMeta{
+ Schema: collSchema,
+ ID: collectionID,
+ }
inCodec := storage.NewInsertCodec(collMeta)
// buffer data to binlogs
@@ -388,7 +421,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
log.Println(".. Clearing buffer")
// 1.5.2 binLogs -> minIO/S3
- collIDStr := strconv.FormatInt(segMeta.GetCollectionID(), 10)
+ collIDStr := strconv.FormatInt(collectionID, 10)
partitionIDStr := strconv.FormatInt(partitionID, 10)
segIDStr := strconv.FormatInt(currentSegID, 10)
keyPrefix := path.Join(ibNode.minioPrifex, collIDStr, partitionIDStr, segIDStr)
@@ -428,6 +461,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
ibNode.outCh <- inBinlogMsg
}
}
+ span.Finish()
}
if len(iMsg.insertMessages) > 0 {
@@ -447,20 +481,24 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
for _, msg := range iMsg.flushMessages {
currentSegID := msg.GetSegmentID()
flushTs := msg.GetTimestamp()
-
+ partitionTag := msg.GetPartitionTag()
+ collectionID := msg.GetCollectionID()
log.Printf(". Receiving flush message segID(%v)...", currentSegID)
if ibNode.insertBuffer.size(currentSegID) > 0 {
log.Println(".. Buffer not empty, flushing ...")
- segMeta, collMeta, err := ibNode.getMeta(currentSegID)
+ collSchema, err := ibNode.getCollectionSchemaByID(collectionID)
if err != nil {
// GOOSE TODO add error handler
log.Println("Get meta wrong: ", err)
}
+ collMeta := &etcdpb.CollectionMeta{
+ Schema: collSchema,
+ ID: collectionID,
+ }
inCodec := storage.NewInsertCodec(collMeta)
// partitionTag -> partitionID
- partitionTag := segMeta.GetPartitionTag()
partitionID, err := typeutil.Hash32String(partitionTag)
if err != nil {
// GOOSE TODO add error handler
@@ -478,7 +516,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
delete(ibNode.insertBuffer.insertData, currentSegID)
// binLogs -> minIO/S3
- collIDStr := strconv.FormatInt(segMeta.GetCollectionID(), 10)
+ collIDStr := strconv.FormatInt(collectionID, 10)
partitionIDStr := strconv.FormatInt(partitionID, 10)
segIDStr := strconv.FormatInt(currentSegID, 10)
keyPrefix := path.Join(ibNode.minioPrifex, collIDStr, partitionIDStr, segIDStr)
@@ -537,31 +575,20 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
return nil
}
-func (ibNode *insertBufferNode) getMeta(segID UniqueID) (*etcdpb.SegmentMeta, *etcdpb.CollectionMeta, error) {
-
- segMeta := &etcdpb.SegmentMeta{}
-
- key := path.Join(SegmentPrefix, strconv.FormatInt(segID, 10))
- value, err := ibNode.kvClient.Load(key)
+func (ibNode *insertBufferNode) getCollectionSchemaByID(collectionID UniqueID) (*schemapb.CollectionSchema, error) {
+ ret, err := ibNode.replica.getCollectionByID(collectionID)
if err != nil {
- return nil, nil, err
- }
- err = proto.UnmarshalText(value, segMeta)
- if err != nil {
- return nil, nil, err
+ return nil, err
}
+ return ret.schema, nil
+}
- collMeta := &etcdpb.CollectionMeta{}
- key = path.Join(CollectionPrefix, strconv.FormatInt(segMeta.GetCollectionID(), 10))
- value, err = ibNode.kvClient.Load(key)
+func (ibNode *insertBufferNode) getCollectionSchemaByName(collectionName string) (*schemapb.CollectionSchema, error) {
+ ret, err := ibNode.replica.getCollectionByName(collectionName)
if err != nil {
- return nil, nil, err
+ return nil, err
}
- err = proto.UnmarshalText(value, collMeta)
- if err != nil {
- return nil, nil, err
- }
- return segMeta, collMeta, nil
+ return ret.schema, nil
}
func (ibNode *insertBufferNode) writeHardTimeTick(ts Timestamp) error {
@@ -582,7 +609,7 @@ func (ibNode *insertBufferNode) writeHardTimeTick(ts Timestamp) error {
return ibNode.pulsarWriteNodeTimeTickStream.Produce(&msgPack)
}
-func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *insertBufferNode {
+func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg, replica collectionReplica) *insertBufferNode {
maxQueueLength := Params.FlowGraphMaxQueueLength
maxParallelism := Params.FlowGraphMaxParallelism
@@ -596,34 +623,18 @@ func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *i
maxSize: maxSize,
}
- // EtcdKV
- ETCDAddr := Params.EtcdAddress
- MetaRootPath := Params.MetaRootPath
- log.Println("metaRootPath: ", MetaRootPath)
- cli, err := clientv3.New(clientv3.Config{
- Endpoints: []string{ETCDAddr},
- DialTimeout: 5 * time.Second,
- })
- if err != nil {
- panic(err)
- }
- kvClient := etcdkv.NewEtcdKV(cli, MetaRootPath)
-
// MinIO
- minioendPoint := Params.MinioAddress
- miniioAccessKeyID := Params.MinioAccessKeyID
- miniioSecretAccessKey := Params.MinioSecretAccessKey
- minioUseSSL := Params.MinioUseSSL
- minioBucketName := Params.MinioBucketName
-
- minioClient, err := minio.New(minioendPoint, &minio.Options{
- Creds: credentials.NewStaticV4(miniioAccessKeyID, miniioSecretAccessKey, ""),
- Secure: minioUseSSL,
- })
- if err != nil {
- panic(err)
+
+ option := &miniokv.Option{
+ Address: Params.MinioAddress,
+ AccessKeyID: Params.MinioAccessKeyID,
+ SecretAccessKeyID: Params.MinioSecretAccessKey,
+ UseSSL: Params.MinioUseSSL,
+ CreateBucket: true,
+ BucketName: Params.MinioBucketName,
}
- minIOKV, err := miniokv.NewMinIOKV(ctx, minioClient, minioBucketName)
+
+ minIOKV, err := miniokv.NewMinIOKV(ctx, option)
if err != nil {
panic(err)
}
@@ -644,12 +655,12 @@ func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *i
return &insertBufferNode{
BaseNode: baseNode,
- kvClient: kvClient,
insertBuffer: iBuffer,
minIOKV: minIOKV,
minioPrifex: minioPrefix,
idAllocator: idAllocator,
outCh: outCh,
pulsarWriteNodeTimeTickStream: wTt,
+ replica: replica,
}
}
diff --git a/internal/writenode/flow_graph_insert_buffer_node_test.go b/internal/writenode/flow_graph_insert_buffer_node_test.go
index ef268dd3ba9c84e0895c0c2264cbb58f72d087f2..567b90c34f29345afca77a45bbed8d9c59843aa0 100644
--- a/internal/writenode/flow_graph_insert_buffer_node_test.go
+++ b/internal/writenode/flow_graph_insert_buffer_node_test.go
@@ -47,7 +47,8 @@ func TestFlowGraphInputBufferNode_Operate(t *testing.T) {
go fService.start()
// Params.FlushInsertBufSize = 2
- iBNode := newInsertBufferNode(ctx, insertChan)
+ replica := newReplica()
+ iBNode := newInsertBufferNode(ctx, insertChan, replica)
newMeta()
inMsg := genInsertMsg()
diff --git a/internal/writenode/flow_graph_message.go b/internal/writenode/flow_graph_message.go
index 5825ef570e97425cbdda95ef44cda20576be9740..147822e7c5035b8574817dc5911c845727a92078 100644
--- a/internal/writenode/flow_graph_message.go
+++ b/internal/writenode/flow_graph_message.go
@@ -46,14 +46,30 @@ func (ksMsg *key2SegMsg) TimeTick() Timestamp {
return ksMsg.timeRange.timestampMax
}
+func (ksMsg *key2SegMsg) DownStreamNodeIdx() int {
+ return 0
+}
+
func (suMsg *ddMsg) TimeTick() Timestamp {
return suMsg.timeRange.timestampMax
}
+func (suMsg *ddMsg) DownStreamNodeIdx() int {
+ return 0
+}
+
func (iMsg *insertMsg) TimeTick() Timestamp {
return iMsg.timeRange.timestampMax
}
+func (iMsg *insertMsg) DownStreamNodeIdx() int {
+ return 0
+}
+
func (dMsg *deleteMsg) TimeTick() Timestamp {
return dMsg.timeRange.timestampMax
}
+
+func (dMsg *deleteMsg) DownStreamNodeIdx() int {
+ return 0
+}
diff --git a/internal/writenode/meta_service.go b/internal/writenode/meta_service.go
new file mode 100644
index 0000000000000000000000000000000000000000..75dfb4a4027c4f9f5e0466dc2110a059ec1f3f4d
--- /dev/null
+++ b/internal/writenode/meta_service.go
@@ -0,0 +1,135 @@
+package writenode
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "path"
+ "reflect"
+ "strings"
+ "time"
+
+ "github.com/golang/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
+
+ etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
+ "github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
+)
+
+type metaService struct {
+ ctx context.Context
+ kvBase *etcdkv.EtcdKV
+ replica collectionReplica
+}
+
+func newMetaService(ctx context.Context, replica collectionReplica) *metaService {
+ ETCDAddr := Params.EtcdAddress
+ MetaRootPath := Params.MetaRootPath
+
+ cli, _ := clientv3.New(clientv3.Config{
+ Endpoints: []string{ETCDAddr},
+ DialTimeout: 5 * time.Second,
+ })
+
+ return &metaService{
+ ctx: ctx,
+ kvBase: etcdkv.NewEtcdKV(cli, MetaRootPath),
+ replica: replica,
+ }
+}
+
+func (mService *metaService) start() {
+ // init from meta
+ err := mService.loadCollections()
+ if err != nil {
+ log.Fatal("metaService loadCollections failed")
+ }
+}
+
+func GetCollectionObjID(key string) string {
+ ETCDRootPath := Params.MetaRootPath
+
+ prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/"
+ return strings.TrimPrefix(key, prefix)
+}
+
+func isCollectionObj(key string) bool {
+ ETCDRootPath := Params.MetaRootPath
+
+ prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/"
+ prefix = strings.TrimSpace(prefix)
+ index := strings.Index(key, prefix)
+
+ return index == 0
+}
+
+func isSegmentObj(key string) bool {
+ ETCDRootPath := Params.MetaRootPath
+
+ prefix := path.Join(ETCDRootPath, SegmentPrefix) + "/"
+ prefix = strings.TrimSpace(prefix)
+ index := strings.Index(key, prefix)
+
+ return index == 0
+}
+
+func printCollectionStruct(obj *etcdpb.CollectionMeta) {
+ v := reflect.ValueOf(obj)
+ v = reflect.Indirect(v)
+ typeOfS := v.Type()
+
+ for i := 0; i < v.NumField(); i++ {
+ if typeOfS.Field(i).Name == "GrpcMarshalString" {
+ continue
+ }
+ fmt.Printf("Field: %s\tValue: %v\n", typeOfS.Field(i).Name, v.Field(i).Interface())
+ }
+}
+
+func (mService *metaService) processCollectionCreate(id string, value string) {
+ //println(fmt.Sprintf("Create Collection:$%s$", id))
+
+ col := mService.collectionUnmarshal(value)
+ if col != nil {
+ schema := col.Schema
+ schemaBlob := proto.MarshalTextString(schema)
+ err := mService.replica.addCollection(col.ID, schemaBlob)
+ if err != nil {
+ log.Println(err)
+ }
+ }
+}
+
+func (mService *metaService) loadCollections() error {
+ keys, values, err := mService.kvBase.LoadWithPrefix(CollectionPrefix)
+ if err != nil {
+ return err
+ }
+
+ for i := range keys {
+ objID := GetCollectionObjID(keys[i])
+ mService.processCollectionCreate(objID, values[i])
+ }
+
+ return nil
+}
+
+//----------------------------------------------------------------------- Unmarshal and Marshal
+func (mService *metaService) collectionUnmarshal(value string) *etcdpb.CollectionMeta {
+ col := etcdpb.CollectionMeta{}
+ err := proto.UnmarshalText(value, &col)
+ if err != nil {
+ log.Println(err)
+ return nil
+ }
+ return &col
+}
+
+func (mService *metaService) collectionMarshal(col *etcdpb.CollectionMeta) string {
+ value := proto.MarshalTextString(col)
+ if value == "" {
+ log.Println("marshal collection failed")
+ return ""
+ }
+ return value
+}
diff --git a/internal/writenode/meta_service_test.go b/internal/writenode/meta_service_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..b1e8a9e038c8647c140d86aae45dc1810a1c1ebd
--- /dev/null
+++ b/internal/writenode/meta_service_test.go
@@ -0,0 +1,100 @@
+package writenode
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMetaService_start(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ replica := newReplica()
+
+ metaService := newMetaService(ctx, replica)
+
+ metaService.start()
+}
+
+func TestMetaService_getCollectionObjId(t *testing.T) {
+ var key = "/collection/collection0"
+ var collectionObjID1 = GetCollectionObjID(key)
+
+ assert.Equal(t, collectionObjID1, "/collection/collection0")
+
+ key = "fakeKey"
+ var collectionObjID2 = GetCollectionObjID(key)
+
+ assert.Equal(t, collectionObjID2, "fakeKey")
+}
+
+func TestMetaService_isCollectionObj(t *testing.T) {
+ var key = Params.MetaRootPath + "/collection/collection0"
+ var b1 = isCollectionObj(key)
+
+ assert.Equal(t, b1, true)
+
+ key = Params.MetaRootPath + "/segment/segment0"
+ var b2 = isCollectionObj(key)
+
+ assert.Equal(t, b2, false)
+}
+
+func TestMetaService_processCollectionCreate(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ replica := newReplica()
+ metaService := newMetaService(ctx, replica)
+ defer cancel()
+ id := "0"
+ value := `schema: <
+ name: "test"
+ fields: <
+ fieldID:100
+ name: "vec"
+ data_type: VECTOR_FLOAT
+ type_params: <
+ key: "dim"
+ value: "16"
+ >
+ index_params: <
+ key: "metric_type"
+ value: "L2"
+ >
+ >
+ fields: <
+ fieldID:101
+ name: "age"
+ data_type: INT32
+ type_params: <
+ key: "dim"
+ value: "1"
+ >
+ >
+ >
+ segmentIDs: 0
+ partition_tags: "default"
+ `
+
+ metaService.processCollectionCreate(id, value)
+
+ collectionNum := replica.getCollectionNum()
+ assert.Equal(t, collectionNum, 1)
+
+ collection, err := replica.getCollectionByName("test")
+ assert.NoError(t, err)
+ assert.Equal(t, collection.ID(), UniqueID(0))
+}
+
+func TestMetaService_loadCollections(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ replica := newReplica()
+
+ metaService := newMetaService(ctx, replica)
+
+ err2 := (*metaService).loadCollections()
+ assert.Nil(t, err2)
+}
diff --git a/internal/writenode/write_node.go b/internal/writenode/write_node.go
index b747bd397be62c5dd08ca7ac42322bb7e1e8e962..319c13e4593c580af568833f3ae24143af4e8878 100644
--- a/internal/writenode/write_node.go
+++ b/internal/writenode/write_node.go
@@ -2,6 +2,12 @@ package writenode
import (
"context"
+ "fmt"
+ "io"
+
+ "github.com/opentracing/opentracing-go"
+ "github.com/uber/jaeger-client-go"
+ "github.com/uber/jaeger-client-go/config"
)
type WriteNode struct {
@@ -9,15 +15,27 @@ type WriteNode struct {
WriteNodeID uint64
dataSyncService *dataSyncService
flushSyncService *flushSyncService
+ metaService *metaService
+ replica collectionReplica
+ tracer opentracing.Tracer
+ closer io.Closer
}
func NewWriteNode(ctx context.Context, writeNodeID uint64) *WriteNode {
+ collections := make([]*Collection, 0)
+
+ var replica collectionReplica = &collectionReplicaImpl{
+ collections: collections,
+ }
+
node := &WriteNode{
ctx: ctx,
WriteNodeID: writeNodeID,
dataSyncService: nil,
flushSyncService: nil,
+ metaService: nil,
+ replica: replica,
}
return node
@@ -28,6 +46,22 @@ func Init() {
}
func (node *WriteNode) Start() error {
+ cfg := &config.Configuration{
+ ServiceName: "tracing",
+ Sampler: &config.SamplerConfig{
+ Type: "const",
+ Param: 1,
+ },
+ Reporter: &config.ReporterConfig{
+ LogSpans: true,
+ },
+ }
+ var err error
+ node.tracer, node.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
+ if err != nil {
+ panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
+ }
+ opentracing.SetGlobalTracer(node.tracer)
// TODO GOOSE Init Size??
chanSize := 100
@@ -35,10 +69,12 @@ func (node *WriteNode) Start() error {
insertChan := make(chan *insertFlushSyncMsg, chanSize)
node.flushSyncService = newFlushSyncService(node.ctx, ddChan, insertChan)
- node.dataSyncService = newDataSyncService(node.ctx, ddChan, insertChan)
+ node.dataSyncService = newDataSyncService(node.ctx, ddChan, insertChan, node.replica)
+ node.metaService = newMetaService(node.ctx, node.replica)
go node.dataSyncService.start()
go node.flushSyncService.start()
+ node.metaService.start()
return nil
}
diff --git a/tools/core_gen/all_generate.py b/tools/core_gen/all_generate.py
index 499022d583bf44465868530a4ddee5aec991523b..a4be00cead810a06504156c058b1755086ac1899 100755
--- a/tools/core_gen/all_generate.py
+++ b/tools/core_gen/all_generate.py
@@ -58,6 +58,10 @@ if __name__ == "__main__":
'visitor_name': "ExecExprVisitor",
"parameter_name": 'expr',
},
+ {
+ 'visitor_name': "VerifyExprVisitor",
+ "parameter_name": 'expr',
+ },
],
'PlanNode': [
{
@@ -68,7 +72,10 @@ if __name__ == "__main__":
'visitor_name': "ExecPlanNodeVisitor",
"parameter_name": 'node',
},
-
+ {
+ 'visitor_name': "VerifyPlanNodeVisitor",
+ "parameter_name": 'node',
+ },
]
}
extract_extra_body(visitor_info, query_path)
diff --git a/tools/core_gen/templates/visitor_derived.h b/tools/core_gen/templates/visitor_derived.h
index cda1ea1a74c15cb90ca13fe26b879bdf1cd021cc..5fe2be775ddaebaa817d4819413898d71d4468a1 100644
--- a/tools/core_gen/templates/visitor_derived.h
+++ b/tools/core_gen/templates/visitor_derived.h
@@ -13,7 +13,7 @@
#include "@@base_visitor@@.h"
namespace @@namespace@@ {
-class @@visitor_name@@ : @@base_visitor@@ {
+class @@visitor_name@@ : public @@base_visitor@@ {
public:
@@body@@