diff --git a/.devcontainer.json b/.devcontainer.json
index a7368ab60cbd5825cf39f67cb0acdc358fa3af48..fff529de85e22edb4389a9a156e547576f704c11 100644
--- a/.devcontainer.json
+++ b/.devcontainer.json
@@ -2,7 +2,7 @@
"name": "Milvus Distributed Dev Container Definition",
"dockerComposeFile": ["./docker-compose-vscode.yml"],
"service": "ubuntu",
- "initializeCommand": "scripts/init_devcontainer.sh && docker-compose -f docker-compose-vscode.yml down || true",
+ "initializeCommand": "scripts/init_devcontainer.sh && docker-compose -f docker-compose-vscode.yml down || true && docker-compose -f docker-compose-vscode.yml pull --ignore-pull-failures ubuntu",
"workspaceFolder": "/go/src/github.com/zilliztech/milvus-distributed",
"shutdownAction": "stopCompose",
"extensions": [
diff --git a/.gitignore b/.gitignore
index 082a135a387080bdfe8dbaf482674e004875ce07..9c71802f8f82d8a89de42b91fa00f6de8f4e8c0c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
**/cmake-build-release/*
internal/core/output/*
internal/core/build/*
+internal/kv/rocksdb/cwrapper/output/*
**/.idea/*
pulsar/client-cpp/build/
pulsar/client-cpp/build/*
@@ -11,7 +12,7 @@ pulsar/client-cpp/build/*
# vscode generated files
.vscode
docker-compose-vscode.yml
-docker-compose-vscode.yml.tmp
+docker-compose-vscode.yml.bak
cmake-build-debug
cmake-build-release
diff --git a/.jenkins/modules/Build/Build.groovy b/.jenkins/modules/Build/Build.groovy
index 1d35fd71ffd92f1cd8317688aa957eeb61092e26..14fd0b9cdd619f108cc65f966543df9df6272716 100644
--- a/.jenkins/modules/Build/Build.groovy
+++ b/.jenkins/modules/Build/Build.groovy
@@ -1,20 +1,22 @@
timeout(time: 20, unit: 'MINUTES') {
-
- sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Ccache artfactory files not found!\"'
- sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Go cache artfactory files not found!\"'
- sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz || echo \"Thirdparty artfactory files not found!\"'
- sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-\$(md5sum go.mod).tar.gz || echo \"Go mod artfactory files not found!\"'
+ dir ("scripts") {
+ sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Ccache artfactory files not found!\"'
+ sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Go cache artfactory files not found!\"'
+ sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz || echo \"Thirdparty artfactory files not found!\"'
+ sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-cache.tar.gz || echo \"Go mod artfactory files not found!\"'
+ }
// Zero the cache statistics (but not the configuration options)
sh 'ccache -z'
sh '. ./scripts/before-install.sh && make install'
sh 'echo -e "===\n=== ccache statistics after build\n===" && ccache --show-stats'
-
- withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
- sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
- sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
- sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz -u ${USERNAME} -p ${PASSWORD}'
- sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-\$(md5sum go.mod).tar.gz -u ${USERNAME} -p ${PASSWORD}'
+ dir ("scripts") {
+ withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+ sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
+ sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
+ sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz -u ${USERNAME} -p ${PASSWORD}'
+ sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-cache.tar.gz -u ${USERNAME} -p ${PASSWORD}'
+ }
}
}
diff --git a/.jenkins/modules/Regression/PythonRegression.groovy b/.jenkins/modules/Regression/PythonRegression.groovy
index 89a16f22eaad6dfdeb17c106af2d367aaac908d2..f726d92da14945da8b90bbf5ceb8e546860b4ad2 100644
--- a/.jenkins/modules/Regression/PythonRegression.groovy
+++ b/.jenkins/modules/Regression/PythonRegression.groovy
@@ -31,14 +31,12 @@ try {
} catch(exc) {
throw exc
} finally {
- dir ('build/docker/deploy') {
- sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} ps | tail -n +3 | awk \'{ print $1 }\' | ( while read arg; do docker logs -t $arg > $arg.log 2>&1; done )'
- archiveArtifacts artifacts: "**.log", allowEmptyArchive: true
- sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} down --rmi all -v || true'
- }
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v pulsar'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v etcd'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v minio'
+ dir ('build/docker/deploy') {
+ sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} down --rmi all -v || true'
+ }
dir ('build/docker/test') {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run --rm regression /bin/bash -c "rm -rf __pycache__ && rm -rf .pytest_cache"'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} down --rmi all -v || true'
diff --git a/Makefile b/Makefile
index 07356e4d578c7eea0758571c6d65da84e0065808..4d8a81973f8b43fbe9f65380cae1bedbc54f2073 100644
--- a/Makefile
+++ b/Makefile
@@ -57,8 +57,14 @@ lint:tools/bin/revive
@echo "Running $@ check"
@tools/bin/revive -formatter friendly -config tools/check/revive.toml ./...
+get-rocksdb:
+ @go env -w CGO_CFLAGS="-I$(PWD)/internal/kv/rocksdb/cwrapper/output/include"
+ @go env -w CGO_LDFLAGS="-L$(PWD)/internal/kv/rocksdb/cwrapper/output/lib -l:librocksdb.a -lstdc++ -lm -lz"
+ @(env bash $(PWD)/internal/kv/rocksdb/cwrapper/build.sh)
+ @go get github.com/tecbot/gorocksdb
+
#TODO: Check code specifications by golangci-lint
-static-check:
+static-check:get-rocksdb
@echo "Running $@ check"
@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/...
@@ -106,7 +112,7 @@ build-cpp-with-unittest:
unittest: test-cpp test-go
#TODO: proxy master query node writer's unittest
-test-go:
+test-go:get-rocksdb
@echo "Running go unittests..."
@(env bash $(PWD)/scripts/run_go_unittest.sh)
@@ -127,6 +133,7 @@ install: all
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/master $(GOPATH)/bin/master
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/proxy $(GOPATH)/bin/proxy
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/writenode $(GOPATH)/bin/writenode
+ @mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/indexbuilder $(GOPATH)/bin/indexbuilder
@mkdir -p $(LIBRARY_PATH) && cp -f $(PWD)/internal/core/output/lib/* $(LIBRARY_PATH)
@echo "Installation successful."
@@ -134,7 +141,10 @@ clean:
@echo "Cleaning up all the generated files"
@find . -name '*.test' | xargs rm -fv
@find . -name '*~' | xargs rm -fv
- @rm -rvf querynode
- @rm -rvf master
- @rm -rvf proxy
- @rm -rvf writenode
+ @rm -rf bin/
+ @rm -rf lib/
+ @rm -rf $(GOPATH)/bin/master
+ @rm -rf $(GOPATH)/bin/proxy
+ @rm -rf $(GOPATH)/bin/querynode
+ @rm -rf $(GOPATH)/bin/writenode
+ @rm -rf $(GOPATH)/bin/indexbuilder
diff --git a/build/docker/deploy/.env b/build/docker/deploy/.env
index 5ce139901f79681aadadd359fc94b2f811b86de8..fe63196a782ecb9306dadc8667bba14cb49a91c0 100644
--- a/build/docker/deploy/.env
+++ b/build/docker/deploy/.env
@@ -6,4 +6,4 @@ PULSAR_ADDRESS=pulsar://pulsar:6650
ETCD_ADDRESS=etcd:2379
MASTER_ADDRESS=master:53100
MINIO_ADDRESS=minio:9000
-INDEX_BUILDER_ADDRESS=indexbuilder:31000
+INDEX_BUILDER_ADDRESS=indexbuider:31000
diff --git a/build/docker/deploy/indexbuilder/DockerFile b/build/docker/deploy/indexbuilder/DockerFile
index 26804b1e4bea655e60af58f98247de599e496a00..4d40f27a981265ebcdc156941f1d82991657e6c0 100644
--- a/build/docker/deploy/indexbuilder/DockerFile
+++ b/build/docker/deploy/indexbuilder/DockerFile
@@ -9,31 +9,12 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
-FROM milvusdb/milvus-distributed-dev:amd64-ubuntu18.04-latest AS openblas
-
-#FROM alpine
-FROM ubuntu:bionic-20200921
-
-RUN apt-get update && apt-get install -y --no-install-recommends libtbb-dev gfortran
-
-#RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
-
-#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories \
-# && apk add --no-cache libtbb gfortran
-
-COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/
-
-RUN ln -s /usr/lib/libopenblas-r0.3.9.so /usr/lib/libopenblas.so.0 && \
- ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so
+FROM alpine:3.12.1
COPY ./bin/indexbuilder /milvus-distributed/bin/indexbuilder
COPY ./configs/ /milvus-distributed/configs/
-COPY ./lib/ /milvus-distributed/lib/
-
-ENV LD_LIBRARY_PATH=/milvus-distributed/lib:$LD_LIBRARY_PATH:/usr/lib
-
WORKDIR /milvus-distributed/
CMD ["./bin/indexbuilder"]
diff --git a/deployments/docker/docker-compose.yml b/deployments/docker/docker-compose.yml
index 60bf5d9fff1f2a0aa47274becc742f778aaeb77a..0ae708a19ecb9ababafe5fcdb6bd5f9d5eac529e 100644
--- a/deployments/docker/docker-compose.yml
+++ b/deployments/docker/docker-compose.yml
@@ -36,6 +36,14 @@ services:
networks:
- milvus
+ jaeger:
+ image: jaegertracing/all-in-one:latest
+ ports:
+ - "6831:6831/udp"
+ - "16686:16686"
+ networks:
+ - milvus
+
networks:
milvus:
diff --git a/docker-compose.yml b/docker-compose.yml
index d78e40515124ab50ed6fac5d1958d2a1bd8cfbb7..4f7f32bcd7030a3d7826f101f67095e26c3e55a7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -86,5 +86,10 @@ services:
networks:
- milvus
+ jaeger:
+ image: jaegertracing/all-in-one:latest
+ networks:
+ - milvus
+
networks:
milvus:
diff --git a/docs/developer_guides/chap03_index_builder.md b/docs/developer_guides/chap03_index_service.md
similarity index 50%
rename from docs/developer_guides/chap03_index_builder.md
rename to docs/developer_guides/chap03_index_service.md
index a3e7495330385a98ba41070f65ed5e0dc716462b..22f3f6671581768e125a097732c135b6284ae21f 100644
--- a/docs/developer_guides/chap03_index_builder.md
+++ b/docs/developer_guides/chap03_index_service.md
@@ -13,8 +13,8 @@
```go
type Client interface {
BuildIndex(req BuildIndexRequest) (BuildIndexResponse, error)
- DescribeIndex(indexID UniqueID) (IndexDescription, error)
- GetIndexFilePaths(indexID UniqueID) (IndexFilePaths, error)
+ GetIndexStates(req IndexStatesRequest) (IndexStatesResponse, error)
+ GetIndexFilePaths(req IndexFilePathRequest) (IndexFilePathsResponse, error)
}
```
@@ -36,19 +36,23 @@ type BuildIndexResponse struct {
-* *DescribeIndex*
+* *GetIndexStates*
```go
-enum IndexStatus {
+type IndexStatesRequest struct {
+ IndexID UniqueID
+}
+
+enum IndexState {
NONE = 0;
UNISSUED = 1;
INPROGRESS = 2;
FINISHED = 3;
}
-type IndexDescription struct {
+type IndexStatesResponse struct {
ID UniqueID
- Status IndexStatus
+ State IndexState
EnqueueTime time.Time
ScheduleTime time.Time
BuildCompleteTime time.Time
@@ -60,8 +64,28 @@ type IndexDescription struct {
* *GetIndexFilePaths*
```go
-type IndexFilePaths struct {
+type IndexFilePathRequest struct {
+ IndexID UniqueID
+}
+
+type IndexFilePathsResponse struct {
FilePaths []string
}
```
+
+
+#### 8.3 Index Node
+
+```go
+type IndexNode interface {
+ Start() error
+ Close() error
+
+// SetTimeTickChannel(channelID string) error
+ SetStatsChannel(channelID string) error
+
+ BuildIndex(req BuildIndexRequest) (BuildIndexResponse, error)
+}
+```
+
diff --git a/docs/developer_guides/chap04_message_stream.md b/docs/developer_guides/chap04_message_stream.md
index 19fefbd34410b731084e32c05129292e37905985..7eee9aa6d23cda3d41eb27314178f9807fb523b7 100644
--- a/docs/developer_guides/chap04_message_stream.md
+++ b/docs/developer_guides/chap04_message_stream.md
@@ -1,5 +1,3 @@
-
-
## 8. Message Stream Service
@@ -8,13 +6,13 @@
-#### 8.2 API
+#### 8.2 Message Stream Service API
```go
type Client interface {
- CreateChannels(req CreateChannelRequest) (ChannelID []string, error)
- DestoryChannels(channelID []string) error
- DescribeChannels(channelID []string) (ChannelDescriptions, error)
+ CreateChannels(req CreateChannelRequest) (CreateChannelResponse, error)
+ DestoryChannels(req DestoryChannelRequest) error
+ DescribeChannels(req DescribeChannelRequest) (DescribeChannelResponse, error)
}
```
@@ -30,7 +28,19 @@ type OwnerDescription struct {
type CreateChannelRequest struct {
OwnerDescription OwnerDescription
- numChannels int
+ NumChannels int
+}
+
+type CreateChannelResponse struct {
+ ChannelIDs []string
+}
+```
+
+* *DestoryChannels*
+
+```go
+type DestoryChannelRequest struct {
+ ChannelIDs []string
}
```
@@ -39,19 +49,22 @@ type CreateChannelRequest struct {
* *DescribeChannels*
```go
+type DescribeChannelRequest struct {
+ ChannelIDs []string
+}
+
type ChannelDescription struct {
+ ChannelID string
Owner OwnerDescription
}
-type ChannelDescriptions struct {
+type DescribeChannelResponse struct {
Descriptions []ChannelDescription
}
```
-
-
#### A.3 Message Stream
``` go
@@ -60,7 +73,7 @@ const {
kInsert MsgType = 400
kDelete MsgType = 401
kSearch MsgType = 500
- KSearchResult MsgType = 1000
+ kSearchResult MsgType = 1000
kSegStatistics MsgType = 1100
@@ -154,3 +167,55 @@ func NewUnmarshalDispatcher() *UnmarshalDispatcher
```
+
+#### A.4 RocksMQ
+
+RocksMQ is a RocksDB-based messaging/streaming library.
+
+```go
+type ProducerMessage struct {
+ Key string
+ Payload []byte
+}
+```
+
+```go
+type ConsumerMessage struct {
+ MsgID MessageID
+ Key string
+ Payload []byte
+}
+```
+
+
+
+```GO
+type RocksMQ struct {
+ CreateChannel(channelName string) error
+ DestroyChannel(channelName string) error
+ CreateConsumerGroup(groupName string) error
+ DestroyConsumerGroup(groupName string) error
+
+ Produce(channelName string, messages []ProducerMessage) error
+ Consume(groupName string, channelName string, n int) ([]ConsumerMessage, error)
+ Seek(groupName string, channelName string, msgID MessageID) error
+}
+```
+
+
+
+##### A.4.1 Meta
+
+* channel meta
+
+```go
+"$(channel_name)/start_id", MessageID
+"$(channel_name)/end_id", MessageID
+```
+
+* consumer group meta
+
+```go
+"$(group_name)/$(channel_name)/current_id", MessageID
+```
+
diff --git a/docs/developer_guides/chap05_proxy.md b/docs/developer_guides/chap05_proxy.md
index 24ec12e94f5aa518708b97029ceccc1c39e7d486..d9f15af375da0c891a0aa314871c0771bb229965 100644
--- a/docs/developer_guides/chap05_proxy.md
+++ b/docs/developer_guides/chap05_proxy.md
@@ -12,23 +12,60 @@
```go
type Client interface {
+ RegisterLink() (ProxyInfo, error)
+ RegisterNode(req NodeInfo) (InitParams, error)
GetTimeTickChannel() (string, error)
GetStatsChannel() (string, error)
}
```
-
-
-#### 6.1 Gateway API
+* *RegisterLink*
```go
type ProxyInfo struct {
Address string
Port int32
}
+```
-type Client interface {
- RegisterLink() (ProxyInfo, error)
+* *RegisterNode*
+
+```go
+type NodeInfo struct {}
+
+type InitParams struct {}
+```
+
+
+
+#### 6.0 ProxyNode
+
+```go
+type ProxyNode interface {
+ Start() error
+ Close() error
+
+ SetTimeTickChannel(channelID string) error
+ SetStatsChannel(channelID string) error
+
+ CreateCollection(req CreateCollectionRequest) error
+ DropCollection(req DropCollectionRequest) error
+ HasCollection(req HasCollectionRequest) (bool, error)
+ DescribeCollection(req DescribeCollectionRequest) (DescribeCollectionResponse, error)
+ GetCollectionStatistics(req CollectionStatsRequest) (CollectionStatsResponse, error)
+ ShowCollections(req ShowCollectionRequest) ([]string, error)
+
+ CreatePartition(req CreatePartitionRequest) error
+ DropPartition(req DropPartitionRequest) error
+ HasPartition(req HasPartitionRequest) (bool, error)
+ GetPartitionStatistics(req PartitionStatsRequest) (PartitionStatsResponse, error)
+ ShowPartitions(req ShowPartitionRequest) ([]string, error)
+
+ CreateIndex(req CreateIndexRequest) error
+ DescribeIndex(DescribeIndexRequest) (DescribeIndexResponse, error)
+
+ Insert(req RowBatch) (InsertResponse, error)
+ Search(req SearchRequest) (SearchResults, error)
}
```
diff --git a/docs/developer_guides/chap06_master.md b/docs/developer_guides/chap06_master.md
index 28c3b599de0fa43baf893702b9341dfba2a75cc1..1e55ebb925882799b6eee0c3e80ddd8da0d1e77c 100644
--- a/docs/developer_guides/chap06_master.md
+++ b/docs/developer_guides/chap06_master.md
@@ -12,15 +12,22 @@ type Client interface {
CreateCollection(req CreateCollectionRequest) error
DropCollection(req DropCollectionRequest) error
HasCollection(req HasCollectionRequest) (bool, error)
- DescribeCollection(req DescribeCollectionRequest) (CollectionDescription, error)
- ShowCollections(req ShowCollectionRequest) ([]string, error)
+ DescribeCollection(req DescribeCollectionRequest) (CollectionDescriptionResponse, error)
+ GetCollectionStatistics(req CollectionStatsRequest) (CollectionStatsResponse, error)
+ ShowCollections(req ShowCollectionRequest) (ShowCollectionResponse, error)
+
CreatePartition(req CreatePartitionRequest) error
DropPartition(req DropPartitionRequest) error
HasPartition(req HasPartitionRequest) (bool, error)
- DescribePartition(req DescribePartitionRequest) (PartitionDescription, error)
- ShowPartitions(req ShowPartitionRequest) ([]string, error)
+ GetPartitionStatistics(req PartitionStatsRequest) (PartitionStatsResponse, error)
+ ShowPartitions(req ShowPartitionRequest) (ShowPartitionResponse, error)
+
+ CreateIndex(req CreateIndexRequest) error
+ DescribeIndex(DescribeIndexRequest) (DescribeIndexResponse, error)
+
AllocTimestamp(req TsoRequest) (TsoResponse, error)
AllocID(req IDRequest) (IDResponse, error)
+
GetDdChannel() (string, error)
GetTimeTickChannel() (string, error)
GetStatsChannel() (string, error)
@@ -29,6 +36,81 @@ type Client interface {
+* *DescribeCollection*
+
+```go
+type DescribeCollectionRequest struct {
+ CollectionName string
+}
+
+type CollectionDescriptionResponse struct {
+ Schema CollectionSchema
+}
+```
+
+* *GetCollectionStatistics*
+
+```go
+type CollectionStatsRequest struct {
+ CollectionName string
+}
+
+type CollectionStatsResponse struct {
+ Stats []KeyValuePair
+}
+```
+
+* *ShowCollections*
+
+```go
+type ShowCollectionResponse struct {
+ CollectionNames []string
+}
+```
+
+* *GetPartitionStatistics*
+
+```go
+type PartitionStatsRequest struct {
+ CollectionName string
+ PartitionTag string
+}
+
+type PartitionStatsResponse struct {
+ Stats []KeyValuePair
+}
+```
+
+* *ShowPartitions*
+
+```go
+type ShowPartitionResponse struct {
+ PartitionTags []string
+}
+```
+
+* *DescribeIndex*
+
+```go
+type DescribeIndexRequest struct {
+ CollectionName string
+ FieldName string
+}
+
+type IndexDescription struct {
+ IndexName string
+ params []KeyValuePair
+}
+
+type DescribeIndexResponse struct {
+ IndexDescriptions []IndexDescription
+}
+```
+
+
+
+
+
#### 10.1 Interfaces (RPC)
| RPC | description |
diff --git a/docs/developer_guides/chap07_query_service.md b/docs/developer_guides/chap07_query_service.md
index 52295480ab8070cc74a34538cf325147377c2dba..be89e767978cb83a659241d364db61c660f0b096 100644
--- a/docs/developer_guides/chap07_query_service.md
+++ b/docs/developer_guides/chap07_query_service.md
@@ -2,20 +2,21 @@
## 8. Query Service
-
-
#### 8.1 Overview
<img src="./figs/query_service.jpeg" width=700>
-#### 8.2 API
+#### 8.2 Query Service API
```go
type Client interface {
- DescribeService() (ServiceDescription, error)
- DescribeParition(req DescribeParitionRequest) (PartitionDescriptions, error)
+ RegisterNode(req NodeInfo) (InitParams, error)
+ GetServiceStates() (ServiceStatesResponse, error)
+ ShowCollections(req ShowCollectionRequest) (ShowCollectionResponse, error)
+ ShowPartitions(req ShowPartitionRequest) (ShowPartitionResponse, error)
+ GetPartitionStates(req PartitionStatesRequest) (PartitionStatesResponse, error)
LoadPartitions(req LoadPartitonRequest) error
ReleasePartitions(req ReleasePartitionRequest) error
CreateQueryChannel() (QueryChannels, error)
@@ -26,62 +27,95 @@ type Client interface {
-* *DescribeService*
+* *RegisterNode*
```go
-type QueryNodeDescription struct {
- ResourceCost ResourceCost
-}
+type NodeInfo struct {}
-type CollectionDescription struct {
- ParitionIDs []UniqueID
-}
+type InitParams struct {}
+```
+
+* *GetServiceStates*
+
+```go
+type NodeState = int
+
+const (
+ INITIALIZING NodeState = 0
+ HEALTHY NodeState = 1
+ ABNORMAL NodeState = 2
+)
+
+//type ResourceCost struct {
+// MemUsage int64
+// CpuUsage float32
+//}
-type DbDescription struct {
- CollectionDescriptions []CollectionDescription
+type QueryNodeStates struct {
+ NodeState NodeState
+ //ResourceCost ResourceCost
}
-type ServiceDescription struct {
- DbDescriptions map[UniqueID]DbDescription
- NodeDescriptions map[UniqueID]QueryNodeDescription
+type ServiceStatesResponse struct {
+ ServiceState NodeState
}
```
+* *ShowCollections*
+```go
+type ShowCollectionRequest struct {
+ DbID UniqueID
+}
-* *DescribeParition*
+type ShowCollectionResponse struct {
+ CollectionIDs []UniqueID
+}
+```
+
+* *ShowPartitions*
```go
-type DescribeParitionRequest struct {
+type ShowPartitionRequest struct {
DbID UniqueID
CollectionID UniqueID
- partitionIDs []UniqueID
}
+type ShowPartitionResponse struct {
+ PartitionIDs []UniqueID
+}
+```
+
+
+
+* *GetPartitionStates*
+
+```go
type PartitionState = int
const (
NOT_EXIST PartitionState = 0
- ON_DISK PartitionState = 1
- PARTIAL_IN_MEMORY PartitionState = 2
- IN_MEMORY PartitionState = 3
- PARTIAL_IN_GPU PartitionState = 4
- IN_GPU PartitionState = 5
+ NOT_PRESENT PartitionState = 1
+ ON_DISK PartitionState = 2
+ PARTIAL_IN_MEMORY PartitionState = 3
+ IN_MEMORY PartitionState = 4
+ PARTIAL_IN_GPU PartitionState = 5
+ IN_GPU PartitionState = 6
)
-type ResourceCost struct {
- MemUsage int64
- CpuUsage float32
+type PartitionStatesRequest struct {
+ DbID UniqueID
+ CollectionID UniqueID
+ partitionIDs []UniqueID
}
-type PartitionDescription struct {
- ID UniqueID
+type PartitionStates struct {
+ PartitionID UniqueID
State PartitionState
- ResourceCost ResourceCost
}
-type PartitionDescriptions struct {
- PartitionDescriptions []PartitionDescription
+type PartitionStatesResponse struct {
+ States []PartitionStates
}
```
@@ -122,6 +156,33 @@ type ReleasePartitionRequest struct {
+* *LoadSegments*
+
+```go
+type LoadSegmentRequest struct {
+ DbID UniqueID
+ CollectionID UniqueID
+ PartitionID UniqueID
+ SegmentIDs []UniqueID
+ FieldIDs []int64
+}
+```
+
+
+
+* *ReleaseSegments*
+
+```go
+type ReleaseSegmentRequest struct {
+ DbID UniqueID
+ CollectionID UniqueID
+ PartitionID UniqueID
+ SegmentIDs []UniqueID
+}
+```
+
+
+
#### 8.2 Query Node
```go
@@ -129,23 +190,20 @@ type QueryNode interface {
Start() error
Close() error
- AddQueryStream(requestStream MsgStream, resultStream MsgStream) error
- RemoveQueryStream(requestStreamID string) error
- WatchDmStreams(insertStreams MsgStream) error
- WatchDdStream(stream MsgStream) error
- SetTimeTickStream(stream MsgStream) error
- SetStatsStream(stream MsgStream) error
+ AddQueryChannel(channelIDs QueryChannels) error
+ RemoveQueryChannel(channelIDs QueryChannels) error
+ WatchDmChannels(insertChannelIDs []string) error
+ //SetTimeTickChannel(channelID string) error
+ //SetStatsChannel(channelID string) error
- LoadSegments(DbID UniqueID, CollID UniqueID, PartitionID UniqueID, SegIDs []UniqueID, FieldIDs []int64) error
- ReleaseSegments(DbID UniqueID, CollID UniqueID, PartitionID UniqueID, SegIDs []UniqueID) error
- DescribeParition(DbID UniqueID, CollID UniqueID, PartitionID UniqueID) (PartitionDescription, error)
+ LoadSegments(req LoadSegmentRequest) error
+ ReleaseSegments(req ReleaseSegmentRequest) error
+ DescribeParition(req DescribeParitionRequest) (PartitionDescriptions, error)
}
```
-
-
#### 8.2 Collection Replica
$collectionReplica$ contains a in-memory local copy of persistent collections. In common cases, the system has multiple query nodes. Data of a collection will be distributed across all the available query nodes, and each query node's $collectionReplica$ will maintain its own share (only part of the collection).
diff --git a/docs/developer_guides/chap09_data_service.md b/docs/developer_guides/chap09_data_service.md
index 084ac5a003966619a461b12be2066393cb4d6295..dc58c2a8194aa44c40c9709cd011550e215f408d 100644
--- a/docs/developer_guides/chap09_data_service.md
+++ b/docs/developer_guides/chap09_data_service.md
@@ -8,14 +8,17 @@
<img src="./figs/data_service.jpeg" width=700>
-
-#### 8.2 API
+#### 8.2 Data Service API
```go
type Client interface {
+ RegisterNode(req NodeInfo) (InitParams, error)
AssignSegmentID(req AssignSegIDRequest) (AssignSegIDResponse, error)
- Flush(req FlushRequest) (error)
+ Flush(req FlushRequest) error
+ ShowSegments(req ShowSegmentRequest) (ShowSegmentResponse, error)
+ GetSegmentStates(req SegmentStatesRequest) (SegmentStatesResponse, error)
GetInsertBinlogPaths(req InsertBinlogPathRequest) (InsertBinlogPathsResponse, error)
+
GetInsertChannels(req InsertChannelRequest) ([]string, error)
GetTimeTickChannel() (string, error)
GetStatsChannel() (string, error)
@@ -24,6 +27,14 @@ type Client interface {
+* *RegisterNode*
+
+```go
+type NodeInfo struct {}
+
+type InitParams struct {}
+```
+
* *AssignSegmentID*
```go
@@ -65,15 +76,53 @@ type FlushRequest struct {
+* *ShowSegments*
+
+```go
+type ShowSegmentRequest struct {
+ CollectionID UniqueID
+ PartitionID UniqueID
+}
+
+type ShowSegmentResponse struct {
+ SegmentIDs []UniqueID
+}
+```
+
+
+
+* *GetSegmentStates*
+
+```go
+enum SegmentState {
+ NONE = 0;
+ NOT_EXIST = 1;
+ GROWING = 2;
+ SEALED = 3;
+}
+
+type SegmentStatesRequest struct {
+ SegmentID UniqueID
+}
+
+type SegmentStatesResponse struct {
+ State SegmentState
+ CreateTime Timestamp
+ SealedTime Timestamp
+}
+```
+
+
+
* *GetInsertBinlogPaths*
```go
type InsertBinlogPathRequest struct {
- segmentID UniqueID
+ SegmentID UniqueID
}
type InsertBinlogPathsResponse struct {
- FieldIdxToPaths map[int32][]string
+ FieldIDToPaths map[int64][]string
}
```
@@ -88,3 +137,21 @@ type InsertChannelRequest struct {
}
```
+
+
+#### 8.2 Data Node API
+
+```go
+type DataNode interface {
+ Start() error
+ Close() error
+
+ WatchDmChannels(channelIDs []string) error
+ WatchDdChannel(channelID string) error
+ SetTimeTickChannel(channelID string) error
+ SetStatsChannel(channelID string) error
+
+ Flush(req FlushRequest) error
+}
+```
+
diff --git a/docs/jaeger_guides/Opentracing User Guide.md b/docs/jaeger_guides/Opentracing User Guide.md
new file mode 100644
index 0000000000000000000000000000000000000000..d5bfffbc822b9fa8fab57bf5c91fe6320a5fbec1
--- /dev/null
+++ b/docs/jaeger_guides/Opentracing User Guide.md
@@ -0,0 +1,108 @@
+# Opentracing User Guide
+
+This guide mainly describes the use of jaeger and the commonly used display meanings.
+
+Jaeger, inspired by [Dapper](https://research.google.com/pubs/pub36356.html) and [OpenZipkin](https://zipkin.io/), is a distributed tracing platform created by [Uber Technologies](https://uber.github.io/) and donated to [Cloud Native Computing Foundation](https://cncf.io/). It can be used for monitoring microservices-based distributed systems:
+
+- Distributed context propagation
+- Distributed transaction monitoring
+- Root cause analysis
+- Service dependency analysis
+- Performance / latency optimization
+
+See also:
+
+- Jaeger [documentation](https://jaegertracing.io/docs/) for getting started, operational details, and other information.
+- Blog post [Evolving Distributed Tracing at Uber](https://eng.uber.com/distributed-tracing/).
+- Tutorial / walkthrough [Take OpenTracing for a HotROD ride](https://medium.com/@YuriShkuro/take-opentracing-for-a-hotrod-ride-f6e3141f7941).
+
+
+
+We mainly use jaeger as a implementation of opentracing.
+
+Two request: **Insert Request** and **Search Request** in milvus-distributed system is traced at this stage.
+
+
+
+## Jaeger Home page
+
+
+
+
+
+### Lookup by Trace ID
+
+The use of the search box requires configuration of the log collection system. For example, if the log collection system collects the log, if it is an error log, find the Trace ID. Search in jaeger to quickly locate the error. So as to quickly solve the problem
+
+### Search
+
+### Service
+
+Filter with service name
+
+### Operation
+
+Operation in Service, eg request name, function name
+
+### Tags
+
+Set tag to facilitate search. Tag is defined in code
+
+### Lookback
+
+Filter with time.
+
+### Min Duraton Max Duration
+
+The minimum and maximum request duration, you can use this condition to find the time-consuming and short-term requests
+
+### Limit Result
+
+The max number of result
+
+## Search result
+
+You can search through the above conditions, and those that meet the conditions will appear in the right half.
+
+
+
+The detailed information of search result.
+
+1. The upper left corner identifies the service name: tracing, Root span name: Insert grpc received and The first half of the Trace ID 46874e2.
+2. The duration of the entire request is shown in the upper right corner.
+3. 10675 Span means that there are 10675 operations, see the number of operations for each service in the middle. And the trace time is shown in the right.
+4. The user can select multiple trace with the box in the upper left, and then compare with them to find something different. For example, different insert request may take a different time. At this time, you can select two for comparison, and the comparison can be very easy to know which trace went wrong
+
+
+
+## Detailed trace information
+
+Click the search result. You can analyze the detail trace information.
+
+
+
+1. Duration: Total time consume.
+2. Service: The number of service called.
+3. Depth: Call chain depth.
+4. Total Spans: This call consists of 10 spans
+5. To enter the details, look at the left half first, showing the call chain of the entire request. The black represents the service name, and the gray kid represents the span name defined in the code.
+6. The duration of the right half of the code call. The length represents the time consumed by the Span in the entire call chain.
+
+
+
+### Span Detail information
+
+Click the Span to see the detailed span information such as the last span in the picture above.
+
+1. Tags contains a series of custom tags. You can mark in the code what type of call this Span is, request method, call result, call, etc. All the information it contains can be filtered by the Tags on the homepage.
+2. Process can locate which specific server processing this data.
+3. Logs are the logs printed by this span during the call.
+
+
+
+
+
+## More
+
+More usage guides will be updated in the future
+
diff --git a/docs/jaeger_guides/figs/jaeger_detailed_trace_info.png b/docs/jaeger_guides/figs/jaeger_detailed_trace_info.png
new file mode 100644
index 0000000000000000000000000000000000000000..e895a2faf0603a2f933f9a8c2281c805bf97419e
Binary files /dev/null and b/docs/jaeger_guides/figs/jaeger_detailed_trace_info.png differ
diff --git a/docs/jaeger_guides/figs/jaeger_home_page.png b/docs/jaeger_guides/figs/jaeger_home_page.png
new file mode 100644
index 0000000000000000000000000000000000000000..c04e54a89bb59fb488f927348df9979f48efc57f
Binary files /dev/null and b/docs/jaeger_guides/figs/jaeger_home_page.png differ
diff --git a/docs/jaeger_guides/figs/jaeger_single_search_result.png b/docs/jaeger_guides/figs/jaeger_single_search_result.png
new file mode 100644
index 0000000000000000000000000000000000000000..d9dd656c063760850652baa3c67878849bdbebe1
Binary files /dev/null and b/docs/jaeger_guides/figs/jaeger_single_search_result.png differ
diff --git a/go.mod b/go.mod
index 47afde2bff278667721c394a64396793788e92f2..be987475707967bb4a492efb90b25c62e6923c3f 100644
--- a/go.mod
+++ b/go.mod
@@ -4,14 +4,19 @@ go 1.15
require (
code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
+ github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
github.com/apache/pulsar-client-go v0.1.1
- github.com/aws/aws-sdk-go v1.30.8
+ github.com/apache/thrift v0.13.0
+ github.com/aws/aws-sdk-go v1.30.8 // indirect
github.com/coreos/etcd v3.3.25+incompatible // indirect
- github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548
+ github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
+ github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
+ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+ github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/frankban/quicktest v1.10.2 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
- github.com/git-hooks/git-hooks v1.3.1 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+ github.com/golang/mock v1.3.1
github.com/golang/protobuf v1.3.2
github.com/google/btree v1.0.0
github.com/klauspost/compress v1.10.11 // indirect
@@ -20,22 +25,25 @@ require (
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/onsi/ginkgo v1.12.1 // indirect
github.com/onsi/gomega v1.10.0 // indirect
- github.com/opentracing/opentracing-go v1.2.0 // indirect
+ github.com/opentracing/opentracing-go v1.2.0
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 // indirect
github.com/pingcap/errors v0.11.4 // indirect
github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 // indirect
- github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48
+ github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
github.com/prometheus/client_golang v1.5.1 // indirect
github.com/prometheus/common v0.10.0 // indirect
github.com/prometheus/procfs v0.1.3 // indirect
- github.com/russross/blackfriday/v2 v2.1.0 // indirect
+ github.com/quasilyte/go-ruleguard v0.2.1 // indirect
github.com/sirupsen/logrus v1.6.0 // indirect
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.3.0
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.6.1
+ github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b
+ github.com/uber/jaeger-client-go v2.25.0+incompatible
+ github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
github.com/urfave/cli v1.22.5 // indirect
github.com/yahoo/athenz v1.9.16 // indirect
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
@@ -50,7 +58,7 @@ require (
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150 // indirect
google.golang.org/grpc v1.31.0
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
- gopkg.in/yaml.v2 v2.3.0
+ gopkg.in/yaml.v2 v2.3.0 // indirect
honnef.co/go/tools v0.0.1-2020.1.4 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)
diff --git a/go.sum b/go.sum
index eb4ef6b6a4059c712a42b92603ceb5d687117066..aaf70aaa9c86d4ccb5d0482c6742d7fd5c37e41b 100644
--- a/go.sum
+++ b/go.sum
@@ -15,6 +15,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw=
+github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
@@ -24,6 +26,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1C
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apache/pulsar-client-go v0.1.1 h1:v/kU+2ZCC6yFIcbZrFtWa9/nvVzVr18L+xYJUvZSxEQ=
github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2czVPb27D477YpyU=
+github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI=
+github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4=
github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI=
github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk=
@@ -40,12 +44,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4=
-github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab h1:1WOH7EEbhb6OZWcIU5RpQx5rmHm1xEUda8Qiw4UzNlU=
-github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab/go.mod h1:5MSsYMW59C/HfIUsthTRDxRoMQctcmAVb1JnNSQXERA=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -68,8 +68,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
@@ -89,6 +87,12 @@ github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaI
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk=
github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
@@ -96,8 +100,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/git-hooks/git-hooks v1.3.1 h1:ypdvNTXMiITXQxuqIl6t1f8R3V1FrUXPi1CwQ4guClo=
-github.com/git-hooks/git-hooks v1.3.1/go.mod h1:RBqjgxUpRSLI4AxVZGV1FsODclMjpsJIaDEyJSoK3ws=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -117,6 +119,7 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18h
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -134,10 +137,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -168,6 +167,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -343,19 +343,19 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/protocolbuffers/protobuf v3.14.0+incompatible h1:8r0H76h/Q/lEnFFY60AuM23NOnaDMi6bd7zuboSYM+o=
+github.com/quasilyte/go-ruleguard v0.2.1 h1:56eRm0daAyny9UhJnmtJW/UyLZQusukBAB8oT8AHKHo=
+github.com/quasilyte/go-ruleguard v0.2.1/go.mod h1:hN2rVc/uS4bQhQKTio2XaSJSafJwqBUWWwtssT3cQmc=
+github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210108021830-2a284c158646 h1:ShKMENtS4KYekn92z5vdicyR7wBOgpxFNj5h0M2rNKg=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -389,7 +389,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v0.0.0-20141015234014-d6577e08ec30/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -398,24 +397,27 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
+github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b h1:VOG2GkM7RpRrT0St7HIIwCWrc3mVdf+DjcT8r2ucusI=
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/uber/jaeger-client-go v1.6.0 h1:3+zLlq+4npI5fg8IsgAje3YsP7TcEdNzJScyqFIzxEQ=
+github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
+github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
+github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo=
+github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ=
+github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4=
github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
-github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ=
github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
-github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
-github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0=
@@ -569,6 +571,7 @@ golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d h1:W07d4xkoAUSNOkOzdzXCdFGxT7o2rW4q8M34tB2i//k=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/internal/allocator/allocator.go b/internal/allocator/allocator.go
index a1aec5d8a234cc8db4388ec014b25986c17620c0..9cd7e935d718e19b84a7423c66cc8775c34c298b 100644
--- a/internal/allocator/allocator.go
+++ b/internal/allocator/allocator.go
@@ -137,10 +137,7 @@ type Allocator struct {
}
func (ta *Allocator) Start() error {
- connectMasterFn := func() error {
- return ta.connectMaster()
- }
- err := Retry(10, time.Millisecond*200, connectMasterFn)
+ err := ta.connectMaster()
if err != nil {
panic("connect to master failed")
}
diff --git a/internal/allocator/retry.go b/internal/allocator/retry.go
deleted file mode 100644
index 89ab43cd00d3cffc036fb4a84f237ab8c7df5f89..0000000000000000000000000000000000000000
--- a/internal/allocator/retry.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package allocator
-
-import (
- "log"
- "time"
-)
-
-// Reference: https://blog.cyeam.com/golang/2018/08/27/retry
-
-func RetryImpl(attempts int, sleep time.Duration, fn func() error, maxSleepTime time.Duration) error {
- if err := fn(); err != nil {
- if s, ok := err.(InterruptError); ok {
- return s.error
- }
-
- if attempts--; attempts > 0 {
- log.Printf("retry func error: %s. attempts #%d after %s.", err.Error(), attempts, sleep)
- time.Sleep(sleep)
- if sleep < maxSleepTime {
- return RetryImpl(attempts, 2*sleep, fn, maxSleepTime)
- }
- return RetryImpl(attempts, maxSleepTime, fn, maxSleepTime)
- }
- return err
- }
- return nil
-}
-
-func Retry(attempts int, sleep time.Duration, fn func() error) error {
- maxSleepTime := time.Millisecond * 1000
- return RetryImpl(attempts, sleep, fn, maxSleepTime)
-}
-
-type InterruptError struct {
- error
-}
-
-func NoRetryError(err error) InterruptError {
- return InterruptError{err}
-}
diff --git a/internal/core/src/indexbuilder/IndexWrapper.cpp b/internal/core/src/indexbuilder/IndexWrapper.cpp
index 6a5db6b1c67fdc4d7685e65b1024d52e04dcb1a2..4484d1e119bc4853e5cd49df6396047e48df8e8f 100644
--- a/internal/core/src/indexbuilder/IndexWrapper.cpp
+++ b/internal/core/src/indexbuilder/IndexWrapper.cpp
@@ -19,7 +19,6 @@
#include "utils/EasyAssert.h"
#include "IndexWrapper.h"
#include "indexbuilder/utils.h"
-#include "index/knowhere/knowhere/index/vector_index/ConfAdapterMgr.h"
namespace milvus {
namespace indexbuilder {
@@ -30,10 +29,10 @@ IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* seria
parse();
- auto index_mode = get_index_mode();
- auto index_type = get_index_type();
- auto metric_type = get_metric_type();
- AssertInfo(!is_unsupported(index_type, metric_type), index_type + " doesn't support metric: " + metric_type);
+ std::map<std::string, knowhere::IndexMode> mode_map = {{"CPU", knowhere::IndexMode::MODE_CPU},
+ {"GPU", knowhere::IndexMode::MODE_GPU}};
+ auto mode = get_config_by_name<std::string>("index_mode");
+ auto index_mode = mode.has_value() ? mode_map[mode.value()] : knowhere::IndexMode::MODE_CPU;
index_ = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(get_index_type(), index_mode);
Assert(index_ != nullptr);
@@ -155,11 +154,6 @@ IndexWrapper::dim() {
void
IndexWrapper::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
auto index_type = get_index_type();
- auto index_mode = get_index_mode();
- config_[knowhere::meta::ROWS] = dataset->Get<int64_t>(knowhere::meta::ROWS);
- auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
- AssertInfo(conf_adapter->CheckTrain(config_, index_mode), "something wrong in index parameters!");
-
if (is_in_need_id_list(index_type)) {
PanicInfo(std::string(index_type) + " doesn't support build without ids yet!");
}
@@ -179,11 +173,6 @@ IndexWrapper::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
void
IndexWrapper::BuildWithIds(const knowhere::DatasetPtr& dataset) {
Assert(dataset->data().find(milvus::knowhere::meta::IDS) != dataset->data().end());
- auto index_type = get_index_type();
- auto index_mode = get_index_mode();
- config_[knowhere::meta::ROWS] = dataset->Get<int64_t>(knowhere::meta::ROWS);
- auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
- AssertInfo(conf_adapter->CheckTrain(config_, index_mode), "something wrong in index parameters!");
// index_->Train(dataset, config_);
// index_->Add(dataset, config_);
index_->BuildAll(dataset, config_);
@@ -274,31 +263,6 @@ IndexWrapper::get_index_type() {
return type.has_value() ? type.value() : knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
}
-std::string
-IndexWrapper::get_metric_type() {
- auto type = get_config_by_name<std::string>(knowhere::Metric::TYPE);
- if (type.has_value()) {
- return type.value();
- } else {
- auto index_type = get_index_type();
- if (is_in_bin_list(index_type)) {
- return knowhere::Metric::JACCARD;
- } else {
- return knowhere::Metric::L2;
- }
- }
-}
-
-knowhere::IndexMode
-IndexWrapper::get_index_mode() {
- static std::map<std::string, knowhere::IndexMode> mode_map = {
- {"CPU", knowhere::IndexMode::MODE_CPU},
- {"GPU", knowhere::IndexMode::MODE_GPU},
- };
- auto mode = get_config_by_name<std::string>("index_mode");
- return mode.has_value() ? mode_map[mode.value()] : knowhere::IndexMode::MODE_CPU;
-}
-
std::unique_ptr<IndexWrapper::QueryResult>
IndexWrapper::Query(const knowhere::DatasetPtr& dataset) {
return std::move(QueryImpl(dataset, config_));
diff --git a/internal/core/src/indexbuilder/IndexWrapper.h b/internal/core/src/indexbuilder/IndexWrapper.h
index 8bf2ed881c8a7e2ee4f86cdb88da7200e47ebfbe..16f2721712c655bff7b2e7d53a235e32ed1d6458 100644
--- a/internal/core/src/indexbuilder/IndexWrapper.h
+++ b/internal/core/src/indexbuilder/IndexWrapper.h
@@ -59,12 +59,6 @@ class IndexWrapper {
std::string
get_index_type();
- std::string
- get_metric_type();
-
- knowhere::IndexMode
- get_index_mode();
-
template <typename T>
std::optional<T>
get_config_by_name(std::string name);
diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp
index e01d98989768acde93cab89f1ee8cb3fe0cc854f..217372700bb56faa63e24816d3b17ac791137fa0 100644
--- a/internal/core/src/indexbuilder/index_c.cpp
+++ b/internal/core/src/indexbuilder/index_c.cpp
@@ -35,7 +35,7 @@ CreateIndex(const char* serialized_type_params, const char* serialized_index_par
*res_index = index.release();
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -59,7 +59,7 @@ BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float*
cIndex->BuildWithoutIds(ds);
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -77,7 +77,7 @@ BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* ve
cIndex->BuildWithoutIds(ds);
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -94,7 +94,7 @@ SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size, char** res_buffer) {
*res_buffer = binary.data;
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -109,7 +109,7 @@ LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, in
cIndex->Load(serialized_sliced_blob_buffer, size);
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -129,7 +129,7 @@ QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -153,7 +153,7 @@ QueryOnFloatVecIndexWithParam(CIndex index,
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -173,7 +173,7 @@ QueryOnBinaryVecIndex(CIndex index, int64_t data_size, const uint8_t* vectors, C
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -197,7 +197,7 @@ QueryOnBinaryVecIndexWithParam(CIndex index,
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -213,7 +213,7 @@ CreateQueryResult(CIndexQueryResult* res) {
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
@@ -259,7 +259,7 @@ DeleteQueryResult(CIndexQueryResult res) {
status.error_code = Success;
status.error_msg = "";
- } catch (std::exception& e) {
+ } catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
diff --git a/internal/core/src/indexbuilder/utils.h b/internal/core/src/indexbuilder/utils.h
index 6e1d89a2d9d82b585b46a358ff8cc47ed9dd510e..e1ed0804965cd9698a0a42cb182931d6c2a94cc3 100644
--- a/internal/core/src/indexbuilder/utils.h
+++ b/internal/core/src/indexbuilder/utils.h
@@ -14,7 +14,6 @@
#include <vector>
#include <string>
#include <algorithm>
-#include <tuple>
#include "index/knowhere/knowhere/index/IndexType.h"
@@ -58,14 +57,6 @@ Need_BuildAll_list() {
return ret;
}
-std::vector<std::tuple<std::string, std::string>>
-unsupported_index_combinations() {
- static std::vector<std::tuple<std::string, std::string>> ret{
- std::make_tuple(std::string(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT), std::string(knowhere::Metric::L2)),
- };
- return ret;
-}
-
template <typename T>
bool
is_in_list(const T& t, std::function<std::vector<T>()> list_func) {
@@ -93,11 +84,5 @@ is_in_need_id_list(const milvus::knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, Need_ID_List);
}
-bool
-is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) {
- return is_in_list<std::tuple<std::string, std::string>>(std::make_tuple(index_type, metric_type),
- unsupported_index_combinations);
-}
-
} // namespace indexbuilder
} // namespace milvus
diff --git a/internal/core/src/query/SearchOnSealed.cpp b/internal/core/src/query/SearchOnSealed.cpp
index 94d111f3eb227d53cd85e3e32e00cd95c57c8f32..981251839821a29cdef880c38a12cf76a89b2b1f 100644
--- a/internal/core/src/query/SearchOnSealed.cpp
+++ b/internal/core/src/query/SearchOnSealed.cpp
@@ -63,8 +63,6 @@ SearchOnSealed(const Schema& schema,
Assert(record.test_readiness(field_offset));
auto indexing_entry = record.get_entry(field_offset);
- std::cout << " SearchOnSealed, indexing_entry->metric:" << indexing_entry->metric_type_ << std::endl;
- std::cout << " SearchOnSealed, query_info.metric_type_:" << query_info.metric_type_ << std::endl;
Assert(indexing_entry->metric_type_ == GetMetricType(query_info.metric_type_));
auto final = [&] {
diff --git a/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp b/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
index 28b0730e6a3538510701abe11c5db5ad329478b0..263390a39a52831c0eb7f1bd71f8dc0cc1cecdb5 100644
--- a/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
+++ b/internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
@@ -61,17 +61,6 @@ InferIndexType(const Json& search_params) {
PanicInfo("failed to infer index type");
}
-static knowhere::IndexType
-InferBinaryIndexType(const Json& search_params) {
- namespace ip = knowhere::IndexParams;
- namespace ie = knowhere::IndexEnum;
- if (search_params.contains(ip::nprobe)) {
- return ie::INDEX_FAISS_BIN_IVFFLAT;
- } else {
- return ie::INDEX_FAISS_BIN_IDMAP;
- }
-}
-
void
VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
auto& search_params = node.query_info_.search_params_;
@@ -90,18 +79,7 @@ VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
void
VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
- auto& search_params = node.query_info_.search_params_;
- auto inferred_type = InferBinaryIndexType(search_params);
- auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(inferred_type);
- auto index_mode = knowhere::IndexMode::MODE_CPU;
-
- // mock the api, topk will be passed from placeholder
- auto params_copy = search_params;
- params_copy[knowhere::meta::TOPK] = 10;
-
- // NOTE: the second parameter is not checked in knowhere, may be redundant
- auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode);
- AssertInfo(passed, "invalid search params");
+ // TODO
}
} // namespace milvus::query
diff --git a/internal/core/src/segcore/load_index_c.cpp b/internal/core/src/segcore/load_index_c.cpp
index 9afd0b6262e35e4aae490cbb5f1ca17754a2d5d8..5c8da297958b9665eccca37228e2b2c961983f92 100644
--- a/internal/core/src/segcore/load_index_c.cpp
+++ b/internal/core/src/segcore/load_index_c.cpp
@@ -133,7 +133,7 @@ AppendBinaryIndex(CBinarySet c_binary_set, void* index_binary, int64_t index_siz
auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
std::string index_key(c_index_key);
uint8_t* index = (uint8_t*)index_binary;
- std::shared_ptr<uint8_t[]> data(index, [](void*) {});
+ std::shared_ptr<uint8_t[]> data(index);
binary_set->Append(index_key, data, index_size);
auto status = CStatus();
diff --git a/internal/indexbuilder/client/client.go b/internal/indexbuilder/client/client.go
index c18e02017b4f9f76a10f2232a6ab581dc4d513aa..622fde9b1507251a67c170a015c106f56fe50e35 100644
--- a/internal/indexbuilder/client/client.go
+++ b/internal/indexbuilder/client/client.go
@@ -2,10 +2,6 @@ package indexbuilderclient
import (
"context"
- "encoding/json"
- "fmt"
- "github.com/zilliztech/milvus-distributed/internal/errors"
- "log"
"time"
"google.golang.org/grpc"
@@ -58,59 +54,20 @@ func (c *Client) BuildIndexWithoutID(columnDataPaths []string, typeParams map[st
if c.tryConnect() != nil {
panic("BuildIndexWithoutID: failed to connect index builder")
}
- parseMap := func(mStr string) (map[string]string, error) {
- buffer := make(map[string]interface{})
- err := json.Unmarshal([]byte(mStr), &buffer)
- if err != nil {
- return nil, errors.New("Unmarshal params failed")
- }
- ret := make(map[string]string)
- for key, value := range buffer {
- valueStr := fmt.Sprintf("%v", value)
- ret[key] = valueStr
- }
- return ret, nil
- }
var typeParamsKV []*commonpb.KeyValuePair
- for key := range typeParams {
- if key == "params" {
- mapParams, err := parseMap(typeParams[key])
- if err != nil {
- log.Println("parse params error: ", err)
- }
- for pk, pv := range mapParams {
- typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
- Key: pk,
- Value: pv,
- })
- }
- } else {
- typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
- Key: key,
- Value: typeParams[key],
- })
- }
+ for typeParam := range typeParams {
+ typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
+ Key: typeParam,
+ Value: typeParams[typeParam],
+ })
}
var indexParamsKV []*commonpb.KeyValuePair
- for key := range indexParams {
- if key == "params" {
- mapParams, err := parseMap(indexParams[key])
- if err != nil {
- log.Println("parse params error: ", err)
- }
- for pk, pv := range mapParams {
- indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
- Key: pk,
- Value: pv,
- })
- }
- } else {
- indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
- Key: key,
- Value: indexParams[key],
- })
- }
+ for indexParam := range indexParams {
+ indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
+ Key: indexParam,
+ Value: indexParams[indexParam],
+ })
}
ctx := context.TODO()
diff --git a/internal/indexbuilder/index.go b/internal/indexbuilder/index.go
index 16439c4b299feafc829e581ee508b5076d8cbab7..fceec4bcf06083a4bff5a29a13a94e0e0864f29d 100644
--- a/internal/indexbuilder/index.go
+++ b/internal/indexbuilder/index.go
@@ -14,7 +14,6 @@ package indexbuilder
import "C"
import (
"errors"
- "fmt"
"strconv"
"unsafe"
@@ -106,13 +105,10 @@ func (index *CIndex) BuildFloatVecIndexWithoutIds(vectors []float32) error {
CStatus
BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors);
*/
- fmt.Println("before BuildFloatVecIndexWithoutIds")
status := C.BuildFloatVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.float)(&vectors[0]))
errorCode := status.error_code
- fmt.Println("BuildFloatVecIndexWithoutIds error code: ", errorCode)
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
- fmt.Println("BuildFloatVecIndexWithoutIds error msg: ", errorMsg)
defer C.free(unsafe.Pointer(status.error_msg))
return errors.New("BuildFloatVecIndexWithoutIds failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
@@ -146,8 +142,6 @@ func (index *CIndex) Delete() error {
}
func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
- fmt.Println("NNNNNNNNNNNNNNNNNNNNNNNNNNN typeParams: ", typeParams)
- fmt.Println("NNNNNNNNNNNNNNNNNNNNNNNNNNN indexParams: ", indexParams)
protoTypeParams := &indexcgopb.TypeParams{
Params: make([]*commonpb.KeyValuePair, 0),
}
@@ -174,14 +168,10 @@ func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
CIndex* res_index);
*/
var indexPtr C.CIndex
- fmt.Println("before create index ........................................")
status := C.CreateIndex(typeParamsPointer, indexParamsPointer, &indexPtr)
- fmt.Println("after create index ........................................")
errorCode := status.error_code
- fmt.Println("EEEEEEEEEEEEEEEEEEEEEEEEEE error code: ", errorCode)
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
- fmt.Println("EEEEEEEEEEEEEEEEEEEEEEEEEE error msg: ", errorMsg)
defer C.free(unsafe.Pointer(status.error_msg))
return nil, errors.New(" failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
diff --git a/internal/indexbuilder/indexbuilder.go b/internal/indexbuilder/indexbuilder.go
index 712d17ef80103b48590e1bec01e8889ee0855d06..4acfffc3d284665158d9f3c237682a5792257d50 100644
--- a/internal/indexbuilder/indexbuilder.go
+++ b/internal/indexbuilder/indexbuilder.go
@@ -54,48 +54,34 @@ func CreateBuilder(ctx context.Context) (*Builder, error) {
loopCancel: cancel,
}
- connectEtcdFn := func() error {
- etcdAddress := Params.EtcdAddress
- etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}})
- if err != nil {
- return err
- }
- etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath)
- metakv, err := NewMetaTable(etcdKV)
- if err != nil {
- return err
- }
- b.metaTable = metakv
- return nil
+ etcdAddress := Params.EtcdAddress
+ etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}})
+ if err != nil {
+ return nil, err
}
- err := Retry(10, time.Millisecond*200, connectEtcdFn)
+ etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath)
+ metakv, err := NewMetaTable(etcdKV)
if err != nil {
return nil, err
}
+ b.metaTable = metakv
idAllocator, err := allocator.NewIDAllocator(b.loopCtx, Params.MasterAddress)
- b.idAllocator = idAllocator
- connectMinIOFn := func() error {
- option := &miniokv.Option{
- Address: Params.MinIOAddress,
- AccessKeyID: Params.MinIOAccessKeyID,
- SecretAccessKeyID: Params.MinIOSecretAccessKey,
- UseSSL: Params.MinIOUseSSL,
- BucketName: Params.MinioBucketName,
- CreateBucket: true,
- }
-
- b.kv, err = miniokv.NewMinIOKV(b.loopCtx, option)
- if err != nil {
- return err
- }
- return nil
+ option := &miniokv.Option{
+ Address: Params.MinIOAddress,
+ AccessKeyID: Params.MinIOAccessKeyID,
+ SecretAccessKeyID: Params.MinIOSecretAccessKey,
+ UseSSL: Params.MinIOUseSSL,
+ BucketName: Params.MinioBucketName,
+ CreateBucket: true,
}
- err = Retry(10, time.Millisecond*200, connectMinIOFn)
+
+ b.kv, err = miniokv.NewMinIOKV(b.loopCtx, option)
if err != nil {
return nil, err
}
+ b.idAllocator = idAllocator
b.sched, err = NewTaskScheduler(b.loopCtx, b.idAllocator, b.kv, b.metaTable)
if err != nil {
diff --git a/internal/indexbuilder/retry.go b/internal/indexbuilder/retry.go
deleted file mode 100644
index 2cf4c6ecf576ddf095b85290b30bdb8f5b108047..0000000000000000000000000000000000000000
--- a/internal/indexbuilder/retry.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package indexbuilder
-
-import (
- "log"
- "time"
-)
-
-// Reference: https://blog.cyeam.com/golang/2018/08/27/retry
-
-func RetryImpl(attempts int, sleep time.Duration, fn func() error, maxSleepTime time.Duration) error {
- if err := fn(); err != nil {
- if s, ok := err.(InterruptError); ok {
- return s.error
- }
-
- if attempts--; attempts > 0 {
- log.Printf("retry func error: %s. attempts #%d after %s.", err.Error(), attempts, sleep)
- time.Sleep(sleep)
- if sleep < maxSleepTime {
- return RetryImpl(attempts, 2*sleep, fn, maxSleepTime)
- }
- return RetryImpl(attempts, maxSleepTime, fn, maxSleepTime)
- }
- return err
- }
- return nil
-}
-
-func Retry(attempts int, sleep time.Duration, fn func() error) error {
- maxSleepTime := time.Millisecond * 1000
- return RetryImpl(attempts, sleep, fn, maxSleepTime)
-}
-
-type InterruptError struct {
- error
-}
-
-func NoRetryError(err error) InterruptError {
- return InterruptError{err}
-}
diff --git a/internal/indexbuilder/task.go b/internal/indexbuilder/task.go
index 73a5b77508b1e36b69a6f5cef440d6451f97b9c9..15647e1563e53f3abab9976ca438ef334212fb34 100644
--- a/internal/indexbuilder/task.go
+++ b/internal/indexbuilder/task.go
@@ -2,7 +2,6 @@ package indexbuilder
import (
"context"
- "fmt"
"log"
"strconv"
"time"
@@ -172,12 +171,10 @@ func (it *IndexBuildTask) Execute() error {
indexParams[key] = value
}
- fmt.Println("before NewCIndex ..........................")
it.index, err = NewCIndex(typeParams, indexParams)
if err != nil {
return err
}
- fmt.Println("after NewCIndex ..........................")
getKeyByPathNaive := func(path string) string {
// splitElements := strings.Split(path, "/")
@@ -226,7 +223,6 @@ func (it *IndexBuildTask) Execute() error {
for _, value := range insertData.Data {
// TODO: BinaryVectorFieldData
- fmt.Println("before build index ..................................")
floatVectorFieldData, fOk := value.(*storage.FloatVectorFieldData)
if fOk {
err = it.index.BuildFloatVecIndexWithoutIds(floatVectorFieldData.Data)
@@ -242,15 +238,12 @@ func (it *IndexBuildTask) Execute() error {
return err
}
}
- fmt.Println("after build index ..................................")
if !fOk && !bOk {
return errors.New("we expect FloatVectorFieldData or BinaryVectorFieldData")
}
- fmt.Println("before serialize .............................................")
indexBlobs, err := it.index.Serialize()
- fmt.Println("after serialize .............................................")
if err != nil {
return err
}
diff --git a/internal/kv/rocksdb/cwrapper/CMakeLists.txt b/internal/kv/rocksdb/cwrapper/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..56602edcda5757e54084216f3558dac94c7c68ec
--- /dev/null
+++ b/internal/kv/rocksdb/cwrapper/CMakeLists.txt
@@ -0,0 +1,55 @@
+cmake_minimum_required(VERSION 3.14...3.17 FATAL_ERROR)
+project(wrapper)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+include( ExternalProject )
+set( ROCKSDB_VERSION "6.15.2" )
+set( ROCKSDB_SOURCE_URL
+ "https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz")
+
+if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH )
+ set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} )
+else()
+ set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download )
+endif()
+message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" )
+
+macro( build_rocksdb )
+ message( STATUS "Building ROCKSDB-${ROCKSDB_VERSION} from source" )
+
+ set( ROCKSDB_CMAKE_ARGS
+ "-DWITH_GFLAGS=OFF"
+ "-DROCKSDB_BUILD_SHARED=OFF"
+# "-DWITH_SNAPPY=ON"
+# "-DWITH_LZ4=ON"
+# "-DWITH_ZSTD=ON"
+# "-DWITH_BZ2=ON"
+ "-DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}"
+ )
+
+ ExternalProject_Add(
+ rocksdb-ep
+ PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/rocksdb-subbuild
+ BINARY_DIR rocksdb-bin
+ DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH}
+ INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}
+ URL ${ROCKSDB_SOURCE_URL}
+ URL_MD5 "67f9e04fda62af551dd039c37b1808ac"
+ CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS}
+ ${EP_LOG_OPTIONS}
+ )
+
+ ExternalProject_Get_Property( rocksdb-ep INSTALL_DIR )
+ ExternalProject_Get_Property( rocksdb-ep BINARY_DIR )
+
+ if( NOT IS_DIRECTORY ${INSTALL_DIR}/include )
+ file( MAKE_DIRECTORY "${INSTALL_DIR}/include" )
+ endif()
+
+endmacro()
+
+build_rocksdb()
+
+#endif()
diff --git a/internal/kv/rocksdb/cwrapper/build.sh b/internal/kv/rocksdb/cwrapper/build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..6ed4a2bd2a8b637464f0aaeeacebeeb6308322fb
--- /dev/null
+++ b/internal/kv/rocksdb/cwrapper/build.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+SOURCE=${BASH_SOURCE[0]}
+while [ -h $SOURCE ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR=$( cd -P $( dirname $SOURCE ) && pwd )
+ SOURCE=$(readlink $SOURCE)
+ [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR=$( cd -P $( dirname $SOURCE ) && pwd )
+# echo $DIR
+
+CGO_CFLAGS="-I$(pwd)/output/include"
+CGO_LDFLAGS="-L$(pwd)/output/lib -l:librocksdb.a -lstdc++ -lm -lz"
+
+OUTPUT_LIB=${DIR}/output
+
+if [ -d ${OUTPUT_LIB} ];then
+ rm -rf ${OUTPUT_LIB}
+fi
+mkdir ${OUTPUT_LIB}
+
+BUILD_TYPE="Debug"
+
+while getopts "t:h:" arg; do
+ case $arg in
+ t)
+ BUILD_TYPE=$OPTARG # BUILD_TYPE
+ ;;
+ h) # help
+ echo "-t: build type(default: Debug)
+-h: help
+ "
+ exit 0
+ ;;
+ ?)
+ echo "ERROR! unknown argument"
+ exit 1
+ ;;
+ esac
+done
+echo "BUILD_TYPE: " $BUILD_TYPE
+
+pushd ${OUTPUT_LIB}
+CMAKE_CMD="cmake \
+-DCMAKE_BUILD_TYPE=${BUILD_TYPE} .."
+
+${CMAKE_CMD}
+echo ${CMAKE_CMD}
+
+if [[ ! ${jobs+1} ]]; then
+ jobs=$(nproc)
+fi
+make -j ${jobs}
diff --git a/internal/kv/rocksdb/rocksdb_kv.go b/internal/kv/rocksdb/rocksdb_kv.go
new file mode 100644
index 0000000000000000000000000000000000000000..318f3078e84d6a3e3c6f21799e7cca26aaceb922
--- /dev/null
+++ b/internal/kv/rocksdb/rocksdb_kv.go
@@ -0,0 +1,128 @@
+package rocksdbkv
+
+import (
+ "github.com/tecbot/gorocksdb"
+)
+
+type RocksdbKV struct {
+ opts *gorocksdb.Options
+ db *gorocksdb.DB
+ writeOptions *gorocksdb.WriteOptions
+ readOptions *gorocksdb.ReadOptions
+ name string
+}
+
+func NewRocksdbKV(name string) (*RocksdbKV, error) {
+ bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
+ bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30))
+ opts := gorocksdb.NewDefaultOptions()
+ opts.SetBlockBasedTableFactory(bbto)
+ opts.SetCreateIfMissing(true)
+
+ ro := gorocksdb.NewDefaultReadOptions()
+ ro.SetFillCache(false)
+
+ wo := gorocksdb.NewDefaultWriteOptions()
+ db, err := gorocksdb.OpenDb(opts, name)
+ if err != nil {
+ return nil, err
+ }
+ return &RocksdbKV{
+ opts: opts,
+ db: db,
+ writeOptions: wo,
+ readOptions: ro,
+ name: name,
+ }, nil
+}
+
+func (kv *RocksdbKV) Close() {
+ kv.db.Close()
+}
+
+func (kv *RocksdbKV) GetName() string {
+ return kv.name
+}
+
+func (kv *RocksdbKV) Load(key string) (string, error) {
+ value, err := kv.db.Get(kv.readOptions, []byte(key))
+ defer value.Free()
+ return string(value.Data()), err
+}
+
+func (kv *RocksdbKV) LoadWithPrefix(key string) ([]string, []string, error) {
+ iter := kv.db.NewIterator(kv.readOptions)
+ keys := make([]string, 0)
+ values := make([]string, 0)
+ iter.Seek([]byte(key))
+ for ; iter.Valid(); iter.Next() {
+ key := iter.Key()
+ value := iter.Value()
+ keys = append(keys, string(key.Data()))
+ values = append(values, string(value.Data()))
+ key.Free()
+ value.Free()
+ }
+ if err := iter.Err(); err != nil {
+ return nil, nil, err
+ }
+ return keys, values, nil
+}
+
+func (kv *RocksdbKV) MultiLoad(keys []string) ([]string, error) {
+ values := make([]string, 0, len(keys))
+ for _, key := range keys {
+ value, err := kv.db.Get(kv.readOptions, []byte(key))
+ if err != nil {
+ return []string{}, err
+ }
+ values = append(values, string(value.Data()))
+ }
+ return values, nil
+}
+
+func (kv *RocksdbKV) Save(key, value string) error {
+ err := kv.db.Put(kv.writeOptions, []byte(key), []byte(value))
+ return err
+}
+
+func (kv *RocksdbKV) MultiSave(kvs map[string]string) error {
+ writeBatch := gorocksdb.NewWriteBatch()
+ defer writeBatch.Clear()
+ for k, v := range kvs {
+ writeBatch.Put([]byte(k), []byte(v))
+ }
+ err := kv.db.Write(kv.writeOptions, writeBatch)
+ return err
+}
+
+func (kv *RocksdbKV) RemoveWithPrefix(prefix string) error {
+ iter := kv.db.NewIterator(kv.readOptions)
+ iter.Seek([]byte(prefix))
+ for ; iter.Valid(); iter.Next() {
+ key := iter.Key()
+ err := kv.db.Delete(kv.writeOptions, key.Data())
+ if err != nil {
+ return nil
+ }
+ }
+ if err := iter.Err(); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (kv *RocksdbKV) Remove(key string) error {
+ err := kv.db.Delete(kv.writeOptions, []byte(key))
+ return err
+}
+
+func (kv *RocksdbKV) MultiRemove(keys []string) error {
+ writeBatch := gorocksdb.NewWriteBatch()
+ defer writeBatch.Clear()
+ for _, key := range keys {
+ writeBatch.Delete([]byte(key))
+ }
+ err := kv.db.Write(kv.writeOptions, writeBatch)
+ return err
+}
diff --git a/internal/kv/rocksdb/rocksdb_kv_test.go b/internal/kv/rocksdb/rocksdb_kv_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..8898415a0594ac7a996bf85efd19b552923ae510
--- /dev/null
+++ b/internal/kv/rocksdb/rocksdb_kv_test.go
@@ -0,0 +1,54 @@
+package rocksdbkv_test
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ rocksdbkv "github.com/zilliztech/milvus-distributed/internal/kv/rocksdb"
+)
+
+func TestRocksdbKV(t *testing.T) {
+ name := "/tmp/rocksdb"
+ rocksdbKV, err := rocksdbkv.NewRocksdbKV(name)
+ if err != nil {
+ panic(err)
+ }
+
+ defer rocksdbKV.Close()
+ // Need to call RemoveWithPrefix
+ defer rocksdbKV.RemoveWithPrefix("")
+
+ err = rocksdbKV.Save("abc", "123")
+ assert.Nil(t, err)
+
+ err = rocksdbKV.Save("abcd", "1234")
+ assert.Nil(t, err)
+
+ val, err := rocksdbKV.Load("abc")
+ assert.Nil(t, err)
+ assert.Equal(t, val, "123")
+
+ keys, vals, err := rocksdbKV.LoadWithPrefix("abc")
+ assert.Nil(t, err)
+ assert.Equal(t, len(keys), len(vals))
+ assert.Equal(t, len(keys), 2)
+
+ assert.Equal(t, keys[0], "abc")
+ assert.Equal(t, keys[1], "abcd")
+ assert.Equal(t, vals[0], "123")
+ assert.Equal(t, vals[1], "1234")
+
+ err = rocksdbKV.Save("key_1", "123")
+ assert.Nil(t, err)
+ err = rocksdbKV.Save("key_2", "456")
+ assert.Nil(t, err)
+ err = rocksdbKV.Save("key_3", "789")
+ assert.Nil(t, err)
+
+ keys = []string{"key_1", "key_2"}
+ vals, err = rocksdbKV.MultiLoad(keys)
+ assert.Nil(t, err)
+ assert.Equal(t, len(vals), len(keys))
+ assert.Equal(t, vals[0], "123")
+ assert.Equal(t, vals[1], "456")
+}
diff --git a/internal/master/client.go b/internal/master/client.go
index a35151767620339a0d20a6b11e3f3ff88f822328..88e44d8f70a02aba66217094d8f2327c6b06c814 100644
--- a/internal/master/client.go
+++ b/internal/master/client.go
@@ -1,7 +1,6 @@
package master
import (
- "sync"
"time"
buildindexclient "github.com/zilliztech/milvus-distributed/internal/indexbuilder/client"
@@ -21,12 +20,9 @@ type MockWriteNodeClient struct {
partitionTag string
timestamp Timestamp
collectionID UniqueID
- lock sync.RWMutex
}
func (m *MockWriteNodeClient) FlushSegment(segmentID UniqueID, collectionID UniqueID, partitionTag string, timestamp Timestamp) error {
- m.lock.Lock()
- defer m.lock.Unlock()
m.flushTime = time.Now()
m.segmentID = segmentID
m.collectionID = collectionID
@@ -37,8 +33,6 @@ func (m *MockWriteNodeClient) FlushSegment(segmentID UniqueID, collectionID Uniq
func (m *MockWriteNodeClient) DescribeSegment(segmentID UniqueID) (*writerclient.SegmentDescription, error) {
now := time.Now()
- m.lock.RLock()
- defer m.lock.RUnlock()
if now.Sub(m.flushTime).Seconds() > 2 {
return &writerclient.SegmentDescription{
SegmentID: segmentID,
diff --git a/internal/master/index_task.go b/internal/master/index_task.go
index c5f320d0db7d01f1abe5de3bd3e67391288b4d38..444b777c9f36691a83b773526cfcec87296a3150 100644
--- a/internal/master/index_task.go
+++ b/internal/master/index_task.go
@@ -24,6 +24,11 @@ func (task *createIndexTask) Ts() (Timestamp, error) {
}
func (task *createIndexTask) Execute() error {
+ // modify schema
+ if err := task.mt.UpdateFieldIndexParams(task.req.CollectionName, task.req.FieldName, task.req.ExtraParams); err != nil {
+ return err
+ }
+ // check if closed segment has the same index build history
collMeta, err := task.mt.GetCollectionByName(task.req.CollectionName)
if err != nil {
return err
@@ -39,20 +44,6 @@ func (task *createIndexTask) Execute() error {
return fmt.Errorf("can not find field name %s", task.req.FieldName)
}
- // pre checks
- isIndexable, err := task.mt.IsIndexable(collMeta.ID, fieldID)
- if err != nil {
- return err
- }
- if !isIndexable {
- return fmt.Errorf("field %s is not vector", task.req.FieldName)
- }
-
- // modify schema
- if err := task.mt.UpdateFieldIndexParams(task.req.CollectionName, task.req.FieldName, task.req.ExtraParams); err != nil {
- return err
- }
- // check if closed segment has the same index build history
for _, segID := range collMeta.SegmentIDs {
segMeta, err := task.mt.GetSegmentByID(segID)
if err != nil {
diff --git a/internal/master/master.go b/internal/master/master.go
index adb002adf804461a267412e6bd13bacfe16de96f..6ade92eaad31c50a8923bdf37d79ed429df7750b 100644
--- a/internal/master/master.go
+++ b/internal/master/master.go
@@ -218,6 +218,7 @@ func CreateServer(ctx context.Context) (*Master, error) {
m.grpcServer = grpc.NewServer()
masterpb.RegisterMasterServer(m.grpcServer, m)
+
return m, nil
}
diff --git a/internal/master/master_test.go b/internal/master/master_test.go
index 90440a699cc3adf007d4990a4b76a2179800a1b8..a605e73aa76127c24918cdd3826fae0d0d186ad8 100644
--- a/internal/master/master_test.go
+++ b/internal/master/master_test.go
@@ -65,8 +65,12 @@ func refreshChannelNames() {
}
func receiveTimeTickMsg(stream *ms.MsgStream) bool {
- result := (*stream).Consume()
- return result != nil
+ for {
+ result := (*stream).Consume()
+ if len(result.Msgs) > 0 {
+ return true
+ }
+ }
}
func getTimeTickMsgPack(ttmsgs [][2]uint64) *ms.MsgPack {
@@ -77,14 +81,6 @@ func getTimeTickMsgPack(ttmsgs [][2]uint64) *ms.MsgPack {
return &msgPack
}
-func mockTimeTickBroadCast(msgStream ms.MsgStream, time Timestamp) error {
- timeTick := [][2]uint64{
- {0, time},
- }
- ttMsgPackForDD := getTimeTickMsgPack(timeTick)
- return msgStream.Broadcast(ttMsgPackForDD)
-}
-
func TestMaster(t *testing.T) {
Init()
refreshMasterAddress()
@@ -114,6 +110,7 @@ func TestMaster(t *testing.T) {
conn, err := grpc.DialContext(ctx, Params.Address, grpc.WithInsecure(), grpc.WithBlock())
require.Nil(t, err)
+
cli := masterpb.NewMasterClient(conn)
t.Run("TestConfigTask", func(t *testing.T) {
@@ -537,15 +534,10 @@ func TestMaster(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS)
- time.Sleep(1000 * time.Millisecond)
- timestampNow := Timestamp(time.Now().Unix())
- err = mockTimeTickBroadCast(svr.timesSyncMsgProducer.ddSyncStream, timestampNow)
- assert.NoError(t, err)
-
//consume msg
- ddMs := ms.NewPulsarTtMsgStream(ctx, 1024)
+ ddMs := ms.NewPulsarMsgStream(ctx, 1024)
ddMs.SetPulsarClient(pulsarAddr)
- ddMs.CreatePulsarConsumers(Params.DDChannelNames, Params.MsgChannelSubName, ms.NewUnmarshalDispatcher(), 1024)
+ ddMs.CreatePulsarConsumers(Params.DDChannelNames, "DDStream", ms.NewUnmarshalDispatcher(), 1024)
ddMs.Start()
var consumeMsg ms.MsgStream = ddMs
@@ -831,16 +823,11 @@ func TestMaster(t *testing.T) {
assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS)
//consume msg
- ddMs := ms.NewPulsarTtMsgStream(ctx, 1024)
+ ddMs := ms.NewPulsarMsgStream(ctx, 1024)
ddMs.SetPulsarClient(pulsarAddr)
- ddMs.CreatePulsarConsumers(Params.DDChannelNames, Params.MsgChannelSubName, ms.NewUnmarshalDispatcher(), 1024)
+ ddMs.CreatePulsarConsumers(Params.DDChannelNames, "DDStream", ms.NewUnmarshalDispatcher(), 1024)
ddMs.Start()
- time.Sleep(1000 * time.Millisecond)
- timestampNow := Timestamp(time.Now().Unix())
- err = mockTimeTickBroadCast(svr.timesSyncMsgProducer.ddSyncStream, timestampNow)
- assert.NoError(t, err)
-
var consumeMsg ms.MsgStream = ddMs
for {
result := consumeMsg.Consume()
@@ -863,19 +850,19 @@ func TestMaster(t *testing.T) {
writeNodeStream.CreatePulsarProducers(Params.WriteNodeTimeTickChannelNames)
writeNodeStream.Start()
- ddMs := ms.NewPulsarTtMsgStream(ctx, 1024)
+ ddMs := ms.NewPulsarMsgStream(ctx, 1024)
ddMs.SetPulsarClient(pulsarAddr)
- ddMs.CreatePulsarConsumers(Params.DDChannelNames, Params.MsgChannelSubName, ms.NewUnmarshalDispatcher(), 1024)
+ ddMs.CreatePulsarConsumers(Params.DDChannelNames, "DDStream", ms.NewUnmarshalDispatcher(), 1024)
ddMs.Start()
- dMMs := ms.NewPulsarTtMsgStream(ctx, 1024)
+ dMMs := ms.NewPulsarMsgStream(ctx, 1024)
dMMs.SetPulsarClient(pulsarAddr)
- dMMs.CreatePulsarConsumers(Params.InsertChannelNames, Params.MsgChannelSubName, ms.NewUnmarshalDispatcher(), 1024)
+ dMMs.CreatePulsarConsumers(Params.InsertChannelNames, "DMStream", ms.NewUnmarshalDispatcher(), 1024)
dMMs.Start()
k2sMs := ms.NewPulsarMsgStream(ctx, 1024)
k2sMs.SetPulsarClient(pulsarAddr)
- k2sMs.CreatePulsarConsumers(Params.K2SChannelNames, Params.MsgChannelSubName, ms.NewUnmarshalDispatcher(), 1024)
+ k2sMs.CreatePulsarConsumers(Params.K2SChannelNames, "K2SStream", ms.NewUnmarshalDispatcher(), 1024)
k2sMs.Start()
ttsoftmsgs := [][2]uint64{
@@ -900,12 +887,6 @@ func TestMaster(t *testing.T) {
var k2sMsgstream ms.MsgStream = k2sMs
assert.True(t, receiveTimeTickMsg(&k2sMsgstream))
- conn, err := grpc.DialContext(ctx, Params.Address, grpc.WithInsecure(), grpc.WithBlock())
- assert.Nil(t, err)
- defer conn.Close()
-
- cli := masterpb.NewMasterClient(conn)
-
sch := schemapb.CollectionSchema{
Name: "name" + strconv.FormatUint(rand.Uint64(), 10),
Description: "test collection",
@@ -916,11 +897,10 @@ func TestMaster(t *testing.T) {
schemaBytes, err := proto.Marshal(&sch)
assert.Nil(t, err)
- ////////////////////////////CreateCollection////////////////////////
createCollectionReq := internalpb.CreateCollectionRequest{
MsgType: internalpb.MsgType_kCreateCollection,
ReqID: 1,
- Timestamp: Timestamp(time.Now().Unix()),
+ Timestamp: uint64(time.Now().Unix()),
ProxyID: 1,
Schema: &commonpb.Blob{Value: schemaBytes},
}
@@ -928,11 +908,6 @@ func TestMaster(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS)
- time.Sleep(1000 * time.Millisecond)
- timestampNow := Timestamp(time.Now().Unix())
- err = mockTimeTickBroadCast(svr.timesSyncMsgProducer.ddSyncStream, timestampNow)
- assert.NoError(t, err)
-
var consumeMsg ms.MsgStream = ddMs
var createCollectionMsg *ms.CreateCollectionMsg
for {
@@ -967,11 +942,6 @@ func TestMaster(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS)
- time.Sleep(1000 * time.Millisecond)
- timestampNow = Timestamp(time.Now().Unix())
- err = mockTimeTickBroadCast(svr.timesSyncMsgProducer.ddSyncStream, timestampNow)
- assert.NoError(t, err)
-
var createPartitionMsg *ms.CreatePartitionMsg
for {
result := consumeMsg.Consume()
@@ -1006,11 +976,6 @@ func TestMaster(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS)
- time.Sleep(1000 * time.Millisecond)
- timestampNow = Timestamp(time.Now().Unix())
- err = mockTimeTickBroadCast(svr.timesSyncMsgProducer.ddSyncStream, timestampNow)
- assert.NoError(t, err)
-
var dropPartitionMsg *ms.DropPartitionMsg
for {
result := consumeMsg.Consume()
@@ -1041,11 +1006,6 @@ func TestMaster(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, st.ErrorCode, commonpb.ErrorCode_SUCCESS)
- time.Sleep(1000 * time.Millisecond)
- timestampNow = Timestamp(time.Now().Unix())
- err = mockTimeTickBroadCast(svr.timesSyncMsgProducer.ddSyncStream, timestampNow)
- assert.NoError(t, err)
-
var dropCollectionMsg *ms.DropCollectionMsg
for {
result := consumeMsg.Consume()
diff --git a/internal/master/scheduler_test.go b/internal/master/scheduler_test.go
index f735a891c18544dfbecd4fc6ca5909457491688d..a40f7584fa5cea0616de4a60494aef3e869519e5 100644
--- a/internal/master/scheduler_test.go
+++ b/internal/master/scheduler_test.go
@@ -46,7 +46,7 @@ func TestMaster_Scheduler_Collection(t *testing.T) {
pulsarDDStream.Start()
defer pulsarDDStream.Close()
- consumeMs := ms.NewPulsarTtMsgStream(ctx, 1024)
+ consumeMs := ms.NewPulsarMsgStream(ctx, 1024)
consumeMs.SetPulsarClient(pulsarAddr)
consumeMs.CreatePulsarConsumers(consumerChannels, consumerSubName, ms.NewUnmarshalDispatcher(), 1024)
consumeMs.Start()
@@ -96,9 +96,6 @@ func TestMaster_Scheduler_Collection(t *testing.T) {
err = createCollectionTask.WaitToFinish(ctx)
assert.Nil(t, err)
- err = mockTimeTickBroadCast(pulsarDDStream, Timestamp(12))
- assert.NoError(t, err)
-
var consumeMsg ms.MsgStream = consumeMs
var createCollectionMsg *ms.CreateCollectionMsg
for {
@@ -121,7 +118,7 @@ func TestMaster_Scheduler_Collection(t *testing.T) {
dropCollectionReq := internalpb.DropCollectionRequest{
MsgType: internalpb.MsgType_kDropCollection,
ReqID: 1,
- Timestamp: 13,
+ Timestamp: 11,
ProxyID: 1,
CollectionName: &servicepb.CollectionName{CollectionName: sch.Name},
}
@@ -141,9 +138,6 @@ func TestMaster_Scheduler_Collection(t *testing.T) {
err = dropCollectionTask.WaitToFinish(ctx)
assert.Nil(t, err)
- err = mockTimeTickBroadCast(pulsarDDStream, Timestamp(14))
- assert.NoError(t, err)
-
var dropCollectionMsg *ms.DropCollectionMsg
for {
result := consumeMsg.Consume()
@@ -190,7 +184,7 @@ func TestMaster_Scheduler_Partition(t *testing.T) {
pulsarDDStream.Start()
defer pulsarDDStream.Close()
- consumeMs := ms.NewPulsarTtMsgStream(ctx, 1024)
+ consumeMs := ms.NewPulsarMsgStream(ctx, 1024)
consumeMs.SetPulsarClient(pulsarAddr)
consumeMs.CreatePulsarConsumers(consumerChannels, consumerSubName, ms.NewUnmarshalDispatcher(), 1024)
consumeMs.Start()
@@ -240,9 +234,6 @@ func TestMaster_Scheduler_Partition(t *testing.T) {
err = createCollectionTask.WaitToFinish(ctx)
assert.Nil(t, err)
- err = mockTimeTickBroadCast(pulsarDDStream, Timestamp(12))
- assert.NoError(t, err)
-
var consumeMsg ms.MsgStream = consumeMs
var createCollectionMsg *ms.CreateCollectionMsg
for {
@@ -266,7 +257,7 @@ func TestMaster_Scheduler_Partition(t *testing.T) {
createPartitionReq := internalpb.CreatePartitionRequest{
MsgType: internalpb.MsgType_kCreatePartition,
ReqID: 1,
- Timestamp: 13,
+ Timestamp: 11,
ProxyID: 1,
PartitionName: &servicepb.PartitionName{
CollectionName: sch.Name,
@@ -288,9 +279,6 @@ func TestMaster_Scheduler_Partition(t *testing.T) {
err = createPartitionTask.WaitToFinish(ctx)
assert.Nil(t, err)
- err = mockTimeTickBroadCast(pulsarDDStream, Timestamp(14))
- assert.NoError(t, err)
-
var createPartitionMsg *ms.CreatePartitionMsg
for {
result := consumeMsg.Consume()
@@ -313,7 +301,7 @@ func TestMaster_Scheduler_Partition(t *testing.T) {
dropPartitionReq := internalpb.DropPartitionRequest{
MsgType: internalpb.MsgType_kDropPartition,
ReqID: 1,
- Timestamp: 15,
+ Timestamp: 11,
ProxyID: 1,
PartitionName: &servicepb.PartitionName{
CollectionName: sch.Name,
@@ -335,9 +323,6 @@ func TestMaster_Scheduler_Partition(t *testing.T) {
err = dropPartitionTask.WaitToFinish(ctx)
assert.Nil(t, err)
- err = mockTimeTickBroadCast(pulsarDDStream, Timestamp(16))
- assert.NoError(t, err)
-
var dropPartitionMsg *ms.DropPartitionMsg
for {
result := consumeMsg.Consume()
diff --git a/internal/master/segment_manager_test.go b/internal/master/segment_manager_test.go
index 69078054d541145a32e09aaf74c04dc42aa162bf..28e16ec7eca16dacc512e598b769dfbf0dad4a34 100644
--- a/internal/master/segment_manager_test.go
+++ b/internal/master/segment_manager_test.go
@@ -126,7 +126,7 @@ func TestSegmentManager_AssignSegment(t *testing.T) {
}
}
- time.Sleep(time.Duration(Params.SegIDAssignExpiration) * time.Millisecond)
+ time.Sleep(time.Duration(Params.SegIDAssignExpiration))
timestamp, err := globalTsoAllocator()
assert.Nil(t, err)
err = mt.UpdateSegment(&pb.SegmentMeta{
@@ -156,124 +156,3 @@ func TestSegmentManager_AssignSegment(t *testing.T) {
assert.Nil(t, err)
assert.NotEqualValues(t, 0, segMeta.CloseTime)
}
-
-func TestSegmentManager_SycnWritenode(t *testing.T) {
- ctx, cancelFunc := context.WithCancel(context.TODO())
- defer cancelFunc()
-
- Init()
- Params.TopicNum = 5
- Params.QueryNodeNum = 3
- Params.SegmentSize = 536870912 / 1024 / 1024
- Params.SegmentSizeFactor = 0.75
- Params.DefaultRecordSize = 1024
- Params.MinSegIDAssignCnt = 1048576 / 1024
- Params.SegIDAssignExpiration = 2000
- etcdAddress := Params.EtcdAddress
- cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}})
- assert.Nil(t, err)
- rootPath := "/test/root"
- _, err = cli.Delete(ctx, rootPath, clientv3.WithPrefix())
- assert.Nil(t, err)
-
- kvBase := etcdkv.NewEtcdKV(cli, rootPath)
- defer kvBase.Close()
- mt, err := NewMetaTable(kvBase)
- assert.Nil(t, err)
-
- collName := "segmgr_test_coll"
- var collID int64 = 1001
- partitionTag := "test_part"
- schema := &schemapb.CollectionSchema{
- Name: collName,
- Fields: []*schemapb.FieldSchema{
- {FieldID: 1, Name: "f1", IsPrimaryKey: false, DataType: schemapb.DataType_INT32},
- {FieldID: 2, Name: "f2", IsPrimaryKey: false, DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{
- {Key: "dim", Value: "128"},
- }},
- },
- }
- err = mt.AddCollection(&pb.CollectionMeta{
- ID: collID,
- Schema: schema,
- CreateTime: 0,
- SegmentIDs: []UniqueID{},
- PartitionTags: []string{},
- })
- assert.Nil(t, err)
- err = mt.AddPartition(collID, partitionTag)
- assert.Nil(t, err)
-
- var cnt int64
- globalIDAllocator := func() (UniqueID, error) {
- val := atomic.AddInt64(&cnt, 1)
- return val, nil
- }
- globalTsoAllocator := func() (Timestamp, error) {
- val := atomic.AddInt64(&cnt, 1)
- phy := time.Now().UnixNano() / int64(time.Millisecond)
- ts := tsoutil.ComposeTS(phy, val)
- return ts, nil
- }
- syncWriteChan := make(chan *msgstream.TimeTickMsg)
- syncProxyChan := make(chan *msgstream.TimeTickMsg)
-
- segAssigner := NewSegmentAssigner(ctx, mt, globalTsoAllocator, syncProxyChan)
- mockScheduler := &MockFlushScheduler{}
- segManager, err := NewSegmentManager(ctx, mt, globalIDAllocator, globalTsoAllocator, syncWriteChan, mockScheduler, segAssigner)
- assert.Nil(t, err)
-
- segManager.Start()
- defer segManager.Close()
- sizePerRecord, err := typeutil.EstimateSizePerRecord(schema)
- assert.Nil(t, err)
- maxCount := uint32(Params.SegmentSize * 1024 * 1024 / float64(sizePerRecord))
-
- req := []*internalpb.SegIDRequest{
- {Count: maxCount, ChannelID: 1, CollName: collName, PartitionTag: partitionTag},
- {Count: maxCount, ChannelID: 2, CollName: collName, PartitionTag: partitionTag},
- {Count: maxCount, ChannelID: 3, CollName: collName, PartitionTag: partitionTag},
- }
- assignSegment, err := segManager.AssignSegment(req)
- assert.Nil(t, err)
- timestamp, err := globalTsoAllocator()
- assert.Nil(t, err)
- for i := 0; i < len(assignSegment); i++ {
- assert.EqualValues(t, maxCount, assignSegment[i].Count)
- assert.EqualValues(t, i+1, assignSegment[i].ChannelID)
-
- err = mt.UpdateSegment(&pb.SegmentMeta{
- SegmentID: assignSegment[i].SegID,
- CollectionID: collID,
- PartitionTag: partitionTag,
- ChannelStart: 0,
- ChannelEnd: 1,
- CloseTime: timestamp,
- NumRows: int64(maxCount),
- MemSize: 500000,
- })
- assert.Nil(t, err)
- }
-
- time.Sleep(time.Duration(Params.SegIDAssignExpiration) * time.Millisecond)
-
- timestamp, err = globalTsoAllocator()
- assert.Nil(t, err)
- tsMsg := &msgstream.TimeTickMsg{
- BaseMsg: msgstream.BaseMsg{
- BeginTimestamp: timestamp, EndTimestamp: timestamp, HashValues: []uint32{},
- },
- TimeTickMsg: internalpb.TimeTickMsg{
- MsgType: internalpb.MsgType_kTimeTick,
- PeerID: 1,
- Timestamp: timestamp,
- },
- }
- syncWriteChan <- tsMsg
- time.Sleep(300 * time.Millisecond)
-
- segManager.mu.RLock()
- defer segManager.mu.RUnlock()
- status := segManager.collStatus[collID]
- assert.Empty(t, status.segments)
-}
diff --git a/internal/master/time_snyc_producer_test.go b/internal/master/time_snyc_producer_test.go
index 3c0cc2e9aa02bd501df86aa296309d846121c021..e55b1ec42756e9014d738d4850a16b6eda75d945 100644
--- a/internal/master/time_snyc_producer_test.go
+++ b/internal/master/time_snyc_producer_test.go
@@ -58,7 +58,6 @@ func initTestPulsarStream(ctx context.Context, pulsarAddress string,
return &input, &output
}
-
func receiveMsg(stream *ms.MsgStream) []uint64 {
receiveCount := 0
var results []uint64
diff --git a/internal/master/timesync.go b/internal/master/timesync.go
index 79863f7ac8db2f6f585b16b81f355e54d3ec85d9..49c388a8dd9249349777a99a3f54fd9b5a2ea472 100644
--- a/internal/master/timesync.go
+++ b/internal/master/timesync.go
@@ -81,7 +81,7 @@ func (ttBarrier *softTimeTickBarrier) Start() error {
// get a legal Timestamp
ts := ttBarrier.minTimestamp()
lastTt := atomic.LoadInt64(&(ttBarrier.lastTt))
- if lastTt != 0 && ttBarrier.minTtInterval > ts-Timestamp(lastTt) {
+ if ttBarrier.lastTt != 0 && ttBarrier.minTtInterval > ts-Timestamp(lastTt) {
continue
}
ttBarrier.outTt <- ts
diff --git a/internal/master/timesync_test.go b/internal/master/timesync_test.go
index cab1c740273ab75cfd31f103746e35a75a5e4d4a..59fb7b27621c8a523e023d3a10f747066eed3876 100644
--- a/internal/master/timesync_test.go
+++ b/internal/master/timesync_test.go
@@ -192,15 +192,15 @@ func TestTt_SoftTtBarrierStart(t *testing.T) {
func TestTt_SoftTtBarrierGetTimeTickClose(t *testing.T) {
channels := []string{"SoftTtBarrierGetTimeTickClose"}
- //ttmsgs := [][2]int{
- // {1, 10},
- // {2, 20},
- // {3, 30},
- // {4, 40},
- // {1, 30},
- // {2, 30},
- //}
- inStream, ttStream := producer(channels, nil)
+ ttmsgs := [][2]int{
+ {1, 10},
+ {2, 20},
+ {3, 30},
+ {4, 40},
+ {1, 30},
+ {2, 30},
+ }
+ inStream, ttStream := producer(channels, ttmsgs)
defer func() {
(*inStream).Close()
(*ttStream).Close()
@@ -259,15 +259,15 @@ func TestTt_SoftTtBarrierGetTimeTickClose(t *testing.T) {
func TestTt_SoftTtBarrierGetTimeTickCancel(t *testing.T) {
channels := []string{"SoftTtBarrierGetTimeTickCancel"}
- //ttmsgs := [][2]int{
- // {1, 10},
- // {2, 20},
- // {3, 30},
- // {4, 40},
- // {1, 30},
- // {2, 30},
- //}
- inStream, ttStream := producer(channels, nil)
+ ttmsgs := [][2]int{
+ {1, 10},
+ {2, 20},
+ {3, 30},
+ {4, 40},
+ {1, 30},
+ {2, 30},
+ }
+ inStream, ttStream := producer(channels, ttmsgs)
defer func() {
(*inStream).Close()
(*ttStream).Close()
diff --git a/internal/msgstream/msg.go b/internal/msgstream/msg.go
index 518bcfa7afe56a34cf86ff1464eb309a42560a9c..3b8f4dacd7ccd5b313e29701852df800fb716ef5 100644
--- a/internal/msgstream/msg.go
+++ b/internal/msgstream/msg.go
@@ -1,6 +1,8 @@
package msgstream
import (
+ "context"
+
"github.com/golang/protobuf/proto"
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
)
@@ -8,6 +10,8 @@ import (
type MsgType = internalPb.MsgType
type TsMsg interface {
+ GetMsgContext() context.Context
+ SetMsgContext(context.Context)
BeginTs() Timestamp
EndTs() Timestamp
Type() MsgType
@@ -17,6 +21,7 @@ type TsMsg interface {
}
type BaseMsg struct {
+ MsgCtx context.Context
BeginTimestamp Timestamp
EndTimestamp Timestamp
HashValues []uint32
@@ -44,6 +49,14 @@ func (it *InsertMsg) Type() MsgType {
return it.MsgType
}
+func (it *InsertMsg) GetMsgContext() context.Context {
+ return it.MsgCtx
+}
+
+func (it *InsertMsg) SetMsgContext(ctx context.Context) {
+ it.MsgCtx = ctx
+}
+
func (it *InsertMsg) Marshal(input TsMsg) ([]byte, error) {
insertMsg := input.(*InsertMsg)
insertRequest := &insertMsg.InsertRequest
@@ -88,6 +101,13 @@ func (fl *FlushMsg) Type() MsgType {
return fl.GetMsgType()
}
+func (fl *FlushMsg) GetMsgContext() context.Context {
+ return fl.MsgCtx
+}
+func (fl *FlushMsg) SetMsgContext(ctx context.Context) {
+ fl.MsgCtx = ctx
+}
+
func (fl *FlushMsg) Marshal(input TsMsg) ([]byte, error) {
flushMsgTask := input.(*FlushMsg)
flushMsg := &flushMsgTask.FlushMsg
@@ -121,6 +141,14 @@ func (dt *DeleteMsg) Type() MsgType {
return dt.MsgType
}
+func (dt *DeleteMsg) GetMsgContext() context.Context {
+ return dt.MsgCtx
+}
+
+func (dt *DeleteMsg) SetMsgContext(ctx context.Context) {
+ dt.MsgCtx = ctx
+}
+
func (dt *DeleteMsg) Marshal(input TsMsg) ([]byte, error) {
deleteTask := input.(*DeleteMsg)
deleteRequest := &deleteTask.DeleteRequest
@@ -165,6 +193,14 @@ func (st *SearchMsg) Type() MsgType {
return st.MsgType
}
+func (st *SearchMsg) GetMsgContext() context.Context {
+ return st.MsgCtx
+}
+
+func (st *SearchMsg) SetMsgContext(ctx context.Context) {
+ st.MsgCtx = ctx
+}
+
func (st *SearchMsg) Marshal(input TsMsg) ([]byte, error) {
searchTask := input.(*SearchMsg)
searchRequest := &searchTask.SearchRequest
@@ -198,6 +234,14 @@ func (srt *SearchResultMsg) Type() MsgType {
return srt.MsgType
}
+func (srt *SearchResultMsg) GetMsgContext() context.Context {
+ return srt.MsgCtx
+}
+
+func (srt *SearchResultMsg) SetMsgContext(ctx context.Context) {
+ srt.MsgCtx = ctx
+}
+
func (srt *SearchResultMsg) Marshal(input TsMsg) ([]byte, error) {
searchResultTask := input.(*SearchResultMsg)
searchResultRequest := &searchResultTask.SearchResult
@@ -231,6 +275,14 @@ func (tst *TimeTickMsg) Type() MsgType {
return tst.MsgType
}
+func (tst *TimeTickMsg) GetMsgContext() context.Context {
+ return tst.MsgCtx
+}
+
+func (tst *TimeTickMsg) SetMsgContext(ctx context.Context) {
+ tst.MsgCtx = ctx
+}
+
func (tst *TimeTickMsg) Marshal(input TsMsg) ([]byte, error) {
timeTickTask := input.(*TimeTickMsg)
timeTick := &timeTickTask.TimeTickMsg
@@ -264,6 +316,14 @@ func (qs *QueryNodeStatsMsg) Type() MsgType {
return qs.MsgType
}
+func (qs *QueryNodeStatsMsg) GetMsgContext() context.Context {
+ return qs.MsgCtx
+}
+
+func (qs *QueryNodeStatsMsg) SetMsgContext(ctx context.Context) {
+ qs.MsgCtx = ctx
+}
+
func (qs *QueryNodeStatsMsg) Marshal(input TsMsg) ([]byte, error) {
queryNodeSegStatsTask := input.(*QueryNodeStatsMsg)
queryNodeSegStats := &queryNodeSegStatsTask.QueryNodeStats
@@ -305,6 +365,14 @@ func (cc *CreateCollectionMsg) Type() MsgType {
return cc.MsgType
}
+func (cc *CreateCollectionMsg) GetMsgContext() context.Context {
+ return cc.MsgCtx
+}
+
+func (cc *CreateCollectionMsg) SetMsgContext(ctx context.Context) {
+ cc.MsgCtx = ctx
+}
+
func (cc *CreateCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
createCollectionMsg := input.(*CreateCollectionMsg)
createCollectionRequest := &createCollectionMsg.CreateCollectionRequest
@@ -337,6 +405,13 @@ type DropCollectionMsg struct {
func (dc *DropCollectionMsg) Type() MsgType {
return dc.MsgType
}
+func (dc *DropCollectionMsg) GetMsgContext() context.Context {
+ return dc.MsgCtx
+}
+
+func (dc *DropCollectionMsg) SetMsgContext(ctx context.Context) {
+ dc.MsgCtx = ctx
+}
func (dc *DropCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
dropCollectionMsg := input.(*DropCollectionMsg)
@@ -361,109 +436,18 @@ func (dc *DropCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
return dropCollectionMsg, nil
}
-/////////////////////////////////////////HasCollection//////////////////////////////////////////
-type HasCollectionMsg struct {
- BaseMsg
- internalPb.HasCollectionRequest
-}
-
-func (hc *HasCollectionMsg) Type() MsgType {
- return hc.MsgType
-}
-
-func (hc *HasCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
- hasCollectionMsg := input.(*HasCollectionMsg)
- hasCollectionRequest := &hasCollectionMsg.HasCollectionRequest
- mb, err := proto.Marshal(hasCollectionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (hc *HasCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
- hasCollectionRequest := internalPb.HasCollectionRequest{}
- err := proto.Unmarshal(input, &hasCollectionRequest)
- if err != nil {
- return nil, err
- }
- hasCollectionMsg := &HasCollectionMsg{HasCollectionRequest: hasCollectionRequest}
- hasCollectionMsg.BeginTimestamp = hasCollectionMsg.Timestamp
- hasCollectionMsg.EndTimestamp = hasCollectionMsg.Timestamp
-
- return hasCollectionMsg, nil
-}
-
-/////////////////////////////////////////DescribeCollection//////////////////////////////////////////
-type DescribeCollectionMsg struct {
- BaseMsg
- internalPb.DescribeCollectionRequest
-}
-
-func (dc *DescribeCollectionMsg) Type() MsgType {
- return dc.MsgType
-}
-
-func (dc *DescribeCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
- describeCollectionMsg := input.(*DescribeCollectionMsg)
- describeCollectionRequest := &describeCollectionMsg.DescribeCollectionRequest
- mb, err := proto.Marshal(describeCollectionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (dc *DescribeCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
- describeCollectionRequest := internalPb.DescribeCollectionRequest{}
- err := proto.Unmarshal(input, &describeCollectionRequest)
- if err != nil {
- return nil, err
- }
- describeCollectionMsg := &DescribeCollectionMsg{DescribeCollectionRequest: describeCollectionRequest}
- describeCollectionMsg.BeginTimestamp = describeCollectionMsg.Timestamp
- describeCollectionMsg.EndTimestamp = describeCollectionMsg.Timestamp
-
- return describeCollectionMsg, nil
-}
-
-/////////////////////////////////////////ShowCollection//////////////////////////////////////////
-type ShowCollectionMsg struct {
+/////////////////////////////////////////CreatePartition//////////////////////////////////////////
+type CreatePartitionMsg struct {
BaseMsg
- internalPb.ShowCollectionRequest
-}
-
-func (sc *ShowCollectionMsg) Type() MsgType {
- return sc.MsgType
-}
-
-func (sc *ShowCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
- showCollectionMsg := input.(*ShowCollectionMsg)
- showCollectionRequest := &showCollectionMsg.ShowCollectionRequest
- mb, err := proto.Marshal(showCollectionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
+ internalPb.CreatePartitionRequest
}
-func (sc *ShowCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
- showCollectionRequest := internalPb.ShowCollectionRequest{}
- err := proto.Unmarshal(input, &showCollectionRequest)
- if err != nil {
- return nil, err
- }
- showCollectionMsg := &ShowCollectionMsg{ShowCollectionRequest: showCollectionRequest}
- showCollectionMsg.BeginTimestamp = showCollectionMsg.Timestamp
- showCollectionMsg.EndTimestamp = showCollectionMsg.Timestamp
-
- return showCollectionMsg, nil
+func (cc *CreatePartitionMsg) GetMsgContext() context.Context {
+ return cc.MsgCtx
}
-/////////////////////////////////////////CreatePartition//////////////////////////////////////////
-type CreatePartitionMsg struct {
- BaseMsg
- internalPb.CreatePartitionRequest
+func (cc *CreatePartitionMsg) SetMsgContext(ctx context.Context) {
+ cc.MsgCtx = ctx
}
func (cc *CreatePartitionMsg) Type() MsgType {
@@ -499,6 +483,14 @@ type DropPartitionMsg struct {
internalPb.DropPartitionRequest
}
+func (dc *DropPartitionMsg) GetMsgContext() context.Context {
+ return dc.MsgCtx
+}
+
+func (dc *DropPartitionMsg) SetMsgContext(ctx context.Context) {
+ dc.MsgCtx = ctx
+}
+
func (dc *DropPartitionMsg) Type() MsgType {
return dc.MsgType
}
@@ -526,105 +518,6 @@ func (dc *DropPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
return dropPartitionMsg, nil
}
-/////////////////////////////////////////HasPartition//////////////////////////////////////////
-type HasPartitionMsg struct {
- BaseMsg
- internalPb.HasPartitionRequest
-}
-
-func (hc *HasPartitionMsg) Type() MsgType {
- return hc.MsgType
-}
-
-func (hc *HasPartitionMsg) Marshal(input TsMsg) ([]byte, error) {
- hasPartitionMsg := input.(*HasPartitionMsg)
- hasPartitionRequest := &hasPartitionMsg.HasPartitionRequest
- mb, err := proto.Marshal(hasPartitionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (hc *HasPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
- hasPartitionRequest := internalPb.HasPartitionRequest{}
- err := proto.Unmarshal(input, &hasPartitionRequest)
- if err != nil {
- return nil, err
- }
- hasPartitionMsg := &HasPartitionMsg{HasPartitionRequest: hasPartitionRequest}
- hasPartitionMsg.BeginTimestamp = hasPartitionMsg.Timestamp
- hasPartitionMsg.EndTimestamp = hasPartitionMsg.Timestamp
-
- return hasPartitionMsg, nil
-}
-
-/////////////////////////////////////////DescribePartition//////////////////////////////////////////
-type DescribePartitionMsg struct {
- BaseMsg
- internalPb.DescribePartitionRequest
-}
-
-func (dc *DescribePartitionMsg) Type() MsgType {
- return dc.MsgType
-}
-
-func (dc *DescribePartitionMsg) Marshal(input TsMsg) ([]byte, error) {
- describePartitionMsg := input.(*DescribePartitionMsg)
- describePartitionRequest := &describePartitionMsg.DescribePartitionRequest
- mb, err := proto.Marshal(describePartitionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (dc *DescribePartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
- describePartitionRequest := internalPb.DescribePartitionRequest{}
- err := proto.Unmarshal(input, &describePartitionRequest)
- if err != nil {
- return nil, err
- }
- describePartitionMsg := &DescribePartitionMsg{DescribePartitionRequest: describePartitionRequest}
- describePartitionMsg.BeginTimestamp = describePartitionMsg.Timestamp
- describePartitionMsg.EndTimestamp = describePartitionMsg.Timestamp
-
- return describePartitionMsg, nil
-}
-
-/////////////////////////////////////////ShowPartition//////////////////////////////////////////
-type ShowPartitionMsg struct {
- BaseMsg
- internalPb.ShowPartitionRequest
-}
-
-func (sc *ShowPartitionMsg) Type() MsgType {
- return sc.MsgType
-}
-
-func (sc *ShowPartitionMsg) Marshal(input TsMsg) ([]byte, error) {
- showPartitionMsg := input.(*ShowPartitionMsg)
- showPartitionRequest := &showPartitionMsg.ShowPartitionRequest
- mb, err := proto.Marshal(showPartitionRequest)
- if err != nil {
- return nil, err
- }
- return mb, nil
-}
-
-func (sc *ShowPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
- showPartitionRequest := internalPb.ShowPartitionRequest{}
- err := proto.Unmarshal(input, &showPartitionRequest)
- if err != nil {
- return nil, err
- }
- showPartitionMsg := &ShowPartitionMsg{ShowPartitionRequest: showPartitionRequest}
- showPartitionMsg.BeginTimestamp = showPartitionMsg.Timestamp
- showPartitionMsg.EndTimestamp = showPartitionMsg.Timestamp
-
- return showPartitionMsg, nil
-}
-
/////////////////////////////////////////LoadIndex//////////////////////////////////////////
type LoadIndexMsg struct {
BaseMsg
@@ -635,6 +528,14 @@ func (lim *LoadIndexMsg) Type() MsgType {
return lim.MsgType
}
+func (lim *LoadIndexMsg) GetMsgContext() context.Context {
+ return lim.MsgCtx
+}
+
+func (lim *LoadIndexMsg) SetMsgContext(ctx context.Context) {
+ lim.MsgCtx = ctx
+}
+
func (lim *LoadIndexMsg) Marshal(input TsMsg) ([]byte, error) {
loadIndexMsg := input.(*LoadIndexMsg)
loadIndexRequest := &loadIndexMsg.LoadIndex
diff --git a/internal/msgstream/msgstream.go b/internal/msgstream/msgstream.go
index bfb212813b23b932d331cf3ba2be5f26c51a6c57..7fe0b58f1c0907d7ff12acaf7b365bd535449f5b 100644
--- a/internal/msgstream/msgstream.go
+++ b/internal/msgstream/msgstream.go
@@ -4,12 +4,15 @@ import (
"context"
"log"
"reflect"
+ "strings"
"sync"
"time"
"github.com/apache/pulsar-client-go/pulsar"
"github.com/golang/protobuf/proto"
-
+ "github.com/opentracing/opentracing-go"
+ "github.com/opentracing/opentracing-go/ext"
+ oplog "github.com/opentracing/opentracing-go/log"
"github.com/zilliztech/milvus-distributed/internal/errors"
commonPb "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
@@ -151,15 +154,35 @@ func (ms *PulsarMsgStream) Close() {
}
}
+type propertiesReaderWriter struct {
+ ppMap map[string]string
+}
+
+func (ppRW *propertiesReaderWriter) Set(key, val string) {
+ // The GRPC HPACK implementation rejects any uppercase keys here.
+ //
+ // As such, since the HTTP_HEADERS format is case-insensitive anyway, we
+ // blindly lowercase the key (which is guaranteed to work in the
+ // Inject/Extract sense per the OpenTracing spec).
+ key = strings.ToLower(key)
+ ppRW.ppMap[key] = val
+}
+
+func (ppRW *propertiesReaderWriter) ForeachKey(handler func(key, val string) error) error {
+ for k, val := range ppRW.ppMap {
+ if err := handler(k, val); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
tsMsgs := msgPack.Msgs
if len(tsMsgs) <= 0 {
log.Printf("Warning: Receive empty msgPack")
return nil
}
- if len(ms.producers) <= 0 {
- return errors.New("nil producer in msg stream")
- }
reBucketValues := make([][]int32, len(tsMsgs))
for channelID, tsMsg := range tsMsgs {
hashValues := tsMsg.HashKeys()
@@ -203,12 +226,51 @@ func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
if err != nil {
return err
}
+
+ msg := &pulsar.ProducerMessage{Payload: mb}
+ var child opentracing.Span
+ if v.Msgs[i].Type() == internalPb.MsgType_kInsert ||
+ v.Msgs[i].Type() == internalPb.MsgType_kSearch ||
+ v.Msgs[i].Type() == internalPb.MsgType_kSearchResult {
+ tracer := opentracing.GlobalTracer()
+ ctx := v.Msgs[i].GetMsgContext()
+ if ctx == nil {
+ ctx = context.Background()
+ }
+
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan("start send pulsar msg",
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan("start send pulsar msg")
+ }
+ child.SetTag("hash keys", v.Msgs[i].HashKeys())
+ child.SetTag("start time", v.Msgs[i].BeginTs())
+ child.SetTag("end time", v.Msgs[i].EndTs())
+ child.SetTag("msg type", v.Msgs[i].Type())
+ msg.Properties = make(map[string]string)
+ err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
+ if err != nil {
+ child.LogFields(oplog.Error(err))
+ child.Finish()
+ return err
+ }
+ child.LogFields(oplog.String("inject success", "inject success"))
+ }
+
if _, err := (*ms.producers[k]).Send(
context.Background(),
- &pulsar.ProducerMessage{Payload: mb},
+ msg,
); err != nil {
+ if child != nil {
+ child.LogFields(oplog.Error(err))
+ child.Finish()
+ }
return err
}
+ if child != nil {
+ child.Finish()
+ }
}
}
return nil
@@ -221,14 +283,50 @@ func (ms *PulsarMsgStream) Broadcast(msgPack *MsgPack) error {
if err != nil {
return err
}
+ msg := &pulsar.ProducerMessage{Payload: mb}
+ var child opentracing.Span
+ if v.Type() == internalPb.MsgType_kInsert ||
+ v.Type() == internalPb.MsgType_kSearch ||
+ v.Type() == internalPb.MsgType_kSearchResult {
+ tracer := opentracing.GlobalTracer()
+ ctx := v.GetMsgContext()
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan("start send pulsar msg",
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan("start send pulsar msg, start time: %d")
+ }
+ child.SetTag("hash keys", v.HashKeys())
+ child.SetTag("start time", v.BeginTs())
+ child.SetTag("end time", v.EndTs())
+ child.SetTag("msg type", v.Type())
+ msg.Properties = make(map[string]string)
+ err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
+ if err != nil {
+ child.LogFields(oplog.Error(err))
+ child.Finish()
+ return err
+ }
+ child.LogFields(oplog.String("inject success", "inject success"))
+ }
for i := 0; i < producerLen; i++ {
if _, err := (*ms.producers[i]).Send(
context.Background(),
- &pulsar.ProducerMessage{Payload: mb},
+ msg,
); err != nil {
+ if child != nil {
+ child.LogFields(oplog.Error(err))
+ child.Finish()
+ }
return err
}
}
+ if child != nil {
+ child.Finish()
+ }
}
return nil
}
@@ -261,6 +359,7 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
for {
select {
case <-ms.ctx.Done():
+ log.Println("done")
return
default:
tsMsgList := make([]TsMsg, 0)
@@ -273,6 +372,7 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
}
pulsarMsg, ok := value.Interface().(pulsar.ConsumerMessage)
+
if !ok {
log.Printf("type assertion failed, not consumer message type")
continue
@@ -286,6 +386,23 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
continue
}
tsMsg, err := ms.unmarshal.Unmarshal(pulsarMsg.Payload(), headerMsg.MsgType)
+ if tsMsg.Type() == internalPb.MsgType_kSearch ||
+ tsMsg.Type() == internalPb.MsgType_kSearchResult {
+ tracer := opentracing.GlobalTracer()
+ spanContext, err := tracer.Extract(opentracing.HTTPHeaders, &propertiesReaderWriter{pulsarMsg.Properties()})
+ if err != nil {
+ log.Println("extract message err")
+ log.Println(err.Error())
+ }
+ span := opentracing.StartSpan("pulsar msg received",
+ ext.RPCServerOption(spanContext))
+ span.SetTag("msg type", tsMsg.Type())
+ span.SetTag("hash keys", tsMsg.HashKeys())
+ span.SetTag("start time", tsMsg.BeginTs())
+ span.SetTag("end time", tsMsg.EndTs())
+ tsMsg.SetMsgContext(opentracing.ContextWithSpan(context.Background(), span))
+ span.Finish()
+ }
if err != nil {
log.Printf("Failed to unmarshal tsMsg, error = %v", err)
continue
@@ -349,6 +466,8 @@ func (ms *PulsarTtMsgStream) bufMsgPackToChannel() {
ms.inputBuf = make([]TsMsg, 0)
isChannelReady := make([]bool, len(ms.consumers))
eofMsgTimeStamp := make(map[int]Timestamp)
+ spans := make(map[Timestamp]opentracing.Span)
+ ctxs := make(map[Timestamp]context.Context)
for {
select {
case <-ms.ctx.Done():
@@ -356,16 +475,15 @@ func (ms *PulsarTtMsgStream) bufMsgPackToChannel() {
default:
wg := sync.WaitGroup{}
mu := sync.Mutex{}
- findMapMutex := sync.RWMutex{}
for i := 0; i < len(ms.consumers); i++ {
if isChannelReady[i] {
continue
}
wg.Add(1)
- go ms.findTimeTick(i, eofMsgTimeStamp, &wg, &mu, &findMapMutex)
+ go ms.findTimeTick(i, eofMsgTimeStamp, &wg, &mu)
}
wg.Wait()
- timeStamp, ok := checkTimeTickMsg(eofMsgTimeStamp, isChannelReady, &findMapMutex)
+ timeStamp, ok := checkTimeTickMsg(eofMsgTimeStamp, isChannelReady)
if !ok || timeStamp <= ms.lastTimeStamp {
log.Printf("All timeTick's timestamps are inconsistent")
continue
@@ -375,8 +493,22 @@ func (ms *PulsarTtMsgStream) bufMsgPackToChannel() {
ms.inputBuf = append(ms.inputBuf, ms.unsolvedBuf...)
ms.unsolvedBuf = ms.unsolvedBuf[:0]
for _, v := range ms.inputBuf {
+ var ctx context.Context
+ var span opentracing.Span
+ if v.Type() == internalPb.MsgType_kInsert {
+ if _, ok := spans[v.BeginTs()]; !ok {
+ span, ctx = opentracing.StartSpanFromContext(v.GetMsgContext(), "after find time tick")
+ ctxs[v.BeginTs()] = ctx
+ spans[v.BeginTs()] = span
+ }
+ }
if v.EndTs() <= timeStamp {
timeTickBuf = append(timeTickBuf, v)
+ if v.Type() == internalPb.MsgType_kInsert {
+ v.SetMsgContext(ctxs[v.BeginTs()])
+ spans[v.BeginTs()].Finish()
+ delete(spans, v.BeginTs())
+ }
} else {
ms.unsolvedBuf = append(ms.unsolvedBuf, v)
}
@@ -398,8 +530,7 @@ func (ms *PulsarTtMsgStream) bufMsgPackToChannel() {
func (ms *PulsarTtMsgStream) findTimeTick(channelIndex int,
eofMsgMap map[int]Timestamp,
wg *sync.WaitGroup,
- mu *sync.Mutex,
- findMapMutex *sync.RWMutex) {
+ mu *sync.Mutex) {
defer wg.Done()
for {
select {
@@ -425,10 +556,26 @@ func (ms *PulsarTtMsgStream) findTimeTick(channelIndex int,
if err != nil {
log.Printf("Failed to unmarshal, error = %v", err)
}
+
+ if tsMsg.Type() == internalPb.MsgType_kInsert {
+ tracer := opentracing.GlobalTracer()
+ spanContext, err := tracer.Extract(opentracing.HTTPHeaders, &propertiesReaderWriter{pulsarMsg.Properties()})
+ if err != nil {
+ log.Println("extract message err")
+ log.Println(err.Error())
+ }
+ span := opentracing.StartSpan("pulsar msg received",
+ ext.RPCServerOption(spanContext))
+ span.SetTag("hash keys", tsMsg.HashKeys())
+ span.SetTag("start time", tsMsg.BeginTs())
+ span.SetTag("end time", tsMsg.EndTs())
+ span.SetTag("msg type", tsMsg.Type())
+ tsMsg.SetMsgContext(opentracing.ContextWithSpan(context.Background(), span))
+ span.Finish()
+ }
+
if headerMsg.MsgType == internalPb.MsgType_kTimeTick {
- findMapMutex.Lock()
eofMsgMap[channelIndex] = tsMsg.(*TimeTickMsg).Timestamp
- findMapMutex.Unlock()
return
}
mu.Lock()
@@ -477,7 +624,7 @@ func (ms *InMemMsgStream) Chan() <- chan *MsgPack {
}
*/
-func checkTimeTickMsg(msg map[int]Timestamp, isChannelReady []bool, mu *sync.RWMutex) (Timestamp, bool) {
+func checkTimeTickMsg(msg map[int]Timestamp, isChannelReady []bool) (Timestamp, bool) {
checkMap := make(map[Timestamp]int)
var maxTime Timestamp = 0
for _, v := range msg {
@@ -492,10 +639,7 @@ func checkTimeTickMsg(msg map[int]Timestamp, isChannelReady []bool, mu *sync.RWM
}
return maxTime, true
}
- for i := range msg {
- mu.RLock()
- v := msg[i]
- mu.Unlock()
+ for i, v := range msg {
if v != maxTime {
isChannelReady[i] = false
} else {
@@ -510,7 +654,7 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
result := make(map[int32]*MsgPack)
for i, request := range tsMsgs {
if request.Type() != internalPb.MsgType_kInsert {
- return nil, errors.New(string("msg's must be Insert"))
+ return nil, errors.New("msg's must be Insert")
}
insertRequest := request.(*InsertMsg)
keys := hashKeys[i]
@@ -521,7 +665,7 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
keysLen := len(keys)
if keysLen != timestampLen || keysLen != rowIDLen || keysLen != rowDataLen {
- return nil, errors.New(string("the length of hashValue, timestamps, rowIDs, RowData are not equal"))
+ return nil, errors.New("the length of hashValue, timestamps, rowIDs, RowData are not equal")
}
for index, key := range keys {
_, ok := result[key]
@@ -544,6 +688,9 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
}
insertMsg := &InsertMsg{
+ BaseMsg: BaseMsg{
+ MsgCtx: request.GetMsgContext(),
+ },
InsertRequest: sliceRequest,
}
result[key].Msgs = append(result[key].Msgs, insertMsg)
@@ -556,7 +703,7 @@ func deleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
result := make(map[int32]*MsgPack)
for i, request := range tsMsgs {
if request.Type() != internalPb.MsgType_kDelete {
- return nil, errors.New(string("msg's must be Delete"))
+ return nil, errors.New("msg's must be Delete")
}
deleteRequest := request.(*DeleteMsg)
keys := hashKeys[i]
@@ -566,7 +713,7 @@ func deleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
keysLen := len(keys)
if keysLen != timestampLen || keysLen != primaryKeysLen {
- return nil, errors.New(string("the length of hashValue, timestamps, primaryKeys are not equal"))
+ return nil, errors.New("the length of hashValue, timestamps, primaryKeys are not equal")
}
for index, key := range keys {
@@ -600,7 +747,7 @@ func defaultRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack,
for i, request := range tsMsgs {
keys := hashKeys[i]
if len(keys) != 1 {
- return nil, errors.New(string("len(msg.hashValue) must equal 1"))
+ return nil, errors.New("len(msg.hashValue) must equal 1")
}
key := keys[0]
_, ok := result[key]
diff --git a/internal/msgstream/msgstream_test.go b/internal/msgstream/msgstream_test.go
index 55c95cebaf2012ddccc7b2e13ede6fc7028725c1..c3b694c3c80e86e9ba8de305de2060a0420c81c8 100644
--- a/internal/msgstream/msgstream_test.go
+++ b/internal/msgstream/msgstream_test.go
@@ -526,6 +526,8 @@ func TestStream_PulsarTtMsgStream_Insert(t *testing.T) {
log.Fatalf("broadcast error = %v", err)
}
receiveMsg(outputStream, len(msgPack1.Msgs))
+ outputTtStream := (*outputStream).(*PulsarTtMsgStream)
+ fmt.Printf("timestamp = %v", outputTtStream.lastTimeStamp)
(*inputStream).Close()
(*outputStream).Close()
}
diff --git a/internal/proxy/grpc_service.go b/internal/proxy/grpc_service.go
index df298ba1ecd3b22d36ec9dcadc111e3d8d777169..29bb0eeecd5ff4354a6eaab3226715ba007321d3 100644
--- a/internal/proxy/grpc_service.go
+++ b/internal/proxy/grpc_service.go
@@ -6,6 +6,7 @@ import (
"log"
"time"
+ "github.com/opentracing/opentracing-go"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
@@ -18,8 +19,13 @@ const (
)
func (p *Proxy) Insert(ctx context.Context, in *servicepb.RowBatch) (*servicepb.IntegerRangeResponse, error) {
+ span, ctx := opentracing.StartSpanFromContext(ctx, "insert grpc received")
+ defer span.Finish()
+ span.SetTag("collection name", in.CollectionName)
+ span.SetTag("partition tag", in.PartitionTag)
log.Println("insert into: ", in.CollectionName)
it := &InsertTask{
+ ctx: ctx,
Condition: NewTaskCondition(ctx),
BaseInsertTask: BaseInsertTask{
BaseMsg: msgstream.BaseMsg{
@@ -119,8 +125,14 @@ func (p *Proxy) CreateCollection(ctx context.Context, req *schemapb.CollectionSc
}
func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.QueryResult, error) {
+ span, ctx := opentracing.StartSpanFromContext(ctx, "search grpc received")
+ defer span.Finish()
+ span.SetTag("collection name", req.CollectionName)
+ span.SetTag("partition tag", req.PartitionTags)
+ span.SetTag("dsl", req.Dsl)
log.Println("search: ", req.CollectionName, req.Dsl)
qt := &QueryTask{
+ ctx: ctx,
Condition: NewTaskCondition(ctx),
SearchRequest: internalpb.SearchRequest{
ProxyID: Params.ProxyID(),
diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go
index f4232bc82234f07d54d713772e89d70f77b5586f..8c5508a815aa101e75292b5a5333716ef27e4f6f 100644
--- a/internal/proxy/proxy.go
+++ b/internal/proxy/proxy.go
@@ -2,6 +2,8 @@ package proxy
import (
"context"
+ "fmt"
+ "io"
"log"
"math/rand"
"net"
@@ -9,6 +11,10 @@ import (
"sync"
"time"
+ "github.com/opentracing/opentracing-go"
+ "github.com/uber/jaeger-client-go"
+ "github.com/uber/jaeger-client-go/config"
+
"google.golang.org/grpc"
"github.com/zilliztech/milvus-distributed/internal/allocator"
@@ -39,6 +45,9 @@ type Proxy struct {
manipulationMsgStream *msgstream.PulsarMsgStream
queryMsgStream *msgstream.PulsarMsgStream
+ tracer opentracing.Tracer
+ closer io.Closer
+
// Add callback functions at different stages
startCallbacks []func()
closeCallbacks []func()
@@ -51,11 +60,28 @@ func Init() {
func CreateProxy(ctx context.Context) (*Proxy, error) {
rand.Seed(time.Now().UnixNano())
ctx1, cancel := context.WithCancel(ctx)
+ var err error
p := &Proxy{
proxyLoopCtx: ctx1,
proxyLoopCancel: cancel,
}
+ cfg := &config.Configuration{
+ ServiceName: "proxy",
+ Sampler: &config.SamplerConfig{
+ Type: "const",
+ Param: 1,
+ },
+ Reporter: &config.ReporterConfig{
+ LogSpans: true,
+ },
+ }
+ p.tracer, p.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
+ if err != nil {
+ panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
+ }
+ opentracing.SetGlobalTracer(p.tracer)
+
pulsarAddress := Params.PulsarAddress()
p.queryMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, Params.MsgStreamSearchBufSize())
@@ -198,12 +224,17 @@ func (p *Proxy) stopProxyLoop() {
p.tick.Close()
p.proxyLoopWg.Wait()
+
}
// Close closes the server.
func (p *Proxy) Close() {
p.stopProxyLoop()
+ if p.closer != nil {
+ p.closer.Close()
+ }
+
for _, cb := range p.closeCallbacks {
cb()
}
diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go
index 486e5a23fb161de8ec5b44410c330de4faa0f57a..3993e6aae5033f53c3c001f474034a09efa33353 100644
--- a/internal/proxy/proxy_test.go
+++ b/internal/proxy/proxy_test.go
@@ -210,7 +210,6 @@ func TestProxy_CreateCollection(t *testing.T) {
wg.Add(1)
go func(group *sync.WaitGroup) {
defer group.Done()
- println("collectionName:", collectionName)
createCollection(t, collectionName)
dropCollection(t, collectionName)
}(&wg)
@@ -489,9 +488,7 @@ func TestProxy_CreateIndex(t *testing.T) {
go func(group *sync.WaitGroup) {
defer group.Done()
createCollection(t, collName)
- if i%2 == 0 {
- createIndex(t, collName, fieldName)
- }
+ createIndex(t, collName, fieldName)
dropCollection(t, collName)
// dropIndex(t, collectionName, fieldName, indexName)
}(&wg)
@@ -513,9 +510,7 @@ func TestProxy_DescribeIndex(t *testing.T) {
go func(group *sync.WaitGroup) {
defer group.Done()
createCollection(t, collName)
- if i%2 == 0 {
- createIndex(t, collName, fieldName)
- }
+ createIndex(t, collName, fieldName)
req := &servicepb.DescribeIndexRequest{
CollectionName: collName,
FieldName: fieldName,
@@ -544,9 +539,7 @@ func TestProxy_DescribeIndexProgress(t *testing.T) {
go func(group *sync.WaitGroup) {
defer group.Done()
createCollection(t, collName)
- if i%2 == 0 {
- createIndex(t, collName, fieldName)
- }
+ createIndex(t, collName, fieldName)
req := &servicepb.DescribeIndexProgressRequest{
CollectionName: collName,
FieldName: fieldName,
diff --git a/internal/proxy/repack_func.go b/internal/proxy/repack_func.go
index 44139999e0403719ca9eaf141f110980b808b6e1..45c4f4d0abc6b5af643e32a7e7229d9442ae298c 100644
--- a/internal/proxy/repack_func.go
+++ b/internal/proxy/repack_func.go
@@ -182,6 +182,7 @@ func insertRepackFunc(tsMsgs []msgstream.TsMsg,
insertMsg := &msgstream.InsertMsg{
InsertRequest: sliceRequest,
}
+ insertMsg.SetMsgContext(request.GetMsgContext())
if together { // all rows with same hash value are accumulated to only one message
if len(result[key].Msgs) <= 0 {
result[key].Msgs = append(result[key].Msgs, insertMsg)
diff --git a/internal/proxy/task.go b/internal/proxy/task.go
index ca793753e9eea8aaa564705890efa2f144ff358c..d28e5654a4c76499f91052aa475bd00c94d12ccc 100644
--- a/internal/proxy/task.go
+++ b/internal/proxy/task.go
@@ -7,6 +7,9 @@ import (
"math"
"strconv"
+ "github.com/opentracing/opentracing-go"
+ oplog "github.com/opentracing/opentracing-go/log"
+
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/allocator"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
@@ -74,12 +77,21 @@ func (it *InsertTask) Type() internalpb.MsgType {
}
func (it *InsertTask) PreExecute() error {
+ span, ctx := opentracing.StartSpanFromContext(it.ctx, "InsertTask preExecute")
+ defer span.Finish()
+ it.ctx = ctx
+ span.SetTag("hash keys", it.ReqID)
+ span.SetTag("start time", it.BeginTs())
collectionName := it.BaseInsertTask.CollectionName
if err := ValidateCollectionName(collectionName); err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
partitionTag := it.BaseInsertTask.PartitionTag
if err := ValidatePartitionTag(partitionTag, true); err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
@@ -87,22 +99,36 @@ func (it *InsertTask) PreExecute() error {
}
func (it *InsertTask) Execute() error {
+ span, ctx := opentracing.StartSpanFromContext(it.ctx, "InsertTask Execute")
+ defer span.Finish()
+ it.ctx = ctx
+ span.SetTag("hash keys", it.ReqID)
+ span.SetTag("start time", it.BeginTs())
collectionName := it.BaseInsertTask.CollectionName
+ span.LogFields(oplog.String("collection_name", collectionName))
if !globalMetaCache.Hit(collectionName) {
err := globalMetaCache.Sync(collectionName)
if err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
}
description, err := globalMetaCache.Get(collectionName)
if err != nil || description == nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
autoID := description.Schema.AutoID
+ span.LogFields(oplog.Bool("auto_id", autoID))
var rowIDBegin UniqueID
var rowIDEnd UniqueID
rowNums := len(it.BaseInsertTask.RowData)
rowIDBegin, rowIDEnd, _ = it.rowIDAllocator.Alloc(uint32(rowNums))
+ span.LogFields(oplog.Int("rowNums", rowNums),
+ oplog.Int("rowIDBegin", int(rowIDBegin)),
+ oplog.Int("rowIDEnd", int(rowIDEnd)))
it.BaseInsertTask.RowIDs = make([]UniqueID, rowNums)
for i := rowIDBegin; i < rowIDEnd; i++ {
offset := i - rowIDBegin
@@ -125,6 +151,8 @@ func (it *InsertTask) Execute() error {
EndTs: it.EndTs(),
Msgs: make([]msgstream.TsMsg, 1),
}
+ tsMsg.SetMsgContext(ctx)
+ span.LogFields(oplog.String("send msg", "send msg"))
msgPack.Msgs[0] = tsMsg
err = it.manipulationMsgStream.Produce(msgPack)
@@ -138,11 +166,14 @@ func (it *InsertTask) Execute() error {
if err != nil {
it.result.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
it.result.Status.Reason = err.Error()
+ span.LogFields(oplog.Error(err))
}
return nil
}
func (it *InsertTask) PostExecute() error {
+ span, _ := opentracing.StartSpanFromContext(it.ctx, "InsertTask postExecute")
+ defer span.Finish()
return nil
}
@@ -352,30 +383,49 @@ func (qt *QueryTask) SetTs(ts Timestamp) {
}
func (qt *QueryTask) PreExecute() error {
+ span, ctx := opentracing.StartSpanFromContext(qt.ctx, "QueryTask preExecute")
+ defer span.Finish()
+ qt.ctx = ctx
+ span.SetTag("hash keys", qt.ReqID)
+ span.SetTag("start time", qt.BeginTs())
+
collectionName := qt.query.CollectionName
if !globalMetaCache.Hit(collectionName) {
err := globalMetaCache.Sync(collectionName)
if err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
}
_, err := globalMetaCache.Get(collectionName)
if err != nil { // err is not nil if collection not exists
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
if err := ValidateCollectionName(qt.query.CollectionName); err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
for _, tag := range qt.query.PartitionTags {
if err := ValidatePartitionTag(tag, false); err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
}
qt.MsgType = internalpb.MsgType_kSearch
+ if qt.query.PartitionTags == nil || len(qt.query.PartitionTags) <= 0 {
+ qt.query.PartitionTags = []string{Params.defaultPartitionTag()}
+ }
queryBytes, err := proto.Marshal(qt.query)
if err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
return err
}
qt.Query = &commonpb.Blob{
@@ -385,6 +435,11 @@ func (qt *QueryTask) PreExecute() error {
}
func (qt *QueryTask) Execute() error {
+ span, ctx := opentracing.StartSpanFromContext(qt.ctx, "QueryTask Execute")
+ defer span.Finish()
+ qt.ctx = ctx
+ span.SetTag("hash keys", qt.ReqID)
+ span.SetTag("start time", qt.BeginTs())
var tsMsg msgstream.TsMsg = &msgstream.SearchMsg{
SearchRequest: qt.SearchRequest,
BaseMsg: msgstream.BaseMsg{
@@ -398,22 +453,31 @@ func (qt *QueryTask) Execute() error {
EndTs: qt.Timestamp,
Msgs: make([]msgstream.TsMsg, 1),
}
+ tsMsg.SetMsgContext(ctx)
msgPack.Msgs[0] = tsMsg
err := qt.queryMsgStream.Produce(msgPack)
log.Printf("[Proxy] length of searchMsg: %v", len(msgPack.Msgs))
if err != nil {
+ span.LogFields(oplog.Error(err))
+ span.Finish()
log.Printf("[Proxy] send search request failed: %v", err)
}
return err
}
func (qt *QueryTask) PostExecute() error {
+ span, _ := opentracing.StartSpanFromContext(qt.ctx, "QueryTask postExecute")
+ defer span.Finish()
+ span.SetTag("hash keys", qt.ReqID)
+ span.SetTag("start time", qt.BeginTs())
for {
select {
case <-qt.ctx.Done():
log.Print("wait to finish failed, timeout!")
+ span.LogFields(oplog.String("wait to finish failed, timeout", "wait to finish failed, timeout"))
return errors.New("wait to finish failed, timeout")
case searchResults := <-qt.resultBuf:
+ span.LogFields(oplog.String("receive result", "receive result"))
filterSearchResult := make([]*internalpb.SearchResult, 0)
var filterReason string
for _, partialSearchResult := range searchResults {
@@ -432,12 +496,13 @@ func (qt *QueryTask) PostExecute() error {
Reason: filterReason,
},
}
+ span.LogFields(oplog.Error(errors.New(filterReason)))
return errors.New(filterReason)
}
hits := make([][]*servicepb.Hits, 0)
for _, partialSearchResult := range filterSearchResult {
- if partialSearchResult.Hits == nil || len(partialSearchResult.Hits) <= 0 {
+ if len(partialSearchResult.Hits) <= 0 {
filterReason += "nq is zero\n"
continue
}
@@ -476,16 +541,7 @@ func (qt *QueryTask) PostExecute() error {
return nil
}
- topk := 0
- getMax := func(a, b int) int {
- if a > b {
- return a
- }
- return b
- }
- for _, hit := range hits {
- topk = getMax(topk, len(hit[0].IDs))
- }
+ topk := len(hits[0][0].IDs)
qt.result = &servicepb.QueryResult{
Status: &commonpb.Status{
ErrorCode: 0,
@@ -503,22 +559,14 @@ func (qt *QueryTask) PostExecute() error {
}
for j := 0; j < topk; j++ {
- valid := false
choice, maxDistance := 0, minFloat32
for q, loc := range locs { // query num, the number of ways to merge
- if loc >= len(hits[q][i].IDs) {
- continue
- }
distance := hits[q][i].Scores[loc]
- if distance > maxDistance || (distance == maxDistance && choice != q) {
+ if distance > maxDistance {
choice = q
maxDistance = distance
- valid = true
}
}
- if !valid {
- break
- }
choiceOffset := locs[choice]
// check if distance is valid, `invalid` here means very very big,
// in this process, distance here is the smallest, so the rest of distance are all invalid
@@ -540,6 +588,7 @@ func (qt *QueryTask) PostExecute() error {
reducedHitsBs, err := proto.Marshal(reducedHits)
if err != nil {
log.Println("marshal error")
+ span.LogFields(oplog.Error(err))
return err
}
qt.result.Hits = append(qt.result.Hits, reducedHitsBs)
@@ -651,7 +700,10 @@ func (dct *DescribeCollectionTask) PreExecute() error {
func (dct *DescribeCollectionTask) Execute() error {
var err error
dct.result, err = dct.masterClient.DescribeCollection(dct.ctx, &dct.DescribeCollectionRequest)
- globalMetaCache.Update(dct.CollectionName.CollectionName, dct.result)
+ if err != nil {
+ return err
+ }
+ err = globalMetaCache.Update(dct.CollectionName.CollectionName, dct.result)
return err
}
diff --git a/internal/proxy/task_scheduler.go b/internal/proxy/task_scheduler.go
index c529e2299148157d0fe97d78882948a923671fcb..b0c7debb2936264dad9db1f5b69ddbb23f802845 100644
--- a/internal/proxy/task_scheduler.go
+++ b/internal/proxy/task_scheduler.go
@@ -14,7 +14,7 @@ import (
type TaskQueue interface {
utChan() <-chan int
- UTEmpty() bool
+ utEmpty() bool
utFull() bool
addUnissuedTask(t task) error
FrontUnissuedTask() task
@@ -44,9 +44,7 @@ func (queue *BaseTaskQueue) utChan() <-chan int {
return queue.utBufChan
}
-func (queue *BaseTaskQueue) UTEmpty() bool {
- queue.utLock.Lock()
- defer queue.utLock.Unlock()
+func (queue *BaseTaskQueue) utEmpty() bool {
return queue.unissuedTasks.Len() == 0
}
@@ -318,7 +316,7 @@ func (sched *TaskScheduler) definitionLoop() {
case <-sched.ctx.Done():
return
case <-sched.DdQueue.utChan():
- if !sched.DdQueue.UTEmpty() {
+ if !sched.DdQueue.utEmpty() {
t := sched.scheduleDdTask()
sched.processTask(t, sched.DdQueue)
}
@@ -333,7 +331,7 @@ func (sched *TaskScheduler) manipulationLoop() {
case <-sched.ctx.Done():
return
case <-sched.DmQueue.utChan():
- if !sched.DmQueue.UTEmpty() {
+ if !sched.DmQueue.utEmpty() {
t := sched.scheduleDmTask()
go sched.processTask(t, sched.DmQueue)
}
@@ -350,7 +348,7 @@ func (sched *TaskScheduler) queryLoop() {
return
case <-sched.DqQueue.utChan():
log.Print("scheduler receive query request ...")
- if !sched.DqQueue.UTEmpty() {
+ if !sched.DqQueue.utEmpty() {
t := sched.scheduleDqTask()
go sched.processTask(t, sched.DqQueue)
} else {
diff --git a/internal/proxy/timetick.go b/internal/proxy/timetick.go
index f47960e3af8a936aec5e6151331f47de8f10e34a..34b79ec26c42ea85ca7d5cb7213042b61179b2d9 100644
--- a/internal/proxy/timetick.go
+++ b/internal/proxy/timetick.go
@@ -24,12 +24,12 @@ type timeTick struct {
tsoAllocator *allocator.TimestampAllocator
tickMsgStream *msgstream.PulsarMsgStream
- peerID UniqueID
- wg sync.WaitGroup
- ctx context.Context
- cancel func()
- timer *time.Ticker
- tickLock sync.RWMutex
+ peerID UniqueID
+ wg sync.WaitGroup
+ ctx context.Context
+ cancel func()
+ timer *time.Ticker
+
checkFunc tickCheckFunc
}
@@ -85,8 +85,6 @@ func (tt *timeTick) tick() error {
} else {
//log.Printf("proxy send time tick message")
}
- tt.tickLock.Lock()
- defer tt.tickLock.Unlock()
tt.lastTick = tt.currentTick
return nil
}
@@ -107,8 +105,6 @@ func (tt *timeTick) tickLoop() {
}
func (tt *timeTick) LastTick() Timestamp {
- tt.tickLock.RLock()
- defer tt.tickLock.RUnlock()
return tt.lastTick
}
diff --git a/internal/querynode/collection_replica.go b/internal/querynode/collection_replica.go
index d36c55aaf97b5db26019c8ac9187d6c8350c5aba..430fd55dd79acb93cc64c83305746f99868f32e1 100644
--- a/internal/querynode/collection_replica.go
+++ b/internal/querynode/collection_replica.go
@@ -332,11 +332,11 @@ func (colReplica *collectionReplicaImpl) getSegmentStatistics() []*internalpb.Se
SegmentID: segmentID,
MemorySize: currentMemSize,
NumRows: segmentNumOfRows,
- RecentlyModified: segment.GetRecentlyModified(),
+ RecentlyModified: segment.recentlyModified,
}
statisticData = append(statisticData, &stat)
- segment.SetRecentlyModified(false)
+ segment.recentlyModified = false
}
return statisticData
diff --git a/internal/querynode/flow_graph_filter_dm_node.go b/internal/querynode/flow_graph_filter_dm_node.go
index fbc8eedb5c82b00e868561b6e5971a9af3f78468..3368e9e31f5f5b0b5ceeb3da1c382c960e947961 100644
--- a/internal/querynode/flow_graph_filter_dm_node.go
+++ b/internal/querynode/flow_graph_filter_dm_node.go
@@ -1,9 +1,11 @@
package querynode
import (
+ "context"
"log"
"math"
+ "github.com/opentracing/opentracing-go"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
@@ -32,6 +34,28 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
// TODO: add error handling
}
+ var childs []opentracing.Span
+ tracer := opentracing.GlobalTracer()
+ if tracer != nil && msgStreamMsg != nil {
+ for _, msg := range msgStreamMsg.TsMessages() {
+ if msg.Type() == internalPb.MsgType_kInsert || msg.Type() == internalPb.MsgType_kSearch {
+ var child opentracing.Span
+ ctx := msg.GetMsgContext()
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan("pass filter node",
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan("pass filter node")
+ }
+ child.SetTag("hash keys", msg.HashKeys())
+ child.SetTag("start time", msg.BeginTs())
+ child.SetTag("end time", msg.EndTs())
+ msg.SetMsgContext(opentracing.ContextWithSpan(ctx, child))
+ childs = append(childs, child)
+ }
+ }
+ }
+
ddMsg, ok := (*in[1]).(*ddMsg)
if !ok {
log.Println("type assertion failed for ddMsg")
@@ -46,11 +70,20 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
timestampMax: msgStreamMsg.TimestampMax(),
},
}
- for _, msg := range msgStreamMsg.TsMessages() {
+ for key, msg := range msgStreamMsg.TsMessages() {
switch msg.Type() {
case internalPb.MsgType_kInsert:
+ var ctx2 context.Context
+ if childs != nil {
+ if childs[key] != nil {
+ ctx2 = opentracing.ContextWithSpan(msg.GetMsgContext(), childs[key])
+ } else {
+ ctx2 = context.Background()
+ }
+ }
resMsg := fdmNode.filterInvalidInsertMessage(msg.(*msgstream.InsertMsg))
if resMsg != nil {
+ resMsg.SetMsgContext(ctx2)
iMsg.insertMessages = append(iMsg.insertMessages, resMsg)
}
// case internalPb.MsgType_kDelete:
@@ -62,6 +95,10 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
iMsg.gcRecord = ddMsg.gcRecord
var res Msg = &iMsg
+
+ for _, child := range childs {
+ child.Finish()
+ }
return []*Msg{&res}
}
diff --git a/internal/querynode/flow_graph_insert_node.go b/internal/querynode/flow_graph_insert_node.go
index 9a2c8ca1f11e34738dfbfae93eaeb8e715b70ef3..f375c824023be164ba3e933104483dded936883f 100644
--- a/internal/querynode/flow_graph_insert_node.go
+++ b/internal/querynode/flow_graph_insert_node.go
@@ -1,11 +1,15 @@
package querynode
import (
+ "context"
"fmt"
"log"
"sync"
+ "github.com/opentracing/opentracing-go"
+ oplog "github.com/opentracing/opentracing-go/log"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+ internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
)
type insertNode struct {
@@ -14,6 +18,7 @@ type insertNode struct {
}
type InsertData struct {
+ insertContext map[int64]context.Context
insertIDs map[UniqueID][]UniqueID
insertTimestamps map[UniqueID][]Timestamp
insertRecords map[UniqueID][]*commonpb.Blob
@@ -38,7 +43,30 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
// TODO: add error handling
}
+ var childs []opentracing.Span
+ tracer := opentracing.GlobalTracer()
+ if tracer != nil && iMsg != nil {
+ for _, msg := range iMsg.insertMessages {
+ if msg.Type() == internalPb.MsgType_kInsert || msg.Type() == internalPb.MsgType_kSearch {
+ var child opentracing.Span
+ ctx := msg.GetMsgContext()
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan("pass insert node",
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan("pass insert node")
+ }
+ child.SetTag("hash keys", msg.HashKeys())
+ child.SetTag("start time", msg.BeginTs())
+ child.SetTag("end time", msg.EndTs())
+ msg.SetMsgContext(opentracing.ContextWithSpan(ctx, child))
+ childs = append(childs, child)
+ }
+ }
+ }
+
insertData := InsertData{
+ insertContext: make(map[int64]context.Context),
insertIDs: make(map[int64][]int64),
insertTimestamps: make(map[int64][]uint64),
insertRecords: make(map[int64][]*commonpb.Blob),
@@ -47,6 +75,7 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
// 1. hash insertMessages to insertData
for _, task := range iMsg.insertMessages {
+ insertData.insertContext[task.SegmentID] = task.GetMsgContext()
insertData.insertIDs[task.SegmentID] = append(insertData.insertIDs[task.SegmentID], task.RowIDs...)
insertData.insertTimestamps[task.SegmentID] = append(insertData.insertTimestamps[task.SegmentID], task.Timestamps...)
insertData.insertRecords[task.SegmentID] = append(insertData.insertRecords[task.SegmentID], task.RowData...)
@@ -85,7 +114,7 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
wg := sync.WaitGroup{}
for segmentID := range insertData.insertRecords {
wg.Add(1)
- go iNode.insert(&insertData, segmentID, &wg)
+ go iNode.insert(insertData.insertContext[segmentID], &insertData, segmentID, &wg)
}
wg.Wait()
@@ -93,15 +122,21 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
gcRecord: iMsg.gcRecord,
timeRange: iMsg.timeRange,
}
+ for _, child := range childs {
+ child.Finish()
+ }
return []*Msg{&res}
}
-func (iNode *insertNode) insert(insertData *InsertData, segmentID int64, wg *sync.WaitGroup) {
+func (iNode *insertNode) insert(ctx context.Context, insertData *InsertData, segmentID int64, wg *sync.WaitGroup) {
+ span, _ := opentracing.StartSpanFromContext(ctx, "insert node insert function")
+ defer span.Finish()
var targetSegment, err = iNode.replica.getSegmentByID(segmentID)
if err != nil {
log.Println("cannot find segment:", segmentID)
// TODO: add error handling
wg.Done()
+ span.LogFields(oplog.Error(err))
return
}
@@ -115,6 +150,7 @@ func (iNode *insertNode) insert(insertData *InsertData, segmentID int64, wg *syn
log.Println(err)
// TODO: add error handling
wg.Done()
+ span.LogFields(oplog.Error(err))
return
}
diff --git a/internal/querynode/load_index_service.go b/internal/querynode/load_index_service.go
index 10857b4f3a4d313c7073e4841c97750e50055cfb..32b276bcf597ad2619d165ac3fb7a6cac4f87204 100644
--- a/internal/querynode/load_index_service.go
+++ b/internal/querynode/load_index_service.go
@@ -100,18 +100,8 @@ func (lis *loadIndexService) start() {
continue
}
// 1. use msg's index paths to get index bytes
- fmt.Println("start load index")
- var err error
- ok, err = lis.checkIndexReady(indexMsg)
- if err != nil {
- log.Println(err)
- continue
- }
- if ok {
- continue
- }
-
var indexBuffer [][]byte
+ var err error
fn := func() error {
indexBuffer, err = lis.loadIndex(indexMsg.IndexPaths)
if err != nil {
@@ -148,13 +138,6 @@ func (lis *loadIndexService) start() {
}
}
-func (lis *loadIndexService) close() {
- if lis.loadIndexMsgStream != nil {
- lis.loadIndexMsgStream.Close()
- }
- lis.cancel()
-}
-
func (lis *loadIndexService) printIndexParams(index []*commonpb.KeyValuePair) {
fmt.Println("=================================================")
for i := 0; i < len(index); i++ {
@@ -207,7 +190,6 @@ func (lis *loadIndexService) updateSegmentIndexStats(indexMsg *msgstream.LoadInd
fieldStatsKey := lis.fieldsStatsIDs2Key(targetSegment.collectionID, indexMsg.FieldID)
_, ok := lis.fieldIndexes[fieldStatsKey]
newIndexParams := indexMsg.IndexParams
-
// sort index params by key
sort.Slice(newIndexParams, func(i, j int) bool { return newIndexParams[i].Key < newIndexParams[j].Key })
if !ok {
@@ -233,7 +215,6 @@ func (lis *loadIndexService) updateSegmentIndexStats(indexMsg *msgstream.LoadInd
})
}
}
- targetSegment.setIndexParam(indexMsg.FieldID, indexMsg.IndexParams)
return nil
}
@@ -305,15 +286,3 @@ func (lis *loadIndexService) sendQueryNodeStats() error {
fmt.Println("sent field stats")
return nil
}
-
-func (lis *loadIndexService) checkIndexReady(loadIndexMsg *msgstream.LoadIndexMsg) (bool, error) {
- segment, err := lis.replica.getSegmentByID(loadIndexMsg.SegmentID)
- if err != nil {
- return false, err
- }
- if !segment.matchIndexParam(loadIndexMsg.FieldID, loadIndexMsg.IndexParams) {
- return false, nil
- }
- return true, nil
-
-}
diff --git a/internal/querynode/load_index_service_test.go b/internal/querynode/load_index_service_test.go
index 852d976366c00285f2953f7201e7635cb47f8ad2..b214b408242fcee05c61afd8381e7d6d71b1cd21 100644
--- a/internal/querynode/load_index_service_test.go
+++ b/internal/querynode/load_index_service_test.go
@@ -22,29 +22,26 @@ import (
"github.com/zilliztech/milvus-distributed/internal/querynode/client"
)
-func TestLoadIndexService_FloatVector(t *testing.T) {
+func TestLoadIndexService(t *testing.T) {
node := newQueryNode()
collectionID := rand.Int63n(1000000)
segmentID := rand.Int63n(1000000)
initTestMeta(t, node, "collection0", collectionID, segmentID)
// loadIndexService and statsService
- suffix := "-test-search" + strconv.FormatInt(rand.Int63n(1000000), 10)
oldSearchChannelNames := Params.SearchChannelNames
- newSearchChannelNames := makeNewChannelNames(oldSearchChannelNames, suffix)
+ var newSearchChannelNames []string
+ for _, channel := range oldSearchChannelNames {
+ newSearchChannelNames = append(newSearchChannelNames, channel+"new")
+ }
Params.SearchChannelNames = newSearchChannelNames
oldSearchResultChannelNames := Params.SearchChannelNames
- newSearchResultChannelNames := makeNewChannelNames(oldSearchResultChannelNames, suffix)
+ var newSearchResultChannelNames []string
+ for _, channel := range oldSearchResultChannelNames {
+ newSearchResultChannelNames = append(newSearchResultChannelNames, channel+"new")
+ }
Params.SearchResultChannelNames = newSearchResultChannelNames
-
- oldLoadIndexChannelNames := Params.LoadIndexChannelNames
- newLoadIndexChannelNames := makeNewChannelNames(oldLoadIndexChannelNames, suffix)
- Params.LoadIndexChannelNames = newLoadIndexChannelNames
-
- oldStatsChannelName := Params.StatsChannelName
- newStatsChannelNames := makeNewChannelNames([]string{oldStatsChannelName}, suffix)
- Params.StatsChannelName = newStatsChannelNames[0]
go node.Start()
//generate insert data
@@ -331,319 +328,9 @@ func TestLoadIndexService_FloatVector(t *testing.T) {
}
Params.SearchChannelNames = oldSearchChannelNames
Params.SearchResultChannelNames = oldSearchResultChannelNames
- Params.LoadIndexChannelNames = oldLoadIndexChannelNames
- Params.StatsChannelName = oldStatsChannelName
fmt.Println("loadIndex floatVector test Done!")
defer assert.Equal(t, findFiledStats, true)
<-node.queryNodeLoopCtx.Done()
node.Close()
}
-
-func TestLoadIndexService_BinaryVector(t *testing.T) {
- node := newQueryNode()
- collectionID := rand.Int63n(1000000)
- segmentID := rand.Int63n(1000000)
- initTestMeta(t, node, "collection0", collectionID, segmentID, true)
-
- // loadIndexService and statsService
- suffix := "-test-search-binary" + strconv.FormatInt(rand.Int63n(1000000), 10)
- oldSearchChannelNames := Params.SearchChannelNames
- newSearchChannelNames := makeNewChannelNames(oldSearchChannelNames, suffix)
- Params.SearchChannelNames = newSearchChannelNames
-
- oldSearchResultChannelNames := Params.SearchChannelNames
- newSearchResultChannelNames := makeNewChannelNames(oldSearchResultChannelNames, suffix)
- Params.SearchResultChannelNames = newSearchResultChannelNames
-
- oldLoadIndexChannelNames := Params.LoadIndexChannelNames
- newLoadIndexChannelNames := makeNewChannelNames(oldLoadIndexChannelNames, suffix)
- Params.LoadIndexChannelNames = newLoadIndexChannelNames
-
- oldStatsChannelName := Params.StatsChannelName
- newStatsChannelNames := makeNewChannelNames([]string{oldStatsChannelName}, suffix)
- Params.StatsChannelName = newStatsChannelNames[0]
- go node.Start()
-
- const msgLength = 1000
- const receiveBufSize = 1024
- const DIM = 128
-
- // generator index data
- var indexRowData []byte
- for n := 0; n < msgLength; n++ {
- for i := 0; i < DIM/8; i++ {
- indexRowData = append(indexRowData, byte(rand.Intn(8)))
- }
- }
-
- //generator insert data
- var insertRowBlob []*commonpb.Blob
- var timestamps []uint64
- var rowIDs []int64
- var hashValues []uint32
- offset := 0
- for n := 0; n < msgLength; n++ {
- rowData := make([]byte, 0)
- rowData = append(rowData, indexRowData[offset:offset+(DIM/8)]...)
- offset += DIM / 8
- age := make([]byte, 4)
- binary.LittleEndian.PutUint32(age, 1)
- rowData = append(rowData, age...)
- blob := &commonpb.Blob{
- Value: rowData,
- }
- insertRowBlob = append(insertRowBlob, blob)
- timestamps = append(timestamps, uint64(n))
- rowIDs = append(rowIDs, int64(n))
- hashValues = append(hashValues, uint32(n))
- }
-
- var insertMsg msgstream.TsMsg = &msgstream.InsertMsg{
- BaseMsg: msgstream.BaseMsg{
- HashValues: hashValues,
- },
- InsertRequest: internalpb.InsertRequest{
- MsgType: internalpb.MsgType_kInsert,
- ReqID: 0,
- CollectionName: "collection0",
- PartitionTag: "default",
- SegmentID: segmentID,
- ChannelID: int64(0),
- ProxyID: int64(0),
- Timestamps: timestamps,
- RowIDs: rowIDs,
- RowData: insertRowBlob,
- },
- }
- insertMsgPack := msgstream.MsgPack{
- BeginTs: 0,
- EndTs: math.MaxUint64,
- Msgs: []msgstream.TsMsg{insertMsg},
- }
-
- // generate timeTick
- timeTickMsg := &msgstream.TimeTickMsg{
- BaseMsg: msgstream.BaseMsg{
- BeginTimestamp: 0,
- EndTimestamp: 0,
- HashValues: []uint32{0},
- },
- TimeTickMsg: internalpb.TimeTickMsg{
- MsgType: internalpb.MsgType_kTimeTick,
- PeerID: UniqueID(0),
- Timestamp: math.MaxUint64,
- },
- }
- timeTickMsgPack := &msgstream.MsgPack{
- Msgs: []msgstream.TsMsg{timeTickMsg},
- }
-
- // pulsar produce
- insertChannels := Params.InsertChannelNames
- ddChannels := Params.DDChannelNames
-
- insertStream := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, receiveBufSize)
- insertStream.SetPulsarClient(Params.PulsarAddress)
- insertStream.CreatePulsarProducers(insertChannels)
- ddStream := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, receiveBufSize)
- ddStream.SetPulsarClient(Params.PulsarAddress)
- ddStream.CreatePulsarProducers(ddChannels)
-
- var insertMsgStream msgstream.MsgStream = insertStream
- insertMsgStream.Start()
- var ddMsgStream msgstream.MsgStream = ddStream
- ddMsgStream.Start()
-
- err := insertMsgStream.Produce(&insertMsgPack)
- assert.NoError(t, err)
- err = insertMsgStream.Broadcast(timeTickMsgPack)
- assert.NoError(t, err)
- err = ddMsgStream.Broadcast(timeTickMsgPack)
- assert.NoError(t, err)
-
- //generate search data and send search msg
- searchRowData := indexRowData[42*(DIM/8) : 43*(DIM/8)]
- dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"JACCARD\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
- placeholderValue := servicepb.PlaceholderValue{
- Tag: "$0",
- Type: servicepb.PlaceholderType_VECTOR_BINARY,
- Values: [][]byte{searchRowData},
- }
- placeholderGroup := servicepb.PlaceholderGroup{
- Placeholders: []*servicepb.PlaceholderValue{&placeholderValue},
- }
- placeGroupByte, err := proto.Marshal(&placeholderGroup)
- if err != nil {
- log.Print("marshal placeholderGroup failed")
- }
- query := servicepb.Query{
- CollectionName: "collection0",
- PartitionTags: []string{"default"},
- Dsl: dslString,
- PlaceholderGroup: placeGroupByte,
- }
- queryByte, err := proto.Marshal(&query)
- if err != nil {
- log.Print("marshal query failed")
- }
- blob := commonpb.Blob{
- Value: queryByte,
- }
- fn := func(n int64) *msgstream.MsgPack {
- searchMsg := &msgstream.SearchMsg{
- BaseMsg: msgstream.BaseMsg{
- HashValues: []uint32{0},
- },
- SearchRequest: internalpb.SearchRequest{
- MsgType: internalpb.MsgType_kSearch,
- ReqID: n,
- ProxyID: int64(1),
- Timestamp: uint64(msgLength),
- ResultChannelID: int64(0),
- Query: &blob,
- },
- }
- return &msgstream.MsgPack{
- Msgs: []msgstream.TsMsg{searchMsg},
- }
- }
- searchStream := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, receiveBufSize)
- searchStream.SetPulsarClient(Params.PulsarAddress)
- searchStream.CreatePulsarProducers(newSearchChannelNames)
- searchStream.Start()
- err = searchStream.Produce(fn(1))
- assert.NoError(t, err)
-
- //get search result
- searchResultStream := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, receiveBufSize)
- searchResultStream.SetPulsarClient(Params.PulsarAddress)
- unmarshalDispatcher := msgstream.NewUnmarshalDispatcher()
- searchResultStream.CreatePulsarConsumers(newSearchResultChannelNames, "loadIndexTestSubSearchResult2", unmarshalDispatcher, receiveBufSize)
- searchResultStream.Start()
- searchResult := searchResultStream.Consume()
- assert.NotNil(t, searchResult)
- unMarshaledHit := servicepb.Hits{}
- err = proto.Unmarshal(searchResult.Msgs[0].(*msgstream.SearchResultMsg).Hits[0], &unMarshaledHit)
- assert.Nil(t, err)
-
- // gen load index message pack
- indexParams := make(map[string]string)
- indexParams["index_type"] = "BIN_IVF_FLAT"
- indexParams["index_mode"] = "cpu"
- indexParams["dim"] = "128"
- indexParams["k"] = "10"
- indexParams["nlist"] = "100"
- indexParams["nprobe"] = "10"
- indexParams["m"] = "4"
- indexParams["nbits"] = "8"
- indexParams["metric_type"] = "JACCARD"
- indexParams["SLICE_SIZE"] = "4"
-
- var indexParamsKV []*commonpb.KeyValuePair
- for key, value := range indexParams {
- indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
- Key: key,
- Value: value,
- })
- }
-
- // generator index
- typeParams := make(map[string]string)
- typeParams["dim"] = "128"
- index, err := indexbuilder.NewCIndex(typeParams, indexParams)
- assert.Nil(t, err)
- err = index.BuildBinaryVecIndexWithoutIds(indexRowData)
- assert.Equal(t, err, nil)
-
- option := &minioKV.Option{
- Address: Params.MinioEndPoint,
- AccessKeyID: Params.MinioAccessKeyID,
- SecretAccessKeyID: Params.MinioSecretAccessKey,
- UseSSL: Params.MinioUseSSLStr,
- BucketName: Params.MinioBucketName,
- CreateBucket: true,
- }
-
- minioKV, err := minioKV.NewMinIOKV(node.queryNodeLoopCtx, option)
- assert.Equal(t, err, nil)
- //save index to minio
- binarySet, err := index.Serialize()
- assert.Equal(t, err, nil)
- indexPaths := make([]string, 0)
- for _, index := range binarySet {
- path := strconv.Itoa(int(segmentID)) + "/" + index.Key
- indexPaths = append(indexPaths, path)
- minioKV.Save(path, string(index.Value))
- }
-
- //test index search result
- indexResult, err := index.QueryOnBinaryVecIndexWithParam(searchRowData, indexParams)
- assert.Equal(t, err, nil)
-
- // create loadIndexClient
- fieldID := UniqueID(100)
- loadIndexChannelNames := Params.LoadIndexChannelNames
- client := client.NewLoadIndexClient(node.queryNodeLoopCtx, Params.PulsarAddress, loadIndexChannelNames)
- client.LoadIndex(indexPaths, segmentID, fieldID, "vec", indexParams)
-
- // init message stream consumer and do checks
- statsMs := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, Params.StatsReceiveBufSize)
- statsMs.SetPulsarClient(Params.PulsarAddress)
- statsMs.CreatePulsarConsumers([]string{Params.StatsChannelName}, Params.MsgChannelSubName, msgstream.NewUnmarshalDispatcher(), Params.StatsReceiveBufSize)
- statsMs.Start()
-
- findFiledStats := false
- for {
- receiveMsg := msgstream.MsgStream(statsMs).Consume()
- assert.NotNil(t, receiveMsg)
- assert.NotEqual(t, len(receiveMsg.Msgs), 0)
-
- for _, msg := range receiveMsg.Msgs {
- statsMsg, ok := msg.(*msgstream.QueryNodeStatsMsg)
- if statsMsg.FieldStats == nil || len(statsMsg.FieldStats) == 0 {
- continue
- }
- findFiledStats = true
- assert.Equal(t, ok, true)
- assert.Equal(t, len(statsMsg.FieldStats), 1)
- fieldStats0 := statsMsg.FieldStats[0]
- assert.Equal(t, fieldStats0.FieldID, fieldID)
- assert.Equal(t, fieldStats0.CollectionID, collectionID)
- assert.Equal(t, len(fieldStats0.IndexStats), 1)
- indexStats0 := fieldStats0.IndexStats[0]
- params := indexStats0.IndexParams
- // sort index params by key
- sort.Slice(indexParamsKV, func(i, j int) bool { return indexParamsKV[i].Key < indexParamsKV[j].Key })
- indexEqual := node.loadIndexService.indexParamsEqual(params, indexParamsKV)
- assert.Equal(t, indexEqual, true)
- }
-
- if findFiledStats {
- break
- }
- }
-
- err = searchStream.Produce(fn(2))
- assert.NoError(t, err)
- searchResult = searchResultStream.Consume()
- assert.NotNil(t, searchResult)
- err = proto.Unmarshal(searchResult.Msgs[0].(*msgstream.SearchResultMsg).Hits[0], &unMarshaledHit)
- assert.Nil(t, err)
-
- idsIndex := indexResult.IDs()
- idsSegment := unMarshaledHit.IDs
- assert.Equal(t, len(idsIndex), len(idsSegment))
- for i := 0; i < len(idsIndex); i++ {
- assert.Equal(t, idsIndex[i], idsSegment[i])
- }
- Params.SearchChannelNames = oldSearchChannelNames
- Params.SearchResultChannelNames = oldSearchResultChannelNames
- Params.LoadIndexChannelNames = oldLoadIndexChannelNames
- Params.StatsChannelName = oldStatsChannelName
- fmt.Println("loadIndex binaryVector test Done!")
-
- defer assert.Equal(t, findFiledStats, true)
- <-node.queryNodeLoopCtx.Done()
- node.Close()
-}
diff --git a/internal/querynode/query_node.go b/internal/querynode/query_node.go
index 41f9391e8bba3e7593d4caca461d222a90aeb935..46a072b7a0ac819d7f32e84de618ebfc87287e1e 100644
--- a/internal/querynode/query_node.go
+++ b/internal/querynode/query_node.go
@@ -14,6 +14,12 @@ import "C"
import (
"context"
+ "fmt"
+ "io"
+
+ "github.com/opentracing/opentracing-go"
+ "github.com/uber/jaeger-client-go"
+ "github.com/uber/jaeger-client-go/config"
)
type QueryNode struct {
@@ -30,6 +36,10 @@ type QueryNode struct {
searchService *searchService
loadIndexService *loadIndexService
statsService *statsService
+
+ //opentracing
+ tracer opentracing.Tracer
+ closer io.Closer
}
func Init() {
@@ -39,31 +49,47 @@ func Init() {
func NewQueryNode(ctx context.Context, queryNodeID uint64) *QueryNode {
ctx1, cancel := context.WithCancel(ctx)
+ q := &QueryNode{
+ queryNodeLoopCtx: ctx1,
+ queryNodeLoopCancel: cancel,
+ QueryNodeID: queryNodeID,
+
+ dataSyncService: nil,
+ metaService: nil,
+ searchService: nil,
+ statsService: nil,
+ }
+
+ var err error
+ cfg := &config.Configuration{
+ ServiceName: "query_node",
+ Sampler: &config.SamplerConfig{
+ Type: "const",
+ Param: 1,
+ },
+ Reporter: &config.ReporterConfig{
+ LogSpans: true,
+ },
+ }
+ q.tracer, q.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
+ if err != nil {
+ panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
+ }
+ opentracing.SetGlobalTracer(q.tracer)
segmentsMap := make(map[int64]*Segment)
collections := make([]*Collection, 0)
tSafe := newTSafe()
- var replica collectionReplica = &collectionReplicaImpl{
+ q.replica = &collectionReplicaImpl{
collections: collections,
segments: segmentsMap,
tSafe: tSafe,
}
- return &QueryNode{
- queryNodeLoopCtx: ctx1,
- queryNodeLoopCancel: cancel,
- QueryNodeID: queryNodeID,
-
- replica: replica,
-
- dataSyncService: nil,
- metaService: nil,
- searchService: nil,
- statsService: nil,
- }
+ return q
}
func (node *QueryNode) Start() error {
@@ -97,10 +123,11 @@ func (node *QueryNode) Close() {
if node.searchService != nil {
node.searchService.close()
}
- if node.loadIndexService != nil {
- node.loadIndexService.close()
- }
if node.statsService != nil {
node.statsService.close()
}
+ if node.closer != nil {
+ node.closer.Close()
+ }
+
}
diff --git a/internal/querynode/query_node_test.go b/internal/querynode/query_node_test.go
index 1217fa3da34d8069dd5a41bb08f5af2d557651bc..34ec092f5287a77247765d1f411958204d74e2f7 100644
--- a/internal/querynode/query_node_test.go
+++ b/internal/querynode/query_node_test.go
@@ -35,7 +35,7 @@ func genTestCollectionMeta(collectionName string, collectionID UniqueID, isBinar
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
- Value: "128",
+ Value: "16",
},
},
IndexParams: []*commonpb.KeyValuePair{
@@ -92,12 +92,8 @@ func genTestCollectionMeta(collectionName string, collectionID UniqueID, isBinar
return &collectionMeta
}
-func initTestMeta(t *testing.T, node *QueryNode, collectionName string, collectionID UniqueID, segmentID UniqueID, optional ...bool) {
- isBinary := false
- if len(optional) > 0 {
- isBinary = optional[0]
- }
- collectionMeta := genTestCollectionMeta(collectionName, collectionID, isBinary)
+func initTestMeta(t *testing.T, node *QueryNode, collectionName string, collectionID UniqueID, segmentID UniqueID) {
+ collectionMeta := genTestCollectionMeta(collectionName, collectionID, false)
schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
assert.NotEqual(t, "", schemaBlob)
diff --git a/internal/querynode/search_service.go b/internal/querynode/search_service.go
index 9700996db3e423feafc5796c826dbd5a7f404661..43512b90192c2a49371ce43c1b292eb858c9e697 100644
--- a/internal/querynode/search_service.go
+++ b/internal/querynode/search_service.go
@@ -5,8 +5,9 @@ import (
"context"
"errors"
"fmt"
+ "github.com/opentracing/opentracing-go"
+ oplog "github.com/opentracing/opentracing-go/log"
"log"
- "regexp"
"sync"
"github.com/golang/protobuf/proto"
@@ -134,22 +135,27 @@ func (ss *searchService) receiveSearchMsg() {
}
searchMsg := make([]msgstream.TsMsg, 0)
serverTime := ss.getServiceableTime()
- for i := range msgPack.Msgs {
- if msgPack.Msgs[i].BeginTs() > serverTime {
- ss.msgBuffer <- msgPack.Msgs[i]
+ for i, msg := range msgPack.Msgs {
+ if msg.BeginTs() > serverTime {
+ ss.msgBuffer <- msg
continue
}
searchMsg = append(searchMsg, msgPack.Msgs[i])
}
for _, msg := range searchMsg {
+ span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "receive search msg")
+ msg.SetMsgContext(ctx)
err := ss.search(msg)
if err != nil {
log.Println(err)
+ span.LogFields(oplog.Error(err))
err2 := ss.publishFailedSearchResult(msg, err.Error())
if err2 != nil {
+ span.LogFields(oplog.Error(err2))
log.Println("publish FailedSearchResult failed, error message: ", err2)
}
}
+ span.Finish()
}
log.Println("ReceiveSearchMsg, do search done, num of searchMsg = ", len(searchMsg))
}
@@ -211,8 +217,12 @@ func (ss *searchService) doUnsolvedMsgSearch() {
// TODO:: cache map[dsl]plan
// TODO: reBatched search requests
func (ss *searchService) search(msg msgstream.TsMsg) error {
+ span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "do search")
+ defer span.Finish()
+ msg.SetMsgContext(ctx)
searchMsg, ok := msg.(*msgstream.SearchMsg)
if !ok {
+ span.LogFields(oplog.Error(errors.New("invalid request type = " + string(msg.Type()))))
return errors.New("invalid request type = " + string(msg.Type()))
}
@@ -221,23 +231,27 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
query := servicepb.Query{}
err := proto.Unmarshal(queryBlob, &query)
if err != nil {
+ span.LogFields(oplog.Error(err))
return errors.New("unmarshal query failed")
}
collectionName := query.CollectionName
- partitionTagsInQuery := query.PartitionTags
+ partitionTags := query.PartitionTags
collection, err := ss.replica.getCollectionByName(collectionName)
if err != nil {
+ span.LogFields(oplog.Error(err))
return err
}
collectionID := collection.ID()
dsl := query.Dsl
plan, err := createPlan(*collection, dsl)
if err != nil {
+ span.LogFields(oplog.Error(err))
return err
}
placeHolderGroupBlob := query.PlaceholderGroup
placeholderGroup, err := parserPlaceholderGroup(plan, placeHolderGroupBlob)
if err != nil {
+ span.LogFields(oplog.Error(err))
return err
}
placeholderGroups := make([]*PlaceholderGroup, 0)
@@ -246,28 +260,15 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
searchResults := make([]*SearchResult, 0)
matchedSegments := make([]*Segment, 0)
- fmt.Println("search msg's partitionTag = ", partitionTagsInQuery)
-
- var partitionTagsInCol []string
- for _, partition := range collection.partitions {
- partitionTag := partition.partitionTag
- partitionTagsInCol = append(partitionTagsInCol, partitionTag)
- }
- var searchPartitionTag []string
- if len(partitionTagsInQuery) == 0 {
- searchPartitionTag = partitionTagsInCol
- } else {
- for _, tag := range partitionTagsInCol {
- for _, toMatchTag := range partitionTagsInQuery {
- re := regexp.MustCompile("^" + toMatchTag + "$")
- if re.MatchString(tag) {
- searchPartitionTag = append(searchPartitionTag, tag)
- }
- }
+ for _, partitionTag := range partitionTags {
+ hasPartition := ss.replica.hasPartition(collectionID, partitionTag)
+ if !hasPartition {
+ span.LogFields(oplog.Error(errors.New("search Failed, invalid partitionTag")))
+ return errors.New("search Failed, invalid partitionTag")
}
}
- for _, partitionTag := range searchPartitionTag {
+ for _, partitionTag := range partitionTags {
partition, _ := ss.replica.getPartitionByTag(collectionID, partitionTag)
for _, segment := range partition.segments {
//fmt.Println("dsl = ", dsl)
@@ -275,6 +276,7 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
searchResult, err := segment.segmentSearch(plan, placeholderGroups, []Timestamp{searchTimestamp})
if err != nil {
+ span.LogFields(oplog.Error(err))
return err
}
searchResults = append(searchResults, searchResult)
@@ -283,55 +285,52 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
}
if len(searchResults) <= 0 {
- for _, group := range placeholderGroups {
- nq := group.getNumOfQuery()
- nilHits := make([][]byte, nq)
- hit := &servicepb.Hits{}
- for i := 0; i < int(nq); i++ {
- bs, err := proto.Marshal(hit)
- if err != nil {
- return err
- }
- nilHits[i] = bs
- }
- var results = internalpb.SearchResult{
- MsgType: internalpb.MsgType_kSearchResult,
- Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_SUCCESS},
- ReqID: searchMsg.ReqID,
- ProxyID: searchMsg.ProxyID,
- QueryNodeID: ss.queryNodeID,
- Timestamp: searchTimestamp,
- ResultChannelID: searchMsg.ResultChannelID,
- Hits: nilHits,
- }
- searchResultMsg := &msgstream.SearchResultMsg{
- BaseMsg: msgstream.BaseMsg{HashValues: []uint32{uint32(searchMsg.ResultChannelID)}},
- SearchResult: results,
- }
- err = ss.publishSearchResult(searchResultMsg)
- if err != nil {
- return err
- }
- return nil
+ var results = internalpb.SearchResult{
+ MsgType: internalpb.MsgType_kSearchResult,
+ Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_SUCCESS},
+ ReqID: searchMsg.ReqID,
+ ProxyID: searchMsg.ProxyID,
+ QueryNodeID: ss.queryNodeID,
+ Timestamp: searchTimestamp,
+ ResultChannelID: searchMsg.ResultChannelID,
+ Hits: nil,
+ }
+ searchResultMsg := &msgstream.SearchResultMsg{
+ BaseMsg: msgstream.BaseMsg{
+ MsgCtx: searchMsg.MsgCtx,
+ HashValues: []uint32{uint32(searchMsg.ResultChannelID)},
+ },
+ SearchResult: results,
+ }
+ err = ss.publishSearchResult(searchResultMsg)
+ if err != nil {
+ span.LogFields(oplog.Error(err))
+ return err
}
+ span.LogFields(oplog.String("publish search research success", "publish search research success"))
+ return nil
}
inReduced := make([]bool, len(searchResults))
numSegment := int64(len(searchResults))
err2 := reduceSearchResults(searchResults, numSegment, inReduced)
if err2 != nil {
+ span.LogFields(oplog.Error(err2))
return err2
}
err = fillTargetEntry(plan, searchResults, matchedSegments, inReduced)
if err != nil {
+ span.LogFields(oplog.Error(err))
return err
}
marshaledHits, err := reorganizeQueryResults(plan, placeholderGroups, searchResults, numSegment, inReduced)
if err != nil {
+ span.LogFields(oplog.Error(err))
return err
}
hitsBlob, err := marshaledHits.getHitsBlob()
if err != nil {
+ span.LogFields(oplog.Error(err))
return err
}
@@ -366,11 +365,14 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
MetricType: plan.getMetricType(),
}
searchResultMsg := &msgstream.SearchResultMsg{
- BaseMsg: msgstream.BaseMsg{HashValues: []uint32{uint32(searchMsg.ResultChannelID)}},
+ BaseMsg: msgstream.BaseMsg{
+ MsgCtx: searchMsg.MsgCtx,
+ HashValues: []uint32{uint32(searchMsg.ResultChannelID)}},
SearchResult: results,
}
err = ss.publishSearchResult(searchResultMsg)
if err != nil {
+ span.LogFields(oplog.Error(err))
return err
}
}
@@ -383,6 +385,9 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
}
func (ss *searchService) publishSearchResult(msg msgstream.TsMsg) error {
+ span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "publish search result")
+ defer span.Finish()
+ msg.SetMsgContext(ctx)
fmt.Println("Public SearchResult", msg.HashKeys())
msgPack := msgstream.MsgPack{}
msgPack.Msgs = append(msgPack.Msgs, msg)
@@ -391,7 +396,9 @@ func (ss *searchService) publishSearchResult(msg msgstream.TsMsg) error {
}
func (ss *searchService) publishFailedSearchResult(msg msgstream.TsMsg, errMsg string) error {
- fmt.Println("Public fail SearchResult!")
+ span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "receive search msg")
+ defer span.Finish()
+ msg.SetMsgContext(ctx)
msgPack := msgstream.MsgPack{}
searchMsg, ok := msg.(*msgstream.SearchMsg)
if !ok {
diff --git a/internal/querynode/segment.go b/internal/querynode/segment.go
index 9333f610d1bc59a80ac4bc8e808dfd4bd4c214a0..037909286549cf3408c94df2c3db9bb938417d97 100644
--- a/internal/querynode/segment.go
+++ b/internal/querynode/segment.go
@@ -13,7 +13,6 @@ package querynode
import "C"
import (
"strconv"
- "sync"
"unsafe"
"github.com/stretchr/testify/assert"
@@ -22,8 +21,6 @@ import (
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
)
-type indexParam = map[string]string
-
type Segment struct {
segmentPtr C.CSegmentBase
segmentID UniqueID
@@ -31,42 +28,25 @@ type Segment struct {
collectionID UniqueID
lastMemSize int64
lastRowCount int64
- mu sync.Mutex
recentlyModified bool
- indexParam map[int64]indexParam
- paramMutex sync.RWMutex
}
func (s *Segment) ID() UniqueID {
return s.segmentID
}
-func (s *Segment) SetRecentlyModified(modify bool) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.recentlyModified = modify
-}
-
-func (s *Segment) GetRecentlyModified() bool {
- s.mu.Lock()
- defer s.mu.Unlock()
- return s.recentlyModified
-}
-
//-------------------------------------------------------------------------------------- constructor and destructor
func newSegment(collection *Collection, segmentID int64, partitionTag string, collectionID UniqueID) *Segment {
/*
CSegmentBase
newSegment(CPartition partition, unsigned long segment_id);
*/
- initIndexParam := make(map[int64]indexParam)
segmentPtr := C.NewSegment(collection.collectionPtr, C.ulong(segmentID))
var newSegment = &Segment{
segmentPtr: segmentPtr,
segmentID: segmentID,
partitionTag: partitionTag,
collectionID: collectionID,
- indexParam: initIndexParam,
}
return newSegment
@@ -181,7 +161,7 @@ func (s *Segment) segmentInsert(offset int64, entityIDs *[]UniqueID, timestamps
return errors.New("Insert failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
- s.SetRecentlyModified(true)
+ s.recentlyModified = true
return nil
}
@@ -276,39 +256,3 @@ func (s *Segment) updateSegmentIndex(loadIndexInfo *LoadIndexInfo) error {
return nil
}
-
-func (s *Segment) setIndexParam(fieldID int64, indexParamKv []*commonpb.KeyValuePair) error {
- s.paramMutex.Lock()
- defer s.paramMutex.Unlock()
- indexParamMap := make(indexParam)
- if indexParamKv == nil {
- return errors.New("loadIndexMsg's indexParam empty")
- }
- for _, param := range indexParamKv {
- indexParamMap[param.Key] = param.Value
- }
- s.indexParam[fieldID] = indexParamMap
- return nil
-}
-
-func (s *Segment) matchIndexParam(fieldID int64, indexParamKv []*commonpb.KeyValuePair) bool {
- s.paramMutex.RLock()
- defer s.paramMutex.RUnlock()
- fieldIndexParam := s.indexParam[fieldID]
- if fieldIndexParam == nil {
- return false
- }
- paramSize := len(s.indexParam)
- matchCount := 0
- for _, param := range indexParamKv {
- value, ok := fieldIndexParam[param.Key]
- if !ok {
- return false
- }
- if param.Value != value {
- return false
- }
- matchCount++
- }
- return paramSize == matchCount
-}
diff --git a/internal/querynode/tsafe.go b/internal/querynode/tsafe.go
index 8673359d655c3bcaff38b4976af983eb3dc1568a..27a1b640046a420a5b0aa2701fb4246b936ebc38 100644
--- a/internal/querynode/tsafe.go
+++ b/internal/querynode/tsafe.go
@@ -44,8 +44,6 @@ func newTSafe() tSafe {
}
func (ts *tSafeImpl) registerTSafeWatcher(t *tSafeWatcher) {
- ts.tSafeMu.Lock()
- defer ts.tSafeMu.Unlock()
ts.watcherList = append(ts.watcherList, t)
}
@@ -57,9 +55,8 @@ func (ts *tSafeImpl) get() Timestamp {
func (ts *tSafeImpl) set(t Timestamp) {
ts.tSafeMu.Lock()
- defer ts.tSafeMu.Unlock()
-
ts.tSafe = t
+ ts.tSafeMu.Unlock()
for _, watcher := range ts.watcherList {
watcher.notify()
}
diff --git a/internal/util/flowgraph/input_node.go b/internal/util/flowgraph/input_node.go
index 7c4271b23be5e31373966c3b64acfc395285916f..26907eddb5c6e5d799093f31b5e10d94e5c5faa5 100644
--- a/internal/util/flowgraph/input_node.go
+++ b/internal/util/flowgraph/input_node.go
@@ -1,8 +1,12 @@
package flowgraph
import (
+ "fmt"
"log"
+ "github.com/opentracing/opentracing-go"
+ "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
+
"github.com/zilliztech/milvus-distributed/internal/msgstream"
)
@@ -25,11 +29,33 @@ func (inNode *InputNode) InStream() *msgstream.MsgStream {
}
// empty input and return one *Msg
-func (inNode *InputNode) Operate(in []*Msg) []*Msg {
+func (inNode *InputNode) Operate([]*Msg) []*Msg {
//fmt.Println("Do InputNode operation")
msgPack := (*inNode.inStream).Consume()
+ var childs []opentracing.Span
+ tracer := opentracing.GlobalTracer()
+ if tracer != nil && msgPack != nil {
+ for _, msg := range msgPack.Msgs {
+ if msg.Type() == internalpb.MsgType_kInsert {
+ var child opentracing.Span
+ ctx := msg.GetMsgContext()
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan(fmt.Sprintf("through msg input node, start time = %d", msg.BeginTs()),
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan(fmt.Sprintf("through msg input node, start time = %d", msg.BeginTs()))
+ }
+ child.SetTag("hash keys", msg.HashKeys())
+ child.SetTag("start time", msg.BeginTs())
+ child.SetTag("end time", msg.EndTs())
+ msg.SetMsgContext(opentracing.ContextWithSpan(ctx, child))
+ childs = append(childs, child)
+ }
+ }
+ }
+
// TODO: add status
if msgPack == nil {
log.Println("null msg pack")
@@ -42,6 +68,10 @@ func (inNode *InputNode) Operate(in []*Msg) []*Msg {
timestampMax: msgPack.EndTs,
}
+ for _, child := range childs {
+ child.Finish()
+ }
+
return []*Msg{&msgStreamMsg}
}
diff --git a/internal/writenode/flow_graph_dd_node.go b/internal/writenode/flow_graph_dd_node.go
index 7cb5ef2ab659d832e7233976054d04a6b5c2775a..8bd71886ebabe7213c0ed29313027e9c3b31839a 100644
--- a/internal/writenode/flow_graph_dd_node.go
+++ b/internal/writenode/flow_graph_dd_node.go
@@ -103,6 +103,8 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg {
return tsMessages[i].BeginTs() < tsMessages[j].BeginTs()
})
+ var flush bool = false
+ var flushSegID UniqueID
// do dd tasks
for _, msg := range tsMessages {
switch msg.Type() {
@@ -116,100 +118,98 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg {
ddNode.dropPartition(msg.(*msgstream.DropPartitionMsg))
case internalPb.MsgType_kFlush:
fMsg := msg.(*msgstream.FlushMsg)
- flushSegID := fMsg.SegmentID
+ flush = true
+ flushSegID = fMsg.SegmentID
ddMsg.flushMessages = append(ddMsg.flushMessages, fMsg)
- ddNode.flush()
-
- log.Println(".. manual flush completed ...")
- ddlFlushMsg := &ddlFlushSyncMsg{
- flushCompleted: true,
- ddlBinlogPathMsg: ddlBinlogPathMsg{
- segID: flushSegID,
- },
- }
-
- ddNode.outCh <- ddlFlushMsg
-
default:
log.Println("Non supporting message type:", msg.Type())
}
}
// generate binlog
- if ddNode.ddBuffer.full() {
- ddNode.flush()
- }
-
- var res Msg = ddNode.ddMsg
- return []*Msg{&res}
-}
-
-func (ddNode *ddNode) flush() {
- // generate binlog
- log.Println(". dd buffer full or receive Flush msg ...")
- ddCodec := &storage.DataDefinitionCodec{}
- for collectionID, data := range ddNode.ddBuffer.ddData {
- // buffer data to binlog
- binLogs, err := ddCodec.Serialize(data.timestamps, data.ddRequestString, data.eventTypes)
- if err != nil {
- log.Println(err)
- continue
- }
- if len(binLogs) != 2 {
- log.Println("illegal binLogs")
- continue
- }
-
- // binLogs -> minIO/S3
- if len(data.ddRequestString) != len(data.timestamps) ||
- len(data.timestamps) != len(data.eventTypes) {
- log.Println("illegal ddBuffer, failed to save binlog")
- continue
- } else {
- log.Println(".. dd buffer flushing ...")
- // Blob key example:
- // ${tenant}/data_definition_log/${collection_id}/ts/${log_idx}
- // ${tenant}/data_definition_log/${collection_id}/ddl/${log_idx}
- keyCommon := path.Join(Params.DdLogRootPath, strconv.FormatInt(collectionID, 10))
-
- // save ts binlog
- timestampLogIdx, err := ddNode.idAllocator.AllocOne()
+ if ddNode.ddBuffer.full() || flush {
+ log.Println(". dd buffer full or receive Flush msg ...")
+ ddCodec := &storage.DataDefinitionCodec{}
+ for collectionID, data := range ddNode.ddBuffer.ddData {
+ // buffer data to binlog
+ binLogs, err := ddCodec.Serialize(data.timestamps, data.ddRequestString, data.eventTypes)
if err != nil {
log.Println(err)
+ continue
}
- timestampKey := path.Join(keyCommon, binLogs[0].GetKey(), strconv.FormatInt(timestampLogIdx, 10))
- err = ddNode.kv.Save(timestampKey, string(binLogs[0].GetValue()))
- if err != nil {
- log.Println(err)
+ if len(binLogs) != 2 {
+ log.Println("illegal binLogs")
+ continue
}
- log.Println("save ts binlog, key = ", timestampKey)
- // save dd binlog
- ddLogIdx, err := ddNode.idAllocator.AllocOne()
- if err != nil {
- log.Println(err)
- }
- ddKey := path.Join(keyCommon, binLogs[1].GetKey(), strconv.FormatInt(ddLogIdx, 10))
- err = ddNode.kv.Save(ddKey, string(binLogs[1].GetValue()))
- if err != nil {
- log.Println(err)
- }
- log.Println("save dd binlog, key = ", ddKey)
-
- ddlFlushMsg := &ddlFlushSyncMsg{
- flushCompleted: false,
- ddlBinlogPathMsg: ddlBinlogPathMsg{
- collID: collectionID,
- paths: []string{timestampKey, ddKey},
- },
+ // binLogs -> minIO/S3
+ if len(data.ddRequestString) != len(data.timestamps) ||
+ len(data.timestamps) != len(data.eventTypes) {
+ log.Println("illegal ddBuffer, failed to save binlog")
+ continue
+ } else {
+ log.Println(".. dd buffer flushing ...")
+ // Blob key example:
+ // ${tenant}/data_definition_log/${collection_id}/ts/${log_idx}
+ // ${tenant}/data_definition_log/${collection_id}/ddl/${log_idx}
+ keyCommon := path.Join(Params.DdLogRootPath, strconv.FormatInt(collectionID, 10))
+
+ // save ts binlog
+ timestampLogIdx, err := ddNode.idAllocator.AllocOne()
+ if err != nil {
+ log.Println(err)
+ }
+ timestampKey := path.Join(keyCommon, binLogs[0].GetKey(), strconv.FormatInt(timestampLogIdx, 10))
+ err = ddNode.kv.Save(timestampKey, string(binLogs[0].GetValue()))
+ if err != nil {
+ log.Println(err)
+ }
+ log.Println("save ts binlog, key = ", timestampKey)
+
+ // save dd binlog
+ ddLogIdx, err := ddNode.idAllocator.AllocOne()
+ if err != nil {
+ log.Println(err)
+ }
+ ddKey := path.Join(keyCommon, binLogs[1].GetKey(), strconv.FormatInt(ddLogIdx, 10))
+ err = ddNode.kv.Save(ddKey, string(binLogs[1].GetValue()))
+ if err != nil {
+ log.Println(err)
+ }
+ log.Println("save dd binlog, key = ", ddKey)
+
+ ddlFlushMsg := &ddlFlushSyncMsg{
+ flushCompleted: false,
+ ddlBinlogPathMsg: ddlBinlogPathMsg{
+ collID: collectionID,
+ paths: []string{timestampKey, ddKey},
+ },
+ }
+
+ ddNode.outCh <- ddlFlushMsg
}
- ddNode.outCh <- ddlFlushMsg
}
+ // clear buffer
+ ddNode.ddBuffer.ddData = make(map[UniqueID]*ddData)
+ }
+
+ if flush {
+
+ log.Println(".. manual flush completed ...")
+ ddlFlushMsg := &ddlFlushSyncMsg{
+ flushCompleted: true,
+ ddlBinlogPathMsg: ddlBinlogPathMsg{
+ segID: flushSegID,
+ },
+ }
+
+ ddNode.outCh <- ddlFlushMsg
}
- // clear buffer
- ddNode.ddBuffer.ddData = make(map[UniqueID]*ddData)
+
+ var res Msg = ddNode.ddMsg
+ return []*Msg{&res}
}
func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
diff --git a/internal/writenode/flow_graph_filter_dm_node.go b/internal/writenode/flow_graph_filter_dm_node.go
index 48ac781ddc4be96502fa79bbe5509e4a41ff9d46..ea60f6c987027c466d614c233af68706960a7f4b 100644
--- a/internal/writenode/flow_graph_filter_dm_node.go
+++ b/internal/writenode/flow_graph_filter_dm_node.go
@@ -1,9 +1,12 @@
package writenode
import (
+ "context"
"log"
"math"
+ "github.com/opentracing/opentracing-go"
+
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
@@ -32,11 +35,34 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
// TODO: add error handling
}
+ var childs []opentracing.Span
+ tracer := opentracing.GlobalTracer()
+ if tracer != nil {
+ for _, msg := range msgStreamMsg.TsMessages() {
+ if msg.Type() == internalPb.MsgType_kInsert {
+ var child opentracing.Span
+ ctx := msg.GetMsgContext()
+ if parent := opentracing.SpanFromContext(ctx); parent != nil {
+ child = tracer.StartSpan("pass filter node",
+ opentracing.FollowsFrom(parent.Context()))
+ } else {
+ child = tracer.StartSpan("pass filter node")
+ }
+ child.SetTag("hash keys", msg.HashKeys())
+ child.SetTag("start time", msg.BeginTs())
+ child.SetTag("end time", msg.EndTs())
+ msg.SetMsgContext(opentracing.ContextWithSpan(ctx, child))
+ childs = append(childs, child)
+ }
+ }
+ }
+
ddMsg, ok := (*in[1]).(*ddMsg)
if !ok {
log.Println("type assertion failed for ddMsg")
// TODO: add error handling
}
+
fdmNode.ddMsg = ddMsg
var iMsg = insertMsg{
@@ -57,11 +83,20 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
}
}
- for _, msg := range msgStreamMsg.TsMessages() {
+ for key, msg := range msgStreamMsg.TsMessages() {
switch msg.Type() {
case internalPb.MsgType_kInsert:
+ var ctx2 context.Context
+ if childs != nil {
+ if childs[key] != nil {
+ ctx2 = opentracing.ContextWithSpan(msg.GetMsgContext(), childs[key])
+ } else {
+ ctx2 = context.Background()
+ }
+ }
resMsg := fdmNode.filterInvalidInsertMessage(msg.(*msgstream.InsertMsg))
if resMsg != nil {
+ resMsg.SetMsgContext(ctx2)
iMsg.insertMessages = append(iMsg.insertMessages, resMsg)
}
// case internalPb.MsgType_kDelete:
@@ -73,6 +108,9 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
iMsg.gcRecord = ddMsg.gcRecord
var res Msg = &iMsg
+ for _, child := range childs {
+ child.Finish()
+ }
return []*Msg{&res}
}
diff --git a/internal/writenode/flow_graph_insert_buffer_node.go b/internal/writenode/flow_graph_insert_buffer_node.go
index 6cc7b7b36ffdaa342bcb3172abe49b03c65535ef..05db320a4fbf8826948c172f48dba15750bc86b3 100644
--- a/internal/writenode/flow_graph_insert_buffer_node.go
+++ b/internal/writenode/flow_graph_insert_buffer_node.go
@@ -4,11 +4,15 @@ import (
"bytes"
"context"
"encoding/binary"
+ "fmt"
"log"
"path"
"strconv"
"unsafe"
+ "github.com/opentracing/opentracing-go"
+ oplog "github.com/opentracing/opentracing-go/log"
+
"github.com/zilliztech/milvus-distributed/internal/allocator"
"github.com/zilliztech/milvus-distributed/internal/kv"
miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
@@ -96,12 +100,23 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
// iMsg is insertMsg
// 1. iMsg -> buffer
for _, msg := range iMsg.insertMessages {
+ ctx := msg.GetMsgContext()
+ var span opentracing.Span
+ if ctx != nil {
+ span, _ = opentracing.StartSpanFromContext(ctx, fmt.Sprintf("insert buffer node, start time = %d", msg.BeginTs()))
+ } else {
+ span = opentracing.StartSpan(fmt.Sprintf("insert buffer node, start time = %d", msg.BeginTs()))
+ }
+ span.SetTag("hash keys", msg.HashKeys())
+ span.SetTag("start time", msg.BeginTs())
+ span.SetTag("end time", msg.EndTs())
if len(msg.RowIDs) != len(msg.Timestamps) || len(msg.RowIDs) != len(msg.RowData) {
log.Println("Error: misaligned messages detected")
continue
}
currentSegID := msg.GetSegmentID()
collectionName := msg.GetCollectionName()
+ span.LogFields(oplog.Int("segment id", int(currentSegID)))
idata, ok := ibNode.insertBuffer.insertData[currentSegID]
if !ok {
@@ -110,6 +125,21 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
}
}
+ // Timestamps
+ _, ok = idata.Data[1].(*storage.Int64FieldData)
+ if !ok {
+ idata.Data[1] = &storage.Int64FieldData{
+ Data: []int64{},
+ NumRows: 0,
+ }
+ }
+ tsData := idata.Data[1].(*storage.Int64FieldData)
+ for _, ts := range msg.Timestamps {
+ tsData.Data = append(tsData.Data, int64(ts))
+ }
+ tsData.NumRows += len(msg.Timestamps)
+ span.LogFields(oplog.Int("tsData numRows", tsData.NumRows))
+
// 1.1 Get CollectionMeta from etcd
collection, err := ibNode.replica.getCollectionByName(collectionName)
//collSchema, err := ibNode.getCollectionSchemaByName(collectionName)
@@ -358,9 +388,11 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
// 1.3 store in buffer
ibNode.insertBuffer.insertData[currentSegID] = idata
+ span.LogFields(oplog.String("store in buffer", "store in buffer"))
// 1.4 if full
// 1.4.1 generate binlogs
+ span.LogFields(oplog.String("generate binlogs", "generate binlogs"))
if ibNode.insertBuffer.full(currentSegID) {
log.Printf(". Insert Buffer full, auto flushing (%v) rows of data...", ibNode.insertBuffer.size(currentSegID))
// partitionTag -> partitionID
@@ -429,6 +461,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
ibNode.outCh <- inBinlogMsg
}
}
+ span.Finish()
}
if len(iMsg.insertMessages) > 0 {
diff --git a/internal/writenode/flush_sync_service.go b/internal/writenode/flush_sync_service.go
index 77e17bda86b2e7db6e4bea3add0d73e34e4f35e2..7b6587b711c5d79c3bde04573e889d2024c7b287 100644
--- a/internal/writenode/flush_sync_service.go
+++ b/internal/writenode/flush_sync_service.go
@@ -112,7 +112,6 @@ func (fService *flushSyncService) start() {
fService.completeInsertFlush(insertFlushMsg.segID)
if fService.FlushCompleted(insertFlushMsg.segID) {
- log.Printf("Seg(%d) flush completed.", insertFlushMsg.segID)
fService.metaTable.CompleteFlush(insertFlushMsg.ts, insertFlushMsg.segID)
}
}
diff --git a/internal/writenode/flush_sync_service_test.go b/internal/writenode/flush_sync_service_test.go
index 59e0442f4244808fda0832dfcff0df51aa02375c..7da80503d6191fcbe38aa299829c2c8e69181e16 100644
--- a/internal/writenode/flush_sync_service_test.go
+++ b/internal/writenode/flush_sync_service_test.go
@@ -90,7 +90,7 @@ func TestFlushSyncService_Start(t *testing.T) {
}
for {
- if len(ddChan) == 0 && len(insertChan) == 0 && fService.FlushCompleted(SegID) {
+ if len(ddChan) == 0 && len(insertChan) == 0 {
break
}
}
@@ -117,6 +117,10 @@ func TestFlushSyncService_Start(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, true, cp)
+ cp, err = fService.metaTable.checkFlushComplete(SegID)
+ assert.NoError(t, err)
+ assert.Equal(t, true, cp)
+
})
}
diff --git a/internal/writenode/meta_table.go b/internal/writenode/meta_table.go
index ea7828874f935da9e41880c46b87d55595f222b5..ab5ac04461587a53ec3a4c0ea6ec4a7bbee1d96c 100644
--- a/internal/writenode/meta_table.go
+++ b/internal/writenode/meta_table.go
@@ -171,8 +171,8 @@ func (mt *metaTable) addSegmentFlush(segmentID UniqueID, timestamp Timestamp) er
}
func (mt *metaTable) getFlushCloseTime(segmentID UniqueID) (Timestamp, error) {
- mt.lock.RLock()
- defer mt.lock.RUnlock()
+ mt.lock.Lock()
+ defer mt.lock.Unlock()
meta, ok := mt.segID2FlushMeta[segmentID]
if !ok {
return typeutil.ZeroTimestamp, errors.Errorf("segment not exists with ID = " + strconv.FormatInt(segmentID, 10))
@@ -181,8 +181,8 @@ func (mt *metaTable) getFlushCloseTime(segmentID UniqueID) (Timestamp, error) {
}
func (mt *metaTable) getFlushOpenTime(segmentID UniqueID) (Timestamp, error) {
- mt.lock.RLock()
- defer mt.lock.RUnlock()
+ mt.lock.Lock()
+ defer mt.lock.Unlock()
meta, ok := mt.segID2FlushMeta[segmentID]
if !ok {
return typeutil.ZeroTimestamp, errors.Errorf("segment not exists with ID = " + strconv.FormatInt(segmentID, 10))
@@ -191,8 +191,8 @@ func (mt *metaTable) getFlushOpenTime(segmentID UniqueID) (Timestamp, error) {
}
func (mt *metaTable) checkFlushComplete(segmentID UniqueID) (bool, error) {
- mt.lock.RLock()
- defer mt.lock.RUnlock()
+ mt.lock.Lock()
+ defer mt.lock.Unlock()
meta, ok := mt.segID2FlushMeta[segmentID]
if !ok {
return false, errors.Errorf("segment not exists with ID = " + strconv.FormatInt(segmentID, 10))
@@ -201,8 +201,9 @@ func (mt *metaTable) checkFlushComplete(segmentID UniqueID) (bool, error) {
}
func (mt *metaTable) getSegBinlogPaths(segmentID UniqueID) (map[int64][]string, error) {
- mt.lock.RLock()
- defer mt.lock.RUnlock()
+ mt.lock.Lock()
+ defer mt.lock.Unlock()
+
meta, ok := mt.segID2FlushMeta[segmentID]
if !ok {
return nil, errors.Errorf("segment not exists with ID = " + strconv.FormatInt(segmentID, 10))
@@ -215,8 +216,9 @@ func (mt *metaTable) getSegBinlogPaths(segmentID UniqueID) (map[int64][]string,
}
func (mt *metaTable) getDDLBinlogPaths(collID UniqueID) (map[UniqueID][]string, error) {
- mt.lock.RLock()
- defer mt.lock.RUnlock()
+ mt.lock.Lock()
+ defer mt.lock.Unlock()
+
meta, ok := mt.collID2DdlMeta[collID]
if !ok {
return nil, errors.Errorf("collection not exists with ID = " + strconv.FormatInt(collID, 10))
diff --git a/internal/writenode/write_node.go b/internal/writenode/write_node.go
index d3ce6f84c70ac40d1fd52fc3a9f13df7d295d8ed..5199148c96cc1476e3bf9525f12cdf649b776c08 100644
--- a/internal/writenode/write_node.go
+++ b/internal/writenode/write_node.go
@@ -2,6 +2,12 @@ package writenode
import (
"context"
+ "fmt"
+ "io"
+
+ "github.com/opentracing/opentracing-go"
+ "github.com/uber/jaeger-client-go"
+ "github.com/uber/jaeger-client-go/config"
)
type WriteNode struct {
@@ -11,6 +17,8 @@ type WriteNode struct {
flushSyncService *flushSyncService
metaService *metaService
replica collectionReplica
+ tracer opentracing.Tracer
+ closer io.Closer
}
func NewWriteNode(ctx context.Context, writeNodeID uint64) *WriteNode {
@@ -38,6 +46,22 @@ func Init() {
}
func (node *WriteNode) Start() error {
+ cfg := &config.Configuration{
+ ServiceName: "write_node",
+ Sampler: &config.SamplerConfig{
+ Type: "const",
+ Param: 1,
+ },
+ Reporter: &config.ReporterConfig{
+ LogSpans: true,
+ },
+ }
+ var err error
+ node.tracer, node.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
+ if err != nil {
+ panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
+ }
+ opentracing.SetGlobalTracer(node.tracer)
// TODO GOOSE Init Size??
chanSize := 100
@@ -61,4 +85,9 @@ func (node *WriteNode) Close() {
if node.dataSyncService != nil {
(*node.dataSyncService).close()
}
+
+ if node.closer != nil {
+ node.closer.Close()
+ }
+
}
diff --git a/scripts/cwrapper_rocksdb_build.sh b/scripts/cwrapper_rocksdb_build.sh
new file mode 100644
index 0000000000000000000000000000000000000000..85a9e3615ad41e84782590ad53ac5f111abadf14
--- /dev/null
+++ b/scripts/cwrapper_rocksdb_build.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+SOURCE=${BASH_SOURCE[0]}
+while [ -h $SOURCE ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR=$( cd -P $( dirname $SOURCE ) && pwd )
+ SOURCE=$(readlink $SOURCE)
+ [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR=$( cd -P $( dirname $SOURCE ) && pwd )
+# echo $DIR
+
+SRC_DIR=${DIR}/../internal/kv/rocksdb/cwrapper
+CGO_CFLAGS="-I$(SRC_DIR)/output/include"
+CGO_LDFLAGS="-L$(SRC_DIR)/output/lib -l:librocksdb.a -lstdc++ -lm -lz"
+
+OUTPUT_LIB=${SRC_DIR}/output
+
+if [ -d ${OUTPUT_LIB} ];then
+ rm -rf ${OUTPUT_LIB}
+fi
+mkdir ${OUTPUT_LIB}
+
+BUILD_TYPE="Debug"
+
+while getopts "t:h:" arg; do
+ case $arg in
+ t)
+ BUILD_TYPE=$OPTARG # BUILD_TYPE
+ ;;
+ h) # help
+ echo "-t: build type(default: Debug)
+-h: help
+ "
+ exit 0
+ ;;
+ ?)
+ echo "ERROR! unknown argument"
+ exit 1
+ ;;
+ esac
+done
+echo "BUILD_TYPE: " $BUILD_TYPE
+
+pushd ${OUTPUT_LIB}
+CMAKE_CMD="cmake \
+-DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${SRC_DIR}"
+
+${CMAKE_CMD}
+echo ${CMAKE_CMD}
+
+if [[ ! ${jobs+1} ]]; then
+ jobs=$(nproc)
+fi
+make -j ${jobs}
diff --git a/scripts/init_devcontainer.sh b/scripts/init_devcontainer.sh
index 80321670b9968754eca42575da89b7cbd78327a7..6b30db984ee3e073987a8e473f4589ac16ac196e 100755
--- a/scripts/init_devcontainer.sh
+++ b/scripts/init_devcontainer.sh
@@ -8,15 +8,6 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
done
ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
-unameOut="$(uname -s)"
-case "${unameOut}" in
- Linux*) machine=Linux;;
- Darwin*) machine=Mac;;
- CYGWIN*) machine=Cygwin;;
- MINGW*) machine=MinGw;;
- *) machine="UNKNOWN:${unameOut}"
-esac
-
# Attempt to run in the container with the same UID/GID as we have on the host,
# as this results in the correct permissions on files created in the shared
# volumes. This isn't always possible, however, as IDs less than 100 are
@@ -30,14 +21,8 @@ gid=$(id -g)
[ "$uid" -lt 500 ] && uid=501
[ "$gid" -lt 500 ] && gid=$uid
-awk 'c&&c--{sub(/^/,"#")} /# Build devcontainer/{c=5} 1' $ROOT_DIR/docker-compose.yml > $ROOT_DIR/docker-compose-vscode.yml.tmp
-
-awk 'c&&c--{sub(/^/,"#")} /# Command/{c=3} 1' $ROOT_DIR/docker-compose-vscode.yml.tmp > $ROOT_DIR/docker-compose-vscode.yml
+awk 'c&&c--{sub(/^/,"#")} /# Build devcontainer/{c=5} 1' $ROOT_DIR/docker-compose.yml > $ROOT_DIR/docker-compose-vscode.yml.bak
-rm $ROOT_DIR/docker-compose-vscode.yml.tmp
+awk 'c&&c--{sub(/^/,"#")} /# Command/{c=3} 1' $ROOT_DIR/docker-compose-vscode.yml.bak > $ROOT_DIR/docker-compose-vscode.yml
-if [ "${machine}" == "Mac" ];then
- sed -i '' "s/# user: {{ CURRENT_ID }}/user: \"$uid:$gid\"/g" $ROOT_DIR/docker-compose-vscode.yml
-else
- sed -i "s/# user: {{ CURRENT_ID }}/user: \"$uid:$gid\"/g" $ROOT_DIR/docker-compose-vscode.yml
-fi
\ No newline at end of file
+sed -i '.bak' "s/# user: {{ CURRENT_ID }}/user: \"$uid:$gid\"/g" $ROOT_DIR/docker-compose-vscode.yml
diff --git a/scripts/run_go_unittest.sh b/scripts/run_go_unittest.sh
index b48b9f6717d353b2f7de2f31af7381a571122d39..f4b4bac81f072b6c8353e675479c7c7fd149a450 100755
--- a/scripts/run_go_unittest.sh
+++ b/scripts/run_go_unittest.sh
@@ -8,15 +8,13 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
-ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
# ignore Minio,S3 unittes
-MILVUS_DIR="${ROOT_DIR}/internal/"
+MILVUS_DIR="${SCRIPTS_DIR}/../internal/"
echo $MILVUS_DIR
-
-go test -race -cover "${MILVUS_DIR}/kv/..." -failfast
-go test -race -cover "${MILVUS_DIR}/proxy/..." -failfast
-go test -race -cover "${MILVUS_DIR}/writenode/..." -failfast
-go test -race -cover "${MILVUS_DIR}/master/..." -failfast
-go test -cover "${MILVUS_DIR}/msgstream/..." "${MILVUS_DIR}/querynode/..." "${MILVUS_DIR}/storage" "${MILVUS_DIR}/util/..." -failfast
-#go test -race -cover "${MILVUS_DIR}/kv/..." "${MILVUS_DIR}/msgstream/..." "${MILVUS_DIR}/master/..." "${MILVUS_DIR}/querynode/..." -failfast
+go test -cover "${MILVUS_DIR}/kv/..." -failfast
+go test -cover "${MILVUS_DIR}/proxy/..." -failfast
+go test -cover "${MILVUS_DIR}/writenode/..." -failfast
+go test -cover "${MILVUS_DIR}/msgstream/..." "${MILVUS_DIR}/master/..." "${MILVUS_DIR}/querynode/..." "${MILVUS_DIR}/storage" "${MILVUS_DIR}/util/..." -failfast
+#go test -cover "${MILVUS_DIR}/kv/..." "${MILVUS_DIR}/msgstream/..." "${MILVUS_DIR}/master/..." "${MILVUS_DIR}/querynode/..." -failfast
diff --git a/tests/python/requirements.txt b/tests/python/requirements.txt
index 98dd4e02c717761d411870a5bc05d3a0a68e2c74..9bee462aee885e91e3b5735d9a8c2ef4f58a45df 100644
--- a/tests/python/requirements.txt
+++ b/tests/python/requirements.txt
@@ -4,5 +4,5 @@ numpy==1.18.1
pytest==5.3.4
pytest-cov==2.8.1
pytest-timeout==1.3.4
-pymilvus-distributed==0.0.14
+pymilvus-distributed==0.0.10
sklearn==0.0
diff --git a/tests/python/test_bulk_insert.py b/tests/python/test_bulk_insert.py
index 97f3724ec03411fd7d509fb03556d27bf403c9b1..d8ff5cbd7abd807492311669b9dd4cd159965b21 100644
--- a/tests/python/test_bulk_insert.py
+++ b/tests/python/test_bulk_insert.py
@@ -101,6 +101,7 @@ class TestInsertBase:
connect.flush([collection])
connect.drop_collection(collection)
+ @pytest.mark.skip("create index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_create_index(self, connect, collection, get_simple_index):
'''
@@ -118,6 +119,7 @@ class TestInsertBase:
if field["name"] == field_name:
assert field["indexes"][0] == get_simple_index
+ @pytest.mark.skip("create index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_create_index_new(self, connect, collection, get_simple_index):
'''
@@ -135,6 +137,7 @@ class TestInsertBase:
if field["name"] == field_name:
assert field["indexes"][0] == get_simple_index
+ @pytest.mark.skip("create index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_after_create_index(self, connect, collection, get_simple_index):
'''
@@ -151,6 +154,7 @@ class TestInsertBase:
if field["name"] == field_name:
assert field["indexes"][0] == get_simple_index
+ # @pytest.mark.skip(" later ")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_search(self, connect, collection):
'''
@@ -641,6 +645,7 @@ class TestInsertBinary:
connect.flush([binary_collection])
assert connect.count_entities(binary_collection) == default_nb
+ @pytest.mark.skip("create index")
def test_insert_binary_after_create_index(self, connect, binary_collection, get_binary_index):
'''
target: test insert binary entities after build index
@@ -657,6 +662,7 @@ class TestInsertBinary:
if field["name"] == binary_field_name:
assert field["indexes"][0] == get_binary_index
+ @pytest.mark.skip("create index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_binary_create_index(self, connect, binary_collection, get_binary_index):
'''
@@ -857,6 +863,7 @@ class TestInsertMultiCollections:
connect.flush([collection_name])
assert len(ids) == 1
+ @pytest.mark.skip("create index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_create_index_insert_vector_another(self, connect, collection, get_simple_index):
'''
@@ -870,7 +877,7 @@ class TestInsertMultiCollections:
ids = connect.bulk_insert(collection, default_entity)
connect.drop_collection(collection_name)
- @pytest.mark.skip("count entities")
+ @pytest.mark.skip("create index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_vector_create_index_another(self, connect, collection, get_simple_index):
'''
@@ -885,7 +892,7 @@ class TestInsertMultiCollections:
count = connect.count_entities(collection_name)
assert count == 0
- @pytest.mark.skip("count entities")
+ @pytest.mark.skip("create index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_vector_sleep_create_index_another(self, connect, collection, get_simple_index):
'''
diff --git a/tests/python/test_index.py b/tests/python/test_index.py
index 8162fa37fe82c30a55a1a423d89b45b1b128cc6f..687e6573a68070eaee449f7d0cf425cf971fb82b 100644
--- a/tests/python/test_index.py
+++ b/tests/python/test_index.py
@@ -17,19 +17,19 @@ query, query_vecs = gen_query_vectors(field_name, default_entities, default_top_
default_index = {"index_type": "IVF_FLAT", "params": {"nlist": 128}, "metric_type": "L2"}
-# @pytest.mark.skip("wait for debugging...")
+@pytest.mark.skip("wait for debugging...")
class TestIndexBase:
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_simple_index(self, request, connect):
- import copy
logging.getLogger().info(request.param)
- if str(connect._cmd("mode")) == "CPU":
- if request.param["index_type"] in index_cpu_not_support():
- pytest.skip("sq8h not support in CPU mode")
- return copy.deepcopy(request.param)
+ # TODO: Determine the service mode
+ # if str(connect._cmd("mode")) == "CPU":
+ if request.param["index_type"] in index_cpu_not_support():
+ pytest.skip("sq8h not support in CPU mode")
+ return request.param
@pytest.fixture(
scope="function",
@@ -132,7 +132,7 @@ class TestIndexBase:
'''
ids = connect.bulk_insert(collection, default_entities)
connect.create_index(collection, field_name, get_simple_index)
- # logging.getLogger().info(connect.get_collection_stats(collection))
+ logging.getLogger().info(connect.get_collection_stats(collection))
nq = get_nq
index_type = get_simple_index["index_type"]
search_param = get_search_param(index_type)
@@ -140,7 +140,6 @@ class TestIndexBase:
res = connect.search(collection, query)
assert len(res) == nq
- @pytest.mark.skip("can't_pass_ci")
@pytest.mark.timeout(BUILD_TIMEOUT)
@pytest.mark.level(2)
def test_create_index_multithread(self, connect, collection, args):
@@ -176,7 +175,6 @@ class TestIndexBase:
with pytest.raises(Exception) as e:
connect.create_index(collection_name, field_name, default_index)
- @pytest.mark.skip("count_entries")
@pytest.mark.level(2)
@pytest.mark.timeout(BUILD_TIMEOUT)
def test_create_index_insert_flush(self, connect, collection, get_simple_index):
@@ -203,7 +201,6 @@ class TestIndexBase:
connect.create_index(collection, field_name, get_simple_index)
# TODO:
- @pytest.mark.skip("get_collection_stats")
@pytest.mark.level(2)
@pytest.mark.timeout(BUILD_TIMEOUT)
def test_create_different_index_repeatedly(self, connect, collection):
@@ -278,7 +275,7 @@ class TestIndexBase:
ids = connect.bulk_insert(collection, default_entities)
get_simple_index["metric_type"] = metric_type
connect.create_index(collection, field_name, get_simple_index)
- # logging.getLogger().info(connect.get_collection_stats(collection))
+ logging.getLogger().info(connect.get_collection_stats(collection))
nq = get_nq
index_type = get_simple_index["index_type"]
search_param = get_search_param(index_type)
@@ -323,7 +320,6 @@ class TestIndexBase:
with pytest.raises(Exception) as e:
connect.create_index(collection_name, field_name, default_index)
- @pytest.mark.skip("count_entries")
@pytest.mark.timeout(BUILD_TIMEOUT)
def test_create_index_no_vectors_insert_ip(self, connect, collection, get_simple_index):
'''
@@ -351,8 +347,6 @@ class TestIndexBase:
connect.create_index(collection, field_name, get_simple_index)
# TODO:
-
- @pytest.mark.skip("get_collection_stats")
@pytest.mark.level(2)
@pytest.mark.timeout(BUILD_TIMEOUT)
def test_create_different_index_repeatedly_ip(self, connect, collection):
@@ -375,7 +369,6 @@ class TestIndexBase:
******************************************************************
"""
- @pytest.mark.skip("drop_index")
def test_drop_index(self, connect, collection, get_simple_index):
'''
target: test drop index interface
@@ -389,7 +382,6 @@ class TestIndexBase:
# assert stats["partitions"][0]["segments"][0]["index_name"] == default_index_type
assert not stats["partitions"][0]["segments"]
- @pytest.mark.skip("drop_index")
@pytest.mark.level(2)
def test_drop_index_repeatly(self, connect, collection, get_simple_index):
'''
@@ -406,7 +398,6 @@ class TestIndexBase:
# assert stats["partitions"][0]["segments"][0]["index_name"] == default_index_type
assert not stats["partitions"][0]["segments"]
- @pytest.mark.skip("drop_index")
@pytest.mark.level(2)
def test_drop_index_without_connect(self, dis_connect, collection):
'''
@@ -417,7 +408,6 @@ class TestIndexBase:
with pytest.raises(Exception) as e:
dis_connect.drop_index(collection, field_name)
- @pytest.mark.skip("drop_index")
def test_drop_index_collection_not_existed(self, connect):
'''
target: test drop index interface when collection name not existed
@@ -429,7 +419,6 @@ class TestIndexBase:
with pytest.raises(Exception) as e:
connect.drop_index(collection_name, field_name)
- @pytest.mark.skip("drop_index")
def test_drop_index_collection_not_create(self, connect, collection):
'''
target: test drop index interface when index not created
@@ -440,7 +429,6 @@ class TestIndexBase:
# no create index
connect.drop_index(collection, field_name)
- @pytest.mark.skip("drop_index")
@pytest.mark.level(2)
def test_create_drop_index_repeatly(self, connect, collection, get_simple_index):
'''
@@ -452,7 +440,6 @@ class TestIndexBase:
connect.create_index(collection, field_name, get_simple_index)
connect.drop_index(collection, field_name)
- @pytest.mark.skip("drop_index")
def test_drop_index_ip(self, connect, collection, get_simple_index):
'''
target: test drop index interface
@@ -467,7 +454,6 @@ class TestIndexBase:
# assert stats["partitions"][0]["segments"][0]["index_name"] == default_index_type
assert not stats["partitions"][0]["segments"]
- @pytest.mark.skip("drop_index")
@pytest.mark.level(2)
def test_drop_index_repeatly_ip(self, connect, collection, get_simple_index):
'''
@@ -485,7 +471,6 @@ class TestIndexBase:
# assert stats["partitions"][0]["segments"][0]["index_name"] == default_index_type
assert not stats["partitions"][0]["segments"]
- @pytest.mark.skip("drop_index")
@pytest.mark.level(2)
def test_drop_index_without_connect_ip(self, dis_connect, collection):
'''
@@ -496,7 +481,6 @@ class TestIndexBase:
with pytest.raises(Exception) as e:
dis_connect.drop_index(collection, field_name)
- @pytest.mark.skip("drop_index")
def test_drop_index_collection_not_create_ip(self, connect, collection):
'''
target: test drop index interface when index not created
@@ -507,7 +491,6 @@ class TestIndexBase:
# no create index
connect.drop_index(collection, field_name)
- @pytest.mark.skip("drop_index")
@pytest.mark.level(2)
def test_create_drop_index_repeatly_ip(self, connect, collection, get_simple_index):
'''
@@ -521,6 +504,7 @@ class TestIndexBase:
connect.drop_index(collection, field_name)
+@pytest.mark.skip("binary")
class TestIndexBinary:
@pytest.fixture(
scope="function",
@@ -606,7 +590,6 @@ class TestIndexBinary:
res = connect.search(binary_collection, query, search_params=search_param)
assert len(res) == nq
- @pytest.mark.skip("get status for build index failed")
@pytest.mark.timeout(BUILD_TIMEOUT)
def test_create_index_invalid_metric_type_binary(self, connect, binary_collection, get_l2_index):
'''
@@ -630,7 +613,6 @@ class TestIndexBinary:
******************************************************************
"""
- @pytest.mark.skip("get_collection_stats does not impl")
def test_get_index_info(self, connect, binary_collection, get_jaccard_index):
'''
target: test describe index interface
@@ -650,7 +632,6 @@ class TestIndexBinary:
if "index_type" in file:
assert file["index_type"] == get_jaccard_index["index_type"]
- @pytest.mark.skip("get_collection_stats does not impl")
def test_get_index_info_partition(self, connect, binary_collection, get_jaccard_index):
'''
target: test describe index interface
@@ -679,7 +660,6 @@ class TestIndexBinary:
******************************************************************
"""
- @pytest.mark.skip("get_collection_stats and drop_index do not impl")
def test_drop_index(self, connect, binary_collection, get_jaccard_index):
'''
target: test drop index interface
@@ -694,7 +674,6 @@ class TestIndexBinary:
# assert stats["partitions"][0]["segments"][0]["index_name"] == default_index_type
assert not stats["partitions"][0]["segments"]
- @pytest.mark.skip("get_collection_stats does not impl")
def test_drop_index_partition(self, connect, binary_collection, get_jaccard_index):
'''
target: test drop index interface
@@ -720,6 +699,7 @@ class TestIndexBinary:
assert False
+@pytest.mark.skip("wait for debugging...")
class TestIndexInvalid(object):
"""
Test create / describe / drop index interfaces with invalid collection names
@@ -758,6 +738,7 @@ class TestIndexInvalid(object):
connect.create_index(collection, field_name, get_simple_index)
+@pytest.mark.skip("wait for debugging...")
class TestIndexAsync:
@pytest.fixture(scope="function", autouse=True)
def skip_http_check(self, args):
diff --git a/tests/python/test_insert.py b/tests/python/test_insert.py
index 3c0420aa1057cd0fd6ff3329f84402d04f7043c0..14948a4c1f1eb42c6d76344bd245f0a09306975b 100644
--- a/tests/python/test_insert.py
+++ b/tests/python/test_insert.py
@@ -101,6 +101,7 @@ class TestInsertBase:
connect.flush([collection])
connect.drop_collection(collection)
+ @pytest.mark.skip("create_index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_create_index(self, connect, collection, get_simple_index):
'''
@@ -118,6 +119,7 @@ class TestInsertBase:
if field["name"] == field_name:
assert field["indexes"][0] == get_simple_index
+ @pytest.mark.skip("create_index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_after_create_index(self, connect, collection, get_simple_index):
'''
@@ -134,6 +136,7 @@ class TestInsertBase:
if field["name"] == field_name:
assert field["indexes"][0] == get_simple_index
+ @pytest.mark.skip(" todo fix search")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_search(self, connect, collection):
'''
@@ -310,6 +313,7 @@ class TestInsertBinary:
connect.flush([binary_collection])
assert connect.count_entities(binary_collection) == default_nb
+ @pytest.mark.skip("create index")
def test_insert_binary_after_create_index(self, connect, binary_collection, get_binary_index):
'''
target: test insert binary entities after build index
@@ -326,6 +330,7 @@ class TestInsertBinary:
if field["name"] == binary_field_name:
assert field["indexes"][0] == get_binary_index
+ @pytest.mark.skip("create index")
@pytest.mark.timeout(ADD_TIMEOUT)
def test_insert_binary_create_index(self, connect, binary_collection, get_binary_index):
'''
diff --git a/tests/python/test_search.py b/tests/python/test_search.py
index d23e4ff0f884cb4f222fc3cebe1f69fc93d8a46b..7ecdcb54958fd2aa9bbba18ed73cbda6ba5cdb4d 100644
--- a/tests/python/test_search.py
+++ b/tests/python/test_search.py
@@ -89,11 +89,10 @@ class TestSearchBase:
params=gen_simple_index()
)
def get_simple_index(self, request, connect):
- import copy
if str(connect._cmd("mode")) == "CPU":
if request.param["index_type"] in index_cpu_not_support():
pytest.skip("sq8h not support in CPU mode")
- return copy.deepcopy(request.param)
+ return request.param
@pytest.fixture(
scope="function",
@@ -256,7 +255,7 @@ class TestSearchBase:
assert res2[0][0].id == res[0][1].id
assert res2[0][0].entity.get("int64") == res[0][1].entity.get("int64")
- # Pass
+ @pytest.mark.skip("search_after_index")
@pytest.mark.level(2)
def test_search_after_index(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
@@ -303,7 +302,7 @@ class TestSearchBase:
assert len(res) == nq
assert len(res[0]) == default_top_k
- # pass
+ @pytest.mark.skip("search_index_partition")
@pytest.mark.level(2)
def test_search_index_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
@@ -334,7 +333,7 @@ class TestSearchBase:
res = connect.search(collection, query, partition_tags=[default_tag])
assert len(res) == nq
- # PASS
+ @pytest.mark.skip("search_index_partition_B")
@pytest.mark.level(2)
def test_search_index_partition_B(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
@@ -384,7 +383,7 @@ class TestSearchBase:
assert len(res) == nq
assert len(res[0]) == 0
- # PASS
+ @pytest.mark.skip("search_index_partitions")
@pytest.mark.level(2)
def test_search_index_partitions(self, connect, collection, get_simple_index, get_top_k):
'''
@@ -418,7 +417,7 @@ class TestSearchBase:
assert res[0]._distances[0] > epsilon
assert res[1]._distances[0] > epsilon
- # Pass
+ @pytest.mark.skip("search_index_partitions_B")
@pytest.mark.level(2)
def test_search_index_partitions_B(self, connect, collection, get_simple_index, get_top_k):
'''
@@ -452,7 +451,7 @@ class TestSearchBase:
assert res[0]._distances[0] < epsilon
assert res[1]._distances[0] < epsilon
- # pass
+ #
# test for ip metric
#
# TODO: reopen after we supporting ip flat
@@ -478,7 +477,7 @@ class TestSearchBase:
with pytest.raises(Exception) as e:
res = connect.search(collection, query)
- # PASS
+ @pytest.mark.skip("search_ip_after_index")
@pytest.mark.level(2)
def test_search_ip_after_index(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
@@ -507,6 +506,7 @@ class TestSearchBase:
assert check_id_result(res[0], ids[0])
assert res[0]._distances[0] >= 1 - gen_inaccuracy(res[0]._distances[0])
+ @pytest.mark.skip("search_ip_index_partition")
@pytest.mark.level(2)
def test_search_ip_index_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
@@ -539,7 +539,7 @@ class TestSearchBase:
res = connect.search(collection, query, partition_tags=[default_tag])
assert len(res) == nq
- # PASS
+ @pytest.mark.skip("search_ip_index_partitions")
@pytest.mark.level(2)
def test_search_ip_index_partitions(self, connect, collection, get_simple_index, get_top_k):
'''
@@ -618,7 +618,7 @@ class TestSearchBase:
res = connect.search(collection, query)
assert abs(np.sqrt(res[0]._distances[0]) - min(distance_0, distance_1)) <= gen_inaccuracy(res[0]._distances[0])
- # Pass
+ @pytest.mark.skip("search_distance_l2_after_index")
def test_search_distance_l2_after_index(self, connect, id_collection, get_simple_index):
'''
target: search collection, and check the result: distance
@@ -672,7 +672,7 @@ class TestSearchBase:
res = connect.search(collection, query)
assert abs(res[0]._distances[0] - max(distance_0, distance_1)) <= epsilon
- # Pass
+ @pytest.mark.skip("search_distance_ip_after_index")
def test_search_distance_ip_after_index(self, connect, id_collection, get_simple_index):
'''
target: search collection, and check the result: distance
@@ -942,7 +942,7 @@ class TestSearchBase:
assert res[i]._distances[0] < epsilon
assert res[i]._distances[1] > epsilon
- @pytest.mark.skip("test_query_entities_with_field_less_than_top_k")
+ @pytest.mark.skip("query_entities_with_field_less_than_top_k")
def test_query_entities_with_field_less_than_top_k(self, connect, id_collection):
"""
target: test search with field, and let return entities less than topk
@@ -1741,7 +1741,8 @@ class TestSearchInvalid(object):
def get_search_params(self, request):
yield request.param
- # Pass
+ # TODO: reopen after we supporting create index
+ @pytest.mark.skip("search_with_invalid_params")
@pytest.mark.level(2)
def test_search_with_invalid_params(self, connect, collection, get_simple_index, get_search_params):
'''
@@ -1762,7 +1763,8 @@ class TestSearchInvalid(object):
with pytest.raises(Exception) as e:
res = connect.search(collection, query)
- # pass
+ # TODO: reopen after we supporting binary type
+ @pytest.mark.skip("search_with_invalid_params_binary")
@pytest.mark.level(2)
def test_search_with_invalid_params_binary(self, connect, binary_collection):
'''
@@ -1781,7 +1783,7 @@ class TestSearchInvalid(object):
with pytest.raises(Exception) as e:
res = connect.search(binary_collection, query)
- # Pass
+ @pytest.mark.skip("search_with_empty_params")
@pytest.mark.level(2)
def test_search_with_empty_params(self, connect, collection, args, get_simple_index):
'''
diff --git a/tests/python/utils.py b/tests/python/utils.py
index 282f7e36f765def05538aa19a9591698389dfcdd..dfc99eb37efd8f8c2f6d48b8627e5aac9db0d07d 100644
--- a/tests/python/utils.py
+++ b/tests/python/utils.py
@@ -55,7 +55,7 @@ default_index_params = [
{"nlist": 128},
{"nlist": 128},
{"nlist": 128},
- {"nlist": 128, "m": 16, "nbits": 8},
+ {"nlist": 128, "m": 16},
{"M": 48, "efConstruction": 500},
# {"search_length": 50, "out_degree": 40, "candidate_pool_size": 100, "knng": 50},
{"n_trees": 50},
@@ -281,9 +281,9 @@ def gen_entities(nb, is_normal=False):
def gen_entities_new(nb, is_normal=False):
vectors = gen_vectors(nb, default_dim, is_normal)
entities = [
- {"name": "int64", "type": DataType.INT64, "values": [i for i in range(nb)]},
- {"name": "float", "type": DataType.FLOAT, "values": [float(i) for i in range(nb)]},
- {"name": default_float_vec_field_name, "type": DataType.FLOAT_VECTOR, "values": vectors}
+ {"name": "int64", "values": [i for i in range(nb)]},
+ {"name": "float", "values": [float(i) for i in range(nb)]},
+ {"name": default_float_vec_field_name, "values": vectors}
]
return entities