Skip to content
Snippets Groups Projects
Commit faa23fde authored by zhenshan.cao's avatar zhenshan.cao Committed by yefu.chen
Browse files

Fix getindexstate failed to fetch indexstate 


Signed-off-by: default avatarzhenshan.cao <zhenshan.cao@zilliz.com>
parent da94fbed
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 445 additions and 385 deletions
internal/core/src/pb/milvus.pb.cc
+ 201
− 156
View file @ faa23fde
This diff is collapsed.
internal/core/src/pb/milvus.pb.h
+ 24
− 3
View file @ faa23fde
......@@ -4912,10 +4912,11 @@ class IndexDescription :
// accessors -------------------------------------------------------
 
enum : int {
kParamsFieldNumber = 2,
kParamsFieldNumber = 3,
kIndexNameFieldNumber = 1,
kIndexIDFieldNumber = 2,
};
// repeated .milvus.proto.common.KeyValuePair params = 2;
// repeated .milvus.proto.common.KeyValuePair params = 3;
int params_size() const;
void clear_params();
::milvus::proto::common::KeyValuePair* mutable_params(int index);
......@@ -4937,6 +4938,11 @@ class IndexDescription :
std::string* release_index_name();
void set_allocated_index_name(std::string* index_name);
 
// int64 indexID = 2;
void clear_indexid();
::PROTOBUF_NAMESPACE_ID::int64 indexid() const;
void set_indexid(::PROTOBUF_NAMESPACE_ID::int64 value);
// @@protoc_insertion_point(class_scope:milvus.proto.milvus.IndexDescription)
private:
class _Internal;
......@@ -4944,6 +4950,7 @@ class IndexDescription :
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair > params_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr index_name_;
::PROTOBUF_NAMESPACE_ID::int64 indexid_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_milvus_2eproto;
};
......@@ -12036,7 +12043,21 @@ inline void IndexDescription::set_allocated_index_name(std::string* index_name)
// @@protoc_insertion_point(field_set_allocated:milvus.proto.milvus.IndexDescription.index_name)
}
 
// repeated .milvus.proto.common.KeyValuePair params = 2;
// int64 indexID = 2;
inline void IndexDescription::clear_indexid() {
indexid_ = PROTOBUF_LONGLONG(0);
}
inline ::PROTOBUF_NAMESPACE_ID::int64 IndexDescription::indexid() const {
// @@protoc_insertion_point(field_get:milvus.proto.milvus.IndexDescription.indexID)
return indexid_;
}
inline void IndexDescription::set_indexid(::PROTOBUF_NAMESPACE_ID::int64 value) {
indexid_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.milvus.IndexDescription.indexID)
}
// repeated .milvus.proto.common.KeyValuePair params = 3;
inline int IndexDescription::params_size() const {
return params_.size();
}
......
internal/indexservice/indexservice.go
+ 25
− 0
View file @ faa23fde
......@@ -128,6 +128,9 @@ func (i *ServiceImpl) Init() error {
}
func (i *ServiceImpl) Start() error {
i.loopWg.Add(1)
go i.tsLoop()
i.sched.Start()
// Start callbacks
for _, cb := range i.startCallbacks {
......@@ -223,6 +226,7 @@ func (i *ServiceImpl) BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.Build
ret.Status.Reason = err.Error()
return ret, nil
}
ret.Status.ErrorCode = commonpb.ErrorCode_SUCCESS
ret.IndexBuildID = t.indexBuildID
return ret, nil
}
......@@ -276,3 +280,24 @@ func (i *ServiceImpl) NotifyBuildIndex(nty *indexpb.BuildIndexNotification) (*co
i.nodeClients.IncPriority(nty.NodeID, -1)
return ret, nil
}
func (i *ServiceImpl) tsLoop() {
tsoTicker := time.NewTicker(UpdateTimestampStep)
defer tsoTicker.Stop()
ctx, cancel := context.WithCancel(i.loopCtx)
defer cancel()
defer i.loopWg.Done()
for {
select {
case <-tsoTicker.C:
if err := i.idAllocator.UpdateID(); err != nil {
log.Println("failed to update id", err)
return
}
case <-ctx.Done():
// Server is closed and it should return nil.
log.Println("tsLoop is closed")
return
}
}
}
internal/masterservice/task.go
+ 1
− 0
View file @ faa23fde
......@@ -630,6 +630,7 @@ func (t *DescribeIndexReqTask) Execute() error {
desc := &milvuspb.IndexDescription{
IndexName: i.IndexName,
Params: i.IndexParams,
IndexID: i.IndexID,
}
t.Rsp.IndexDescriptions = append(t.Rsp.IndexDescriptions, desc)
}
......
internal/proto/milvus.proto
+ 2
− 1
View file @ faa23fde
......@@ -186,7 +186,8 @@ message DescribeIndexRequest {
message IndexDescription {
string index_name = 1;
repeated common.KeyValuePair params = 2;
int64 indexID = 2;
repeated common.KeyValuePair params = 3;
}
message DescribeIndexResponse {
......
internal/proto/milvuspb/milvus.pb.go
+ 131
− 123
View file @ faa23fde
This diff is collapsed.
internal/proxynode/impl.go
+ 4
− 4
View file @ faa23fde
......@@ -545,10 +545,10 @@ func (node *NodeImpl) GetIndexState(request *milvuspb.IndexStateRequest) (*milvu
ctx, cancel := context.WithTimeout(context.Background(), reqTimeoutInterval)
defer cancel()
dipt := &GetIndexStateTask{
Condition: NewTaskCondition(ctx),
IndexStateRequest: request,
indexServiceClient: node.indexServiceClient,
masterClientInterface: node.masterClient,
Condition: NewTaskCondition(ctx),
IndexStateRequest: request,
indexServiceClient: node.indexServiceClient,
masterClient: node.masterClient,
}
err := node.sched.DdQueue.Enqueue(dipt)
......
internal/proxynode/paramtable.go
+ 11
− 89
View file @ faa23fde
......@@ -51,6 +51,7 @@ type ParamTable struct {
MaxFieldNum int64
MaxDimension int64
DefaultPartitionTag string
DefaultIndexName string
}
var Params ParamTable
......@@ -108,95 +109,6 @@ func (pt *ParamTable) LoadConfigFromInitParams(initParams *internalpb2.InitParam
}
pt.initParams()
//
//pulsarPort := config.GetString(PulsarPort)
//pulsarHost := config.GetString(PulsarHost)
//pt.PulsarAddress = pulsarHost + ":" + pulsarPort
//
//
//queryNodeIDList := config.GetString(QueryNodeIDList)
//pt.QueryNodeIDList = nil
//queryNodeIDs := strings.Split(queryNodeIDList, ",")
//for _, queryNodeID := range queryNodeIDs {
// v, err := strconv.Atoi(queryNodeID)
// if err != nil {
// return err
// }
// pt.QueryNodeIDList = append(pt.QueryNodeIDList, typeutil.UniqueID(v))
//}
//pt.QueryNodeNum = len(pt.QueryNodeIDList)
//
//timeTickInterval := config.GetString(TimeTickInterval)
//interval, err := strconv.Atoi(timeTickInterval)
//if err != nil {
// return err
//}
//pt.TimeTickInterval = time.Duration(interval) * time.Millisecond
//
//subName := config.GetString(SubName)
//pt.ProxySubName = subName
//
//timeTickChannelNames := config.GetString(TimeTickChannelNames)
//pt.ProxyTimeTickChannelNames = []string{timeTickChannelNames}
//
//msgStreamInsertBufSizeStr := config.GetString(MsgStreamInsertBufSize)
//msgStreamInsertBufSize, err := strconv.Atoi(msgStreamInsertBufSizeStr)
//if err != nil {
// return err
//}
//pt.MsgStreamInsertBufSize = int64(msgStreamInsertBufSize)
//
//msgStreamSearchBufSizeStr := config.GetString(MsgStreamSearchBufSize)
//msgStreamSearchBufSize, err := strconv.Atoi(msgStreamSearchBufSizeStr)
//if err != nil {
// return err
//}
//pt.MsgStreamSearchBufSize = int64(msgStreamSearchBufSize)
//
//msgStreamSearchResultBufSizeStr := config.GetString(MsgStreamSearchResultBufSize)
//msgStreamSearchResultBufSize, err := strconv.Atoi(msgStreamSearchResultBufSizeStr)
//if err != nil {
// return err
//}
//pt.MsgStreamSearchResultBufSize = int64(msgStreamSearchResultBufSize)
//
//msgStreamSearchResultPulsarBufSizeStr := config.GetString(MsgStreamSearchResultPulsarBufSize)
//msgStreamSearchResultPulsarBufSize, err := strconv.Atoi(msgStreamSearchResultPulsarBufSizeStr)
//if err != nil {
// return err
//}
//pt.MsgStreamSearchResultPulsarBufSize = int64(msgStreamSearchResultPulsarBufSize)
//
//msgStreamTimeTickBufSizeStr := config.GetString(MsgStreamTimeTickBufSize)
//msgStreamTimeTickBufSize, err := strconv.Atoi(msgStreamTimeTickBufSizeStr)
//if err != nil {
// return err
//}
//pt.MsgStreamTimeTickBufSize = int64(msgStreamTimeTickBufSize)
//
//maxNameLengthStr := config.GetString(MaxNameLength)
//maxNameLength, err := strconv.Atoi(maxNameLengthStr)
//if err != nil {
// return err
//}
//pt.MaxNameLength = int64(maxNameLength)
//
//maxFieldNumStr := config.GetString(MaxFieldNum)
//maxFieldNum, err := strconv.Atoi(maxFieldNumStr)
//if err != nil {
// return err
//}
//pt.MaxFieldNum = int64(maxFieldNum)
//
//maxDimensionStr := config.GetString(MaxDimension)
//maxDimension, err := strconv.Atoi(maxDimensionStr)
//if err != nil {
// return err
//}
//pt.MaxDimension = int64(maxDimension)
//
//defaultPartitionTag := config.GetString(DefaultPartitionTag)
//pt.DefaultPartitionTag = defaultPartitionTag
return nil
}
......@@ -235,6 +147,8 @@ func (pt *ParamTable) initParams() {
pt.initMaxFieldNum()
pt.initMaxDimension()
pt.initDefaultPartitionTag()
pt.initDefaultIndexName()
}
func (pt *ParamTable) initPulsarAddress() {
......@@ -475,3 +389,11 @@ func (pt *ParamTable) initDefaultPartitionTag() {
}
pt.DefaultPartitionTag = tag
}
func (pt *ParamTable) initDefaultIndexName() {
name, err := pt.Load("common.defaultIndexName")
if err != nil {
panic(err)
}
pt.DefaultIndexName = name
}
internal/proxynode/task.go
+ 45
− 8
View file @ faa23fde
......@@ -3,6 +3,7 @@ package proxynode
import (
"context"
"errors"
"fmt"
"log"
"math"
"strconv"
......@@ -1346,9 +1347,9 @@ func (dit *DescribeIndexTask) PostExecute() error {
type GetIndexStateTask struct {
Condition
*milvuspb.IndexStateRequest
indexServiceClient IndexServiceClient
masterClientInterface MasterClient
result *milvuspb.IndexStateResponse
indexServiceClient IndexServiceClient
masterClient MasterClient
result *milvuspb.IndexStateResponse
}
func (dipt *GetIndexStateTask) OnEnqueue() error {
......@@ -1415,11 +1416,46 @@ func (dipt *GetIndexStateTask) Execute() error {
CollectionName: collectionName,
CollectionID: collectionID,
}
partitions, err := dipt.masterClientInterface.ShowPartitions(showPartitionRequest)
partitions, err := dipt.masterClient.ShowPartitions(showPartitionRequest)
if err != nil {
return err
}
if dipt.IndexName == "" {
dipt.IndexName = Params.DefaultIndexName
}
describeIndexReq := milvuspb.DescribeIndexRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_kDescribeIndex,
MsgID: dipt.Base.MsgID,
Timestamp: dipt.Base.Timestamp,
SourceID: Params.ProxyID,
},
DbName: dipt.DbName,
CollectionName: dipt.CollectionName,
FieldName: dipt.FieldName,
IndexName: dipt.IndexName,
}
indexDescriptionResp, err2 := dipt.masterClient.DescribeIndex(&describeIndexReq)
if err2 != nil {
return err2
}
matchIndexID := int64(-1)
foundIndexID := false
for _, desc := range indexDescriptionResp.IndexDescriptions {
if desc.IndexName == dipt.IndexName {
matchIndexID = desc.IndexID
foundIndexID = true
break
}
}
if !foundIndexID {
return errors.New(fmt.Sprint("Can't found IndexID for indexName", dipt.IndexName))
}
for _, partitionID := range partitions.PartitionIDs {
showSegmentsRequest := &milvuspb.ShowSegmentRequest{
Base: &commonpb.MsgBase{
......@@ -1431,7 +1467,7 @@ func (dipt *GetIndexStateTask) Execute() error {
CollectionID: collectionID,
PartitionID: partitionID,
}
segments, err := dipt.masterClientInterface.ShowSegments(showSegmentsRequest)
segments, err := dipt.masterClient.ShowSegments(showSegmentsRequest)
if err != nil {
return err
}
......@@ -1450,12 +1486,13 @@ func (dipt *GetIndexStateTask) Execute() error {
CollectionID: collectionID,
SegmentID: segmentID,
}
segmentDesc, err := dipt.masterClientInterface.DescribeSegment(describeSegmentRequest)
segmentDesc, err := dipt.masterClient.DescribeSegment(describeSegmentRequest)
if err != nil {
return err
}
getIndexStatesRequest.IndexBuildIDs = append(getIndexStatesRequest.IndexBuildIDs, segmentDesc.BuildID)
if segmentDesc.IndexID == matchIndexID {
getIndexStatesRequest.IndexBuildIDs = append(getIndexStatesRequest.IndexBuildIDs, segmentDesc.BuildID)
}
}
states, err := dipt.indexServiceClient.GetIndexStates(getIndexStatesRequest)
......
tests/python/requirements.txt
+ 1
− 1
View file @ faa23fde
......@@ -2,7 +2,7 @@ grpcio==1.26.0
grpcio-tools==1.26.0
numpy==1.18.1
pytest-cov==2.8.1
pymilvus-distributed==0.0.22
pymilvus-distributed==0.0.23
sklearn==0.0
pytest==4.5.0
pytest-timeout==1.3.3
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment