Update proto:add partition id

Signed-off-by: neza2017 <yefu.chen@zilliz.com>

cmd/indexbuilder/indexbuilder.go

@@ -7,14 +7,14 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/zilliztech/milvus-distributed/internal/indexnode"
+	"github.com/zilliztech/milvus-distributed/internal/indexbuilder"
 	"go.uber.org/zap"
 )
 
 func main() {
-	indexnode.Init()
+	indexbuilder.Init()
 	ctx, cancel := context.WithCancel(context.Background())
-	svr, err := indexnode.CreateBuilder(ctx)
+	svr, err := indexbuilder.CreateBuilder(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
 	}

cmd/proxy/proxy.go

@@ -8,15 +8,15 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/zilliztech/milvus-distributed/internal/proxynode"
+	"github.com/zilliztech/milvus-distributed/internal/proxy"
 	"go.uber.org/zap"
 )
 
 func main() {
-	proxynode.Init()
-	fmt.Println("ProxyID is", proxynode.Params.ProxyID())
+	proxy.Init()
+	fmt.Println("ProxyID is", proxy.Params.ProxyID())
 	ctx, cancel := context.WithCancel(context.Background())
-	svr, err := proxynode.CreateProxy(ctx)
+	svr, err := proxy.CreateProxy(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
 	}

cmd/singlenode/main.go

@@ -14,9 +14,9 @@ import (
 
 	"go.uber.org/zap"
 
-	"github.com/zilliztech/milvus-distributed/internal/indexnode"
+	"github.com/zilliztech/milvus-distributed/internal/indexbuilder"
 	"github.com/zilliztech/milvus-distributed/internal/master"
-	"github.com/zilliztech/milvus-distributed/internal/proxynode"
+	"github.com/zilliztech/milvus-distributed/internal/proxy"
 	"github.com/zilliztech/milvus-distributed/internal/querynode"
 	"github.com/zilliztech/milvus-distributed/internal/writenode"
 )
@@ -62,10 +62,10 @@ func InitMaster(cpuprofile *string, wg *sync.WaitGroup) {
 
 func InitProxy(wg *sync.WaitGroup) {
 	defer wg.Done()
-	proxynode.Init()
-	fmt.Println("ProxyID is", proxynode.Params.ProxyID())
+	proxy.Init()
+	fmt.Println("ProxyID is", proxy.Params.ProxyID())
 	ctx, cancel := context.WithCancel(context.Background())
-	svr, err := proxynode.CreateProxy(ctx)
+	svr, err := proxy.CreateProxy(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
 	}
@@ -138,9 +138,9 @@ func InitQueryNode(wg *sync.WaitGroup) {
 
 func InitIndexBuilder(wg *sync.WaitGroup) {
 	defer wg.Done()
-	indexnode.Init()
+	indexbuilder.Init()
 	ctx, cancel := context.WithCancel(context.Background())
-	svr, err := indexnode.CreateBuilder(ctx)
+	svr, err := indexbuilder.CreateBuilder(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
 	}

configs/advanced/proxy_node.yaml → configs/advanced/proxy.yaml

@@ -9,7 +9,7 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 
-proxyNode:
+proxy:
   timeTickInterval: 200 # ms
 
   msgStream:
@@ -29,4 +29,4 @@ proxyNode:
 
   maxNameLength: 255
   maxFieldNum: 64
-  maxDimension: 32768
+  maxDimension: 32768
\ No newline at end of file

configs/milvus.yaml

@@ -44,7 +44,7 @@ master:
   address: localhost
   port: 53100
 
-proxyNode:
+proxy:
   address: localhost
   port: 19530
 

docs/developer_guides/chap03_index_service.md

@@ -26,7 +26,7 @@ type IndexService interface {
 
 ```go
 type RegisterNodeRequest struct {
-  MsgBase
+  RequestBase
   Address string
   Port int64
 }

docs/developer_guides/chap04_message_stream.md

-## 8. Message Stream
+## 8. Message Stream Service
 
 
 
+#### 8.1 Overview
+
 
 
 #### 8.2 Message Stream Service API
@@ -65,14 +67,6 @@ type DescribeChannelResponse struct {
 
 #### A.3 Message Stream
 
-* Overview
-
-
-
-<img src="./figs/msg_stream_input_output.jpeg" width=700>
-
-* Interface
-
 ``` go
 type MsgType uint32
 const {

docs/developer_guides/chap05_proxy.md

@@ -19,14 +19,14 @@ type ProxyService interface {
 
 
 
-* *MsgBase*
+* *RequestBase*
 
 ```go
-type MsgBase struct {
+type RequestBase struct {
   MsgType MsgType
-  MsgID	UniqueID
+  ReqID	UniqueID
   Timestamp Timestamp
-  SourceID UniqueID
+  RequestorID UniqueID
 }
 ```
 
@@ -43,7 +43,7 @@ type RegisterLinkResponse struct {
 
 ```go
 type RegisterNodeRequest struct {
-  MsgBase
+  RequestBase
   Address string
   Port int64
 }
@@ -57,7 +57,7 @@ type RegisterNodeResponse struct {
 
 ```go
 type InvalidateCollMetaCacheRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }
@@ -117,7 +117,7 @@ See *Master API* for detailed definitions.
 
 ```go
 type LoadCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }
@@ -127,7 +127,7 @@ type LoadCollectionRequest struct {
 
 ```go
 type ReleaseCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }
@@ -161,7 +161,7 @@ See *Master API* for detailed definitions.
 
 ```go
 type LoadPartitonRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   PartitionNames []string
@@ -172,7 +172,7 @@ type LoadPartitonRequest struct {
 
 ```go
 type ReleasePartitionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   PartitionNames []string
@@ -199,7 +199,7 @@ See *Master API* for detailed definitions.
 
 ```go
 type InsertRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   PartitionName string
@@ -217,7 +217,7 @@ type InsertResponse struct {
 
 ```go
 type SearchRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   PartitionNames []string
@@ -230,7 +230,7 @@ type SearchRequest struct {
 
 ```go
 type FlushRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }

docs/developer_guides/chap06_master.md

@@ -35,14 +35,14 @@ type Master interface {
 
 
 
-* *MsgBase*
+* *RequestBase*
 
 ```go
-type MsgBase struct {
+type RequestBase struct {
   MsgType MsgType
-  MsgID	UniqueID
+  ReqID	UniqueID
   Timestamp Timestamp
-  SourceID UniqueID
+  RequestorID UniqueID
 }
 ```
 
@@ -50,7 +50,7 @@ type MsgBase struct {
 
 ```go
 type CreateCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   Schema []bytes
@@ -61,7 +61,7 @@ type CreateCollectionRequest struct {
 
 ```go
 type DropCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }
@@ -71,7 +71,7 @@ type DropCollectionRequest struct {
 
 ```go
 type HasCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }
@@ -81,7 +81,7 @@ type HasCollectionRequest struct {
 
 ```go
 type DescribeCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }
@@ -95,7 +95,7 @@ type DescribeCollectionResponse struct {
 
 ```go
 type CollectionStatsRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }
@@ -109,7 +109,7 @@ type CollectionStatsResponse struct {
 
 ```go
 type ShowCollectionRequest struct {
-  MsgBase
+  RequestBase
 	DbName string
 }
 
@@ -122,7 +122,7 @@ type ShowCollectionResponse struct {
 
 ```go
 type CreatePartitionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   PartitionName string
@@ -133,7 +133,7 @@ type CreatePartitionRequest struct {
 
 ```go
 type DropPartitionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   PartitionName string
@@ -144,7 +144,7 @@ type DropPartitionRequest struct {
 
 ```go
 type HasPartitionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   PartitionName string
@@ -155,7 +155,7 @@ type HasPartitionRequest struct {
 
 ```go
 type PartitionStatsRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   PartitionName string
@@ -170,7 +170,7 @@ type PartitionStatsResponse struct {
 
 ```go
 type ShowPartitionRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
 }
@@ -184,7 +184,7 @@ type ShowPartitionResponse struct {
 
 ```go
 type CreateIndexRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   FieldName string
@@ -196,7 +196,7 @@ type CreateIndexRequest struct {
 
 ```go
 type DescribeIndexRequest struct {
-  MsgBase
+  RequestBase
   DbName string
   CollectionName string
   FieldName string
@@ -216,7 +216,7 @@ type DescribeIndexResponse struct {
 
 ```go
 type TsoRequest struct {
-  MsgBase
+  RequestBase
   Count uint32
 }
 
@@ -230,7 +230,7 @@ type TsoResponse struct {
 
 ```go
 type IDRequest struct {
-  MsgBase
+  RequestBase
   Count uint32
 }
 
@@ -242,75 +242,25 @@ type IDResponse struct {
 
 
 
-#### 10.2 Dd (Data definitions) Channel
-
-* *CreateCollection*
+#### 10.1 Interfaces (RPC)
 
-```go
-type CreateCollectionRequest struct {
-  RequestBase
-  DbName string
-  CollectionName string
-  DbID UniqueID
-  CollectionID UniqueID
-  Schema []bytes
-}
-```
-
-* *DropCollection*
-
-```go
-type DropCollectionRequest struct {
-  RequestBase
-  DbName string
-  CollectionName string
-  DbID UniqueID
-  CollectionID UniqueID
-}
-```
-
-* *CreatePartition*
-
-```go
-type CreatePartitionRequest struct {
-  RequestBase
-  DbName string
-  CollectionName string
-  PartitionName string
-  DbID UniqueID
-  CollectionID UniqueID
-  PartitionID UniqueID
-}
-```
-
-* *DropPartition*
-
-```go
-type DropPartitionRequest struct {
-  RequestBase
-  DbName string
-  CollectionName string
-  PartitionName string
-  DbID UniqueID
-  CollectionID UniqueID
-  PartitionID UniqueID
-}
-```
-
-* *CreateIndex*
-
-```go
-type CreateIndexRequest struct {
-  RequestBase
-  DbName string
-  CollectionName string
-  FieldName string
-  DbID UniqueID
-  CollectionID UniqueID
-  FieldID int64
-  Params [] KeyValuePair
-}
-```
+| RPC                | description                                                  |
+| :----------------- | ------------------------------------------------------------ |
+| CreateCollection   | create a collection base on schema statement                 |
+| DropCollection     | drop a collection                                            |
+| HasCollection      | whether or not a collection exists                           |
+| DescribeCollection | show a collection's schema and its descriptive statistics    |
+| ShowCollections    | list all collections                                         |
+| CreatePartition    | create a partition                                           |
+| DropPartition      | drop a partition                                             |
+| HasPartition       | whether or not a partition exists                            |
+| DescribePartition  | show a partition's name and its descriptive statistics       |
+| ShowPartitions     | list a collection's all partitions                           |
+| AllocTimestamp     | allocate a batch of consecutive timestamps                   |
+| AllocID            | allocate a batch of consecutive IDs                          |
+| AssignSegmentID    | assign segment id to insert rows (master determines which segment these rows belong to) |
+| GetSysConfigs      | get system configurations                                    |
+|                    |                                                              |
 
 
 

docs/developer_guides/chap07_query_service.md

@@ -31,14 +31,14 @@ type QueryService interface {
 
 
 
-* *MsgBase*
+* *RequestBase*
 
 ```go
-type MsgBase struct {
+type RequestBase struct {
   MsgType MsgType
-  MsgID	UniqueID
+  ReqID	UniqueID
   Timestamp Timestamp
-  SourceID UniqueID
+  RequestorID UniqueID
 }
 ```
 
@@ -46,7 +46,7 @@ type MsgBase struct {
 
 ```go
 type RegisterNodeRequest struct {
-  MsgBase
+  RequestBase
   Address string
   Port int64
 }
@@ -60,7 +60,7 @@ type RegisterNodeResponse struct {
 
 ```go
 type ShowCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
 }
 
@@ -73,7 +73,7 @@ type ShowCollectionResponse struct {
 
 ```go
 type LoadCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
 }
@@ -83,7 +83,7 @@ type LoadCollectionRequest struct {
 
 ```go
 type ReleaseCollectionRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
 }
@@ -93,7 +93,7 @@ type ReleaseCollectionRequest struct {
 
 ```go
 type ShowPartitionRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
 }
@@ -119,7 +119,7 @@ const (
 )
 
 type PartitionStatesRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
   PartitionIDs []UniqueID
@@ -139,7 +139,7 @@ type PartitionStatesResponse struct {
 
 ```go
 type LoadPartitonRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
   PartitionIDs []UniqueID
@@ -150,7 +150,7 @@ type LoadPartitonRequest struct {
 
 ```go
 type ReleasePartitionRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
   PartitionIDs []UniqueID
@@ -168,24 +168,6 @@ type CreateQueryChannelResponse struct {
 
 
 
-#### 8.2 Query Channel
-
-```go
-type SearchRequest struct {
-  RequestBase
-  DbName string
-  CollectionName string
-  PartitionNames []string
-  DbID UniqueID
-  CollectionID UniqueID
-  PartitionIDs []UniqueID
-  Dsl string
-  PlaceholderGroup []byte
-}
-```
-
-
-
 #### 8.2 Query Node Interface
 
 ```go
@@ -210,7 +192,7 @@ type QueryNode interface {
 
 ```go
 type AddQueryChannelRequest struct {
-  MsgBase
+  RequestBase
   RequestChannelName string
   ResultChannelName string
 }
@@ -237,7 +219,7 @@ type WatchDmChannelRequest struct {
 
 ```go
 type LoadSegmentRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
   PartitionID UniqueID
@@ -250,7 +232,7 @@ type LoadSegmentRequest struct {
 
 ```go
 type ReleaseSegmentRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
   PartitionID UniqueID

docs/developer_guides/chap09_data_service.md

@@ -28,14 +28,14 @@ type DataService interface {
 
 
 
-* *MsgBase*
+* *RequestBase*
 
 ```go
-type MsgBase struct {
+type RequestBase struct {
   MsgType MsgType
-  MsgID	UniqueID
+  ReqID	UniqueID
   Timestamp Timestamp
-  SourceID UniqueID
+  RequestorID UniqueID
 }
 ```
 
@@ -43,7 +43,7 @@ type MsgBase struct {
 
 ```go
 type RegisterNodeRequest struct {
-  MsgBase
+  RequestBase
   Address string
   Port int64
 }
@@ -64,7 +64,7 @@ type SegIDRequest struct {
 }
 
 type AssignSegIDRequest struct {
-  MsgBase
+  RequestBase
   PerChannelRequest []SegIDRequest
 }
 
@@ -88,7 +88,7 @@ type AssignSegIDResponse struct {
 
 ```go
 type FlushRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
 }
@@ -100,7 +100,7 @@ type FlushRequest struct {
 
 ```go
 type ShowSegmentRequest struct {
-  MsgBase
+  RequestBase
   CollectionID UniqueID
   PartitionID UniqueID
 }
@@ -123,7 +123,7 @@ enum SegmentState {
 }
 
 type SegmentStatesRequest struct {
-  MsgBase
+  RequestBase
   SegmentID UniqueID
 }
 
@@ -140,7 +140,7 @@ type SegmentStatesResponse struct {
 
 ```go
 type InsertBinlogPathRequest struct {
-  MsgBase
+  RequestBase
   SegmentID UniqueID
 }
 
@@ -155,27 +155,9 @@ type InsertBinlogPathsResponse struct {
 
 ```go
 type InsertChannelRequest struct {
-  MsgBase
-  DbID UniqueID
-  CollectionID UniqueID
-}
-```
-
-
-
-#### 8.2 Insert Channel
-
-```go
-type InsertRequest struct {
   RequestBase
-  DbName string
-  CollectionName string
-  PartitionName string
   DbID UniqueID
   CollectionID UniqueID
-  PartitionID UniqueID
-  RowData []Blob
-  HashKeys []uint32
 }
 ```
 
@@ -202,7 +184,7 @@ type DataNode interface {
 
 ```go
 type WatchDmChannelRequest struct {
-  MsgBase
+  RequestBase
   InsertChannelNames []string
 }
 ```
@@ -211,7 +193,7 @@ type WatchDmChannelRequest struct {
 
 ```go
 type FlushSegRequest struct {
-  MsgBase
+  RequestBase
   DbID UniqueID
   CollectionID UniqueID
   SegmentID []UniqueID

internal/core/src/common/Span.h

@@ -13,8 +13,6 @@
 #include <type_traits>
 #include "common/Types.h"
 #include <cassert>
-#include "VectorTrait.h"
-
 namespace milvus {
 // type erasure to work around virtual restriction
 class SpanBase {
@@ -88,8 +86,14 @@ class Span<T, typename std::enable_if_t<std::is_fundamental_v<T>>> {
     const int64_t row_count_;
 };
 
+namespace segcore {
+class VectorTrait;
+class FloatVector;
+class BinaryVector;
+}  // namespace segcore
+
 template <typename VectorType>
-class Span<VectorType, typename std::enable_if_t<std::is_base_of_v<VectorTrait, VectorType>>> {
+class Span<VectorType, typename std::enable_if_t<std::is_base_of_v<segcore::VectorTrait, VectorType>>> {
  public:
     using embedded_type = typename VectorType::embedded_type;
 

internal/core/src/common/Types.h

@@ -11,7 +11,6 @@
 
 #pragma once
 #include "utils/Types.h"
-#include "faiss/utils/BitsetView.h"
 #include <faiss/MetricType.h>
 #include <string>
 #include <boost/align/aligned_allocator.hpp>
@@ -76,14 +75,4 @@ using FieldId = fluent::NamedType<int64_t, struct FieldIdTag, fluent::Comparable
 using FieldName = fluent::NamedType<std::string, struct FieldNameTag, fluent::Comparable, fluent::Hashable>;
 using FieldOffset = fluent::NamedType<int64_t, struct FieldOffsetTag, fluent::Comparable, fluent::Hashable>;
 
-using BitsetView = faiss::BitsetView;
-inline BitsetView
-BitsetSubView(const BitsetView& view, int64_t offset, int64_t size) {
-    if (view.empty()) {
-        return BitsetView();
-    }
-    assert(offset % 8 == 0);
-    return BitsetView(view.data() + offset / 8, size);
-}
-
 }  // namespace milvus

internal/core/src/common/VectorTrait.h

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#pragma once
-#include "Types.h"
-
-namespace milvus {
-
-class VectorTrait {};
-
-class FloatVector : public VectorTrait {
- public:
-    using embedded_type = float;
-    static constexpr auto metric_type = DataType::VECTOR_FLOAT;
-};
-
-class BinaryVector : public VectorTrait {
- public:
-    using embedded_type = uint8_t;
-    static constexpr auto metric_type = DataType::VECTOR_BINARY;
-};
-
-template <typename VectorType>
-inline constexpr int64_t
-get_element_sizeof(int64_t dim) {
-    static_assert(std::is_base_of_v<VectorType, VectorTrait>);
-    if constexpr (std::is_same_v<VectorType, FloatVector>) {
-        return dim * sizeof(float);
-    } else {
-        return dim / 8;
-    }
-}
-
-template <typename T>
-constexpr bool IsVector = std::is_base_of_v<VectorTrait, T>;
-
-template <typename T>
-constexpr bool IsScalar = std::is_fundamental_v<T>;
-
-template <typename T, typename Enabled = void>
-struct EmbeddedTypeImpl;
-
-template <typename T>
-struct EmbeddedTypeImpl<T, std::enable_if_t<IsScalar<T>>> {
-    using type = T;
-};
-
-template <typename T>
-struct EmbeddedTypeImpl<T, std::enable_if_t<IsVector<T>>> {
-    using type = std::conditional_t<std::is_same_v<T, FloatVector>, float, uint8_t>;
-};
-
-template <typename T>
-using EmbeddedType = typename EmbeddedTypeImpl<T>::type;
-
-}  // namespace milvus

internal/core/src/pb/etcd_meta.pb.cc

@@ -169,6 +169,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_etcd_5fmeta_2eproto::offsets[]
   PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, create_time_),
   PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, segmentids_),
   PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, partition_tags_),
+  PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, partitionids_),
   ~0u,  // no _has_bits_
   PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::FieldBinlogFiles, _internal_metadata_),
   ~0u,  // no _extensions_
@@ -207,9 +208,9 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB
   { 0, -1, sizeof(::milvus::proto::etcd::TenantMeta)},
   { 9, -1, sizeof(::milvus::proto::etcd::ProxyMeta)},
   { 17, -1, sizeof(::milvus::proto::etcd::CollectionMeta)},
-  { 27, -1, sizeof(::milvus::proto::etcd::FieldBinlogFiles)},
-  { 34, -1, sizeof(::milvus::proto::etcd::SegmentMeta)},
-  { 49, -1, sizeof(::milvus::proto::etcd::FieldIndexMeta)},
+  { 28, -1, sizeof(::milvus::proto::etcd::FieldBinlogFiles)},
+  { 35, -1, sizeof(::milvus::proto::etcd::SegmentMeta)},
+  { 50, -1, sizeof(::milvus::proto::etcd::FieldIndexMeta)},
 };
 
 static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
@@ -229,25 +230,26 @@ const char descriptor_table_protodef_etcd_5fmeta_2eproto[] PROTOBUF_SECTION_VARI
   "\003(\t\022\027\n\017query_channelID\030\004 \001(\t\"a\n\tProxyMet"
   "a\022\n\n\002ID\030\001 \001(\003\022-\n\007address\030\002 \001(\0132\034.milvus."
   "proto.common.Address\022\031\n\021result_channelID"
-  "s\030\003 \003(\t\"\224\001\n\016CollectionMeta\022\n\n\002ID\030\001 \001(\003\0225"
+  "s\030\003 \003(\t\"\252\001\n\016CollectionMeta\022\n\n\002ID\030\001 \001(\003\0225"
   "\n\006schema\030\002 \001(\0132%.milvus.proto.schema.Col"
   "lectionSchema\022\023\n\013create_time\030\003 \001(\004\022\022\n\nse"
-  "gmentIDs\030\004 \003(\003\022\026\n\016partition_tags\030\005 \003(\t\"9"
-  "\n\020FieldBinlogFiles\022\017\n\007fieldID\030\001 \001(\003\022\024\n\014b"
-  "inlog_files\030\002 \003(\t\"\204\002\n\013SegmentMeta\022\021\n\tseg"
-  "mentID\030\001 \001(\003\022\024\n\014collectionID\030\002 \001(\003\022\025\n\rpa"
-  "rtition_tag\030\003 \001(\t\022\025\n\rchannel_start\030\004 \001(\005"
-  "\022\023\n\013channel_end\030\005 \001(\005\022\021\n\topen_time\030\006 \001(\004"
-  "\022\022\n\nclose_time\030\007 \001(\004\022\020\n\010num_rows\030\010 \001(\003\022\020"
-  "\n\010mem_size\030\t \001(\003\022>\n\021binlog_file_paths\030\n "
-  "\003(\0132#.milvus.proto.etcd.FieldBinlogFiles"
-  "\"\313\001\n\016FieldIndexMeta\022\021\n\tsegmentID\030\001 \001(\003\022\017"
-  "\n\007fieldID\030\002 \001(\003\022\017\n\007indexID\030\003 \001(\003\0227\n\014inde"
-  "x_params\030\004 \003(\0132!.milvus.proto.common.Key"
-  "ValuePair\0221\n\006status\030\005 \001(\0162!.milvus.proto"
-  ".service.IndexStatus\022\030\n\020index_file_paths"
-  "\030\006 \003(\tB@Z>github.com/zilliztech/milvus-d"
-  "istributed/internal/proto/etcdpbb\006proto3"
+  "gmentIDs\030\004 \003(\003\022\026\n\016partition_tags\030\005 \003(\t\022\024"
+  "\n\014partitionIDs\030\006 \003(\003\"9\n\020FieldBinlogFiles"
+  "\022\017\n\007fieldID\030\001 \001(\003\022\024\n\014binlog_files\030\002 \003(\t\""
+  "\204\002\n\013SegmentMeta\022\021\n\tsegmentID\030\001 \001(\003\022\024\n\014co"
+  "llectionID\030\002 \001(\003\022\025\n\rpartition_tag\030\003 \001(\t\022"
+  "\025\n\rchannel_start\030\004 \001(\005\022\023\n\013channel_end\030\005 "
+  "\001(\005\022\021\n\topen_time\030\006 \001(\004\022\022\n\nclose_time\030\007 \001"
+  "(\004\022\020\n\010num_rows\030\010 \001(\003\022\020\n\010mem_size\030\t \001(\003\022>"
+  "\n\021binlog_file_paths\030\n \003(\0132#.milvus.proto"
+  ".etcd.FieldBinlogFiles\"\313\001\n\016FieldIndexMet"
+  "a\022\021\n\tsegmentID\030\001 \001(\003\022\017\n\007fieldID\030\002 \001(\003\022\017\n"
+  "\007indexID\030\003 \001(\003\0227\n\014index_params\030\004 \003(\0132!.m"
+  "ilvus.proto.common.KeyValuePair\0221\n\006statu"
+  "s\030\005 \001(\0162!.milvus.proto.service.IndexStat"
+  "us\022\030\n\020index_file_paths\030\006 \003(\tB@Z>github.c"
+  "om/zilliztech/milvus-distributed/interna"
+  "l/proto/etcdpbb\006proto3"
   ;
 static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_etcd_5fmeta_2eproto_deps[3] = {
   &::descriptor_table_common_2eproto,
@@ -265,7 +267,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_etc
 static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_etcd_5fmeta_2eproto_once;
 static bool descriptor_table_etcd_5fmeta_2eproto_initialized = false;
 const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_etcd_5fmeta_2eproto = {
-  &descriptor_table_etcd_5fmeta_2eproto_initialized, descriptor_table_protodef_etcd_5fmeta_2eproto, "etcd_meta.proto", 1040,
+  &descriptor_table_etcd_5fmeta_2eproto_initialized, descriptor_table_protodef_etcd_5fmeta_2eproto, "etcd_meta.proto", 1062,
   &descriptor_table_etcd_5fmeta_2eproto_once, descriptor_table_etcd_5fmeta_2eproto_sccs, descriptor_table_etcd_5fmeta_2eproto_deps, 6, 3,
   schemas, file_default_instances, TableStruct_etcd_5fmeta_2eproto::offsets,
   file_level_metadata_etcd_5fmeta_2eproto, 6, file_level_enum_descriptors_etcd_5fmeta_2eproto, file_level_service_descriptors_etcd_5fmeta_2eproto,
@@ -1104,7 +1106,8 @@ CollectionMeta::CollectionMeta(const CollectionMeta& from)
   : ::PROTOBUF_NAMESPACE_ID::Message(),
       _internal_metadata_(nullptr),
       segmentids_(from.segmentids_),
-      partition_tags_(from.partition_tags_) {
+      partition_tags_(from.partition_tags_),
+      partitionids_(from.partitionids_) {
   _internal_metadata_.MergeFrom(from._internal_metadata_);
   if (from.has_schema()) {
     schema_ = new ::milvus::proto::schema::CollectionSchema(*from.schema_);
@@ -1150,6 +1153,7 @@ void CollectionMeta::Clear() {
 
   segmentids_.Clear();
   partition_tags_.Clear();
+  partitionids_.Clear();
   if (GetArenaNoVirtual() == nullptr && schema_ != nullptr) {
     delete schema_;
   }
@@ -1211,6 +1215,16 @@ const char* CollectionMeta::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE
           } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42);
         } else goto handle_unusual;
         continue;
+      // repeated int64 partitionIDs = 6;
+      case 6:
+        if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) {
+          ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(mutable_partitionids(), ptr, ctx);
+          CHK_(ptr);
+        } else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 48) {
+          add_partitionids(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr));
+          CHK_(ptr);
+        } else goto handle_unusual;
+        continue;
       default: {
       handle_unusual:
         if ((tag & 7) == 4 || tag == 0) {
@@ -1310,6 +1324,22 @@ bool CollectionMeta::MergePartialFromCodedStream(
         break;
       }
 
+      // repeated int64 partitionIDs = 6;
+      case 6: {
+        if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (50 & 0xFF)) {
+          DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPackedPrimitive<
+                   ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
+                 input, this->mutable_partitionids())));
+        } else if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (48 & 0xFF)) {
+          DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadRepeatedPrimitiveNoInline<
+                   ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
+                 1, 50u, input, this->mutable_partitionids())));
+        } else {
+          goto handle_unusual;
+        }
+        break;
+      }
+
       default: {
       handle_unusual:
         if (tag == 0) {
@@ -1374,6 +1404,17 @@ void CollectionMeta::SerializeWithCachedSizes(
       5, this->partition_tags(i), output);
   }
 
+  // repeated int64 partitionIDs = 6;
+  if (this->partitionids_size() > 0) {
+    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTag(6, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output);
+    output->WriteVarint32(_partitionids_cached_byte_size_.load(
+        std::memory_order_relaxed));
+  }
+  for (int i = 0, n = this->partitionids_size(); i < n; i++) {
+    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64NoTag(
+      this->partitionids(i), output);
+  }
+
   if (_internal_metadata_.have_unknown_fields()) {
     ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
         _internal_metadata_.unknown_fields(), output);
@@ -1427,6 +1468,19 @@ void CollectionMeta::SerializeWithCachedSizes(
       WriteStringToArray(5, this->partition_tags(i), target);
   }
 
+  // repeated int64 partitionIDs = 6;
+  if (this->partitionids_size() > 0) {
+    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTagToArray(
+      6,
+      ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED,
+      target);
+    target = ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream::WriteVarint32ToArray(
+        _partitionids_cached_byte_size_.load(std::memory_order_relaxed),
+         target);
+    target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
+      WriteInt64NoTagToArray(this->partitionids_, target);
+  }
+
   if (_internal_metadata_.have_unknown_fields()) {
     target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
         _internal_metadata_.unknown_fields(), target);
@@ -1471,6 +1525,21 @@ size_t CollectionMeta::ByteSizeLong() const {
       this->partition_tags(i));
   }
 
+  // repeated int64 partitionIDs = 6;
+  {
+    size_t data_size = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
+      Int64Size(this->partitionids_);
+    if (data_size > 0) {
+      total_size += 1 +
+        ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
+            static_cast<::PROTOBUF_NAMESPACE_ID::int32>(data_size));
+    }
+    int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(data_size);
+    _partitionids_cached_byte_size_.store(cached_size,
+                                    std::memory_order_relaxed);
+    total_size += data_size;
+  }
+
   // .milvus.proto.schema.CollectionSchema schema = 2;
   if (this->has_schema()) {
     total_size += 1 +
@@ -1521,6 +1590,7 @@ void CollectionMeta::MergeFrom(const CollectionMeta& from) {
 
   segmentids_.MergeFrom(from.segmentids_);
   partition_tags_.MergeFrom(from.partition_tags_);
+  partitionids_.MergeFrom(from.partitionids_);
   if (from.has_schema()) {
     mutable_schema()->::milvus::proto::schema::CollectionSchema::MergeFrom(from.schema());
   }
@@ -1555,6 +1625,7 @@ void CollectionMeta::InternalSwap(CollectionMeta* other) {
   _internal_metadata_.Swap(&other->_internal_metadata_);
   segmentids_.InternalSwap(&other->segmentids_);
   partition_tags_.InternalSwap(CastToBase(&other->partition_tags_));
+  partitionids_.InternalSwap(&other->partitionids_);
   swap(schema_, other->schema_);
   swap(id_, other->id_);
   swap(create_time_, other->create_time_);

internal/core/src/pb/etcd_meta.pb.h

@@ -540,6 +540,7 @@ class CollectionMeta :
   enum : int {
     kSegmentIDsFieldNumber = 4,
     kPartitionTagsFieldNumber = 5,
+    kPartitionIDsFieldNumber = 6,
     kSchemaFieldNumber = 2,
     kIDFieldNumber = 1,
     kCreateTimeFieldNumber = 3,
@@ -572,6 +573,17 @@ class CollectionMeta :
   const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& partition_tags() const;
   ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_partition_tags();
 
+  // repeated int64 partitionIDs = 6;
+  int partitionids_size() const;
+  void clear_partitionids();
+  ::PROTOBUF_NAMESPACE_ID::int64 partitionids(int index) const;
+  void set_partitionids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value);
+  void add_partitionids(::PROTOBUF_NAMESPACE_ID::int64 value);
+  const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >&
+      partitionids() const;
+  ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >*
+      mutable_partitionids();
+
   // .milvus.proto.schema.CollectionSchema schema = 2;
   bool has_schema() const;
   void clear_schema();
@@ -598,6 +610,8 @@ class CollectionMeta :
   ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > segmentids_;
   mutable std::atomic<int> _segmentids_cached_byte_size_;
   ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> partition_tags_;
+  ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > partitionids_;
+  mutable std::atomic<int> _partitionids_cached_byte_size_;
   ::milvus::proto::schema::CollectionSchema* schema_;
   ::PROTOBUF_NAMESPACE_ID::int64 id_;
   ::PROTOBUF_NAMESPACE_ID::uint64 create_time_;
@@ -1599,6 +1613,36 @@ CollectionMeta::mutable_partition_tags() {
   return &partition_tags_;
 }
 
+// repeated int64 partitionIDs = 6;
+inline int CollectionMeta::partitionids_size() const {
+  return partitionids_.size();
+}
+inline void CollectionMeta::clear_partitionids() {
+  partitionids_.Clear();
+}
+inline ::PROTOBUF_NAMESPACE_ID::int64 CollectionMeta::partitionids(int index) const {
+  // @@protoc_insertion_point(field_get:milvus.proto.etcd.CollectionMeta.partitionIDs)
+  return partitionids_.Get(index);
+}
+inline void CollectionMeta::set_partitionids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value) {
+  partitionids_.Set(index, value);
+  // @@protoc_insertion_point(field_set:milvus.proto.etcd.CollectionMeta.partitionIDs)
+}
+inline void CollectionMeta::add_partitionids(::PROTOBUF_NAMESPACE_ID::int64 value) {
+  partitionids_.Add(value);
+  // @@protoc_insertion_point(field_add:milvus.proto.etcd.CollectionMeta.partitionIDs)
+}
+inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >&
+CollectionMeta::partitionids() const {
+  // @@protoc_insertion_point(field_list:milvus.proto.etcd.CollectionMeta.partitionIDs)
+  return partitionids_;
+}
+inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >*
+CollectionMeta::mutable_partitionids() {
+  // @@protoc_insertion_point(field_mutable_list:milvus.proto.etcd.CollectionMeta.partitionIDs)
+  return &partitionids_;
+}
+
 // -------------------------------------------------------------------
 
 // FieldBinlogFiles

internal/core/src/query/Search.cpp

@@ -40,7 +40,7 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
             const float* query_data,
             int64_t num_queries,
             Timestamp timestamp,
-            const BitsetView& bitset,
+            std::optional<const BitmapSimple*> bitmaps_opt,
             QueryResult& results) {
     auto& schema = segment.get_schema();
     auto& indexing_record = segment.get_indexing_record();
@@ -79,21 +79,20 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
 
     // TODO: use sub_qr
     for (int chunk_id = 0; chunk_id < max_indexed_id; ++chunk_id) {
-        auto chunk_size = indexing_entry.get_chunk_size();
+        auto bitset = create_bitmap_view(bitmaps_opt, chunk_id);
         auto indexing = indexing_entry.get_vec_indexing(chunk_id);
-
-        auto sub_view = BitsetSubView(bitset, chunk_id * chunk_size, chunk_size);
-        auto sub_qr = SearchOnIndex(query_dataset, *indexing, search_conf, sub_view);
+        auto sub_qr = SearchOnIndex(query_dataset, *indexing, search_conf, bitset);
 
         // convert chunk uid to segment uid
         for (auto& x : sub_qr.mutable_labels()) {
             if (x != -1) {
-                x += chunk_id * chunk_size;
+                x += chunk_id * indexing_entry.get_chunk_size();
             }
         }
 
         final_qr.merge(sub_qr);
     }
+    using segcore::FloatVector;
     auto vec_ptr = record.get_entity<FloatVector>(vecfield_offset);
 
     // step 4: brute force search where small indexing is unavailable
@@ -102,14 +101,15 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
     auto max_chunk = upper_div(ins_barrier, vec_chunk_size);
 
     for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
+        auto bitmap_view = create_bitmap_view(bitmaps_opt, chunk_id);
+
         auto& chunk = vec_ptr->get_chunk(chunk_id);
 
         auto element_begin = chunk_id * vec_chunk_size;
         auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_chunk_size);
         auto chunk_size = element_end - element_begin;
 
-        auto sub_view = BitsetSubView(bitset, element_begin, chunk_size);
-        auto sub_qr = FloatSearchBruteForce(query_dataset, chunk.data(), chunk_size, sub_view);
+        auto sub_qr = FloatSearchBruteForce(query_dataset, chunk.data(), chunk_size, bitmap_view);
 
         // convert chunk uid to segment uid
         for (auto& x : sub_qr.mutable_labels()) {
@@ -134,7 +134,7 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
              const uint8_t* query_data,
              int64_t num_queries,
              Timestamp timestamp,
-             const faiss::BitsetView& bitset,
+             std::optional<const BitmapSimple*> bitmaps_opt,
              QueryResult& results) {
     auto& schema = segment.get_schema();
     auto& indexing_record = segment.get_indexing_record();
@@ -161,8 +161,11 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
     auto total_count = topK * num_queries;
 
     // step 3: small indexing search
+    // TODO: this is too intrusive
+    // TODO: use QuerySubResult instead
     query::dataset::BinaryQueryDataset query_dataset{metric_type, num_queries, topK, dim, query_data};
 
+    using segcore::BinaryVector;
     auto vec_ptr = record.get_entity<BinaryVector>(vecfield_offset);
 
     auto max_indexed_id = 0;
@@ -177,8 +180,8 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
         auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_chunk_size);
         auto nsize = element_end - element_begin;
 
-        auto sub_view = BitsetSubView(bitset, element_begin, nsize);
-        auto sub_result = BinarySearchBruteForce(query_dataset, chunk.data(), nsize, sub_view);
+        auto bitmap_view = create_bitmap_view(bitmaps_opt, chunk_id);
+        auto sub_result = BinarySearchBruteForce(query_dataset, chunk.data(), nsize, bitmap_view);
 
         // convert chunk uid to segment uid
         for (auto& x : sub_result.mutable_labels()) {

internal/core/src/query/Search.h

@@ -28,7 +28,7 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
             const float* query_data,
             int64_t num_queries,
             Timestamp timestamp,
-            const faiss::BitsetView& bitset,
+            std::optional<const BitmapSimple*> bitmap_opt,
             QueryResult& results);
 
 Status
@@ -37,6 +37,6 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
              const uint8_t* query_data,
              int64_t num_queries,
              Timestamp timestamp,
-             const faiss::BitsetView& bitset,
+             std::optional<const BitmapSimple*> bitmaps_opt,
              QueryResult& results);
 }  // namespace milvus::query

internal/core/src/query/SearchOnSealed.cpp

@@ -20,9 +20,8 @@
 
 namespace milvus::query {
 
-// negate bitset, and merge them into one
 aligned_vector<uint8_t>
-AssembleNegBitmap(const BitmapSimple& bitmap_simple) {
+AssembleBitmap(const BitmapSimple& bitmap_simple) {
     int64_t N = 0;
 
     for (auto& bitmap : bitmap_simple) {
@@ -53,7 +52,7 @@ SearchOnSealed(const Schema& schema,
                const void* query_data,
                int64_t num_queries,
                Timestamp timestamp,
-               const faiss::BitsetView& bitset,
+               std::optional<const BitmapSimple*> bitmaps_opt,
                QueryResult& result) {
     auto topK = query_info.topK_;
 
@@ -74,7 +73,12 @@ SearchOnSealed(const Schema& schema,
         auto conf = query_info.search_params_;
         conf[milvus::knowhere::meta::TOPK] = query_info.topK_;
         conf[milvus::knowhere::Metric::TYPE] = MetricTypeToName(indexing_entry->metric_type_);
-        return indexing_entry->indexing_->Query(ds, conf, bitset);
+        if (bitmaps_opt.has_value()) {
+            auto bitmap = AssembleBitmap(*bitmaps_opt.value());
+            return indexing_entry->indexing_->Query(ds, conf, faiss::BitsetView(bitmap.data(), num_queries));
+        } else {
+            return indexing_entry->indexing_->Query(ds, conf, nullptr);
+        }
     }();
 
     auto ids = final->Get<idx_t*>(knowhere::meta::IDS);