Remove not using code in DataNode (#5476)

Signed-off-by: yangxuan <xuan.yang@zilliz.com>

internal/datanode/binlog_meta.go

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-// GOOSE TODO remove this
-
-package datanode
-
-import (
-	"path"
-	"strconv"
-
-	"github.com/golang/protobuf/proto"
-	"github.com/milvus-io/milvus/internal/kv"
-	"github.com/milvus-io/milvus/internal/proto/datapb"
-)
-
-// binlogMeta persists binlog paths into etcd.
-// ddl binlog etcd meta key:
-//   ${prefix}/${collectionID}/${idx}
-// segment binlog etcd meta key:
-//   ${prefix}/${segmentID}/${fieldID}/${idx}
-type binlogMeta struct {
-	client      kv.TxnKV // etcd kv
-	idAllocator allocatorInterface
-}
-
-func NewBinlogMeta(kv kv.TxnKV, idAllocator allocatorInterface) (*binlogMeta, error) {
-	mt := &binlogMeta{
-		client:      kv,
-		idAllocator: idAllocator,
-	}
-	return mt, nil
-}
-
-func (bm *binlogMeta) allocID() (key UniqueID, err error) {
-	return bm.idAllocator.allocID()
-}
-
-// genKey gives a valid key string for lists of UniqueIDs:
-//  if alloc is true, the returned keys will have a generated-unique ID at the end.
-//  if alloc is false, the returned keys will only consist of provided ids.
-func (bm *binlogMeta) genKey(alloc bool, ids ...UniqueID) (key string, err error) {
-	if alloc {
-		idx, err := bm.idAllocator.allocID()
-		if err != nil {
-			return "", err
-		}
-		ids = append(ids, idx)
-	}
-
-	idStr := make([]string, len(ids))
-	for _, id := range ids {
-		idStr = append(idStr, strconv.FormatInt(id, 10))
-	}
-
-	key = path.Join(idStr...)
-	return
-}
-
-// SaveSegmentBinlogMetaTxn stores all fields' binlog paths of a segment in a transaction.
-// segment binlog etcd meta key:
-//   ${prefix}/${segmentID}/${fieldID}/${idx}
-func (bm *binlogMeta) SaveSegmentBinlogMetaTxn(segmentID UniqueID, field2Path map[UniqueID][]string) error {
-
-	etcdKey2binlogPath := make(map[string]string)
-	for fieldID, paths := range field2Path {
-		for _, p := range paths {
-			key, err := bm.genKey(true, segmentID, fieldID)
-			if err != nil {
-				return err
-			}
-
-			binlogPath := proto.MarshalTextString(&datapb.SegmentFieldBinlogMeta{
-				FieldID:    fieldID,
-				BinlogPath: p,
-			})
-			etcdKey2binlogPath[path.Join(Params.SegFlushMetaSubPath, key)] = binlogPath
-		}
-	}
-	return bm.client.MultiSave(etcdKey2binlogPath)
-}
-
-func (bm *binlogMeta) getFieldBinlogMeta(segmentID UniqueID,
-	fieldID UniqueID) (metas []*datapb.SegmentFieldBinlogMeta, err error) {
-
-	prefix, err := bm.genKey(false, segmentID, fieldID)
-	if err != nil {
-		return nil, err
-	}
-
-	_, vs, err := bm.client.LoadWithPrefix(path.Join(Params.SegFlushMetaSubPath, prefix))
-	if err != nil {
-		return nil, err
-	}
-
-	for _, blob := range vs {
-		m := &datapb.SegmentFieldBinlogMeta{}
-		if err = proto.UnmarshalText(blob, m); err != nil {
-			return nil, err
-		}
-
-		metas = append(metas, m)
-	}
-
-	return
-}
-
-func (bm *binlogMeta) getSegmentBinlogMeta(segmentID UniqueID) (metas []*datapb.SegmentFieldBinlogMeta, err error) {
-
-	prefix, err := bm.genKey(false, segmentID)
-	if err != nil {
-		return nil, err
-	}
-
-	_, vs, err := bm.client.LoadWithPrefix(path.Join(Params.SegFlushMetaSubPath, prefix))
-	if err != nil {
-		return nil, err
-	}
-
-	for _, blob := range vs {
-		m := &datapb.SegmentFieldBinlogMeta{}
-		if err = proto.UnmarshalText(blob, m); err != nil {
-			return nil, err
-		}
-
-		metas = append(metas, m)
-	}
-	return
-}
-
-// SaveDDLBinlogMetaTxn stores timestamp and ddl binlog path pair into etcd in a transaction.
-// ddl binlog meta key:
-//   ${prefix}/${collectionID}/${idx}
-func (bm *binlogMeta) SaveDDLBinlogMetaTxn(collID UniqueID, ddlBinlogMeta *datapb.DDLBinlogMeta) error {
-
-	uniqueKey, err := bm.genKey(true, collID)
-	if err != nil {
-		return err
-	}
-	binlogPathPair := proto.MarshalTextString(ddlBinlogMeta)
-
-	return bm.client.Save(path.Join(Params.DDLFlushMetaSubPath, uniqueKey), binlogPathPair)
-}
-
-func (bm *binlogMeta) getDDLBinlogMete(collID UniqueID) (metas []*datapb.DDLBinlogMeta, err error) {
-	prefix, err := bm.genKey(false, collID)
-	if err != nil {
-		return nil, err
-	}
-
-	_, vs, err := bm.client.LoadWithPrefix(path.Join(Params.DDLFlushMetaSubPath, prefix))
-	if err != nil {
-		return nil, err
-	}
-
-	for _, blob := range vs {
-		m := &datapb.DDLBinlogMeta{}
-		if err = proto.UnmarshalText(blob, m); err != nil {
-			return nil, err
-		}
-
-		metas = append(metas, m)
-	}
-	return
-}

internal/datanode/binlog_meta_test.go

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-
-package datanode
-
-import (
-	"strings"
-	"testing"
-
-	memkv "github.com/milvus-io/milvus/internal/kv/mem"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMetaTable_Basic(t *testing.T) {
-
-	kvMock := memkv.NewMemoryKV()
-	allocMock := NewAllocatorFactory(22222)
-	meta, err := NewBinlogMeta(kvMock, allocMock)
-	require.NoError(t, err)
-	defer meta.client.Close()
-
-	t.Run("TestBasic_genKey", func(t *testing.T) {
-		// 0/1
-		alloc := true
-		k, err := meta.genKey(alloc, 0)
-		assert.NoError(t, err)
-		assert.True(t, strings.HasPrefix(k, "0/"))
-
-		// rand int64
-		_, err = meta.genKey(alloc)
-		assert.NoError(t, err)
-
-		// 1/2/3/1
-		k, err = meta.genKey(alloc, 1, 2, 3)
-		assert.NoError(t, err)
-		assert.True(t, strings.HasPrefix(k, "1/2/3/"))
-
-		// 0
-		alloc = false
-		k, err = meta.genKey(alloc, 0)
-		assert.NoError(t, err)
-		assert.Equal(t, "0", k)
-
-		// ""
-		k, err = meta.genKey(alloc)
-		assert.NoError(t, err)
-		assert.Equal(t, "", k)
-
-		// 1/2/3
-		k, err = meta.genKey(alloc, 1, 2, 3)
-		assert.NoError(t, err)
-		assert.Equal(t, "1/2/3", k)
-	})
-
-	t.Run("TestBasic_SaveSegmentBinlogMetaTxn", func(t *testing.T) {
-		segID := UniqueID(999999)
-		fieldID2Path := map[UniqueID][]string{
-			100: {"a"},
-			200: {"b"},
-			300: {"c"},
-		}
-
-		err := meta.SaveSegmentBinlogMetaTxn(segID, fieldID2Path)
-		assert.NoError(t, err)
-
-		metas, err := meta.getFieldBinlogMeta(segID, 100)
-		assert.NoError(t, err)
-		assert.Equal(t, 1, len(metas))
-		assert.Equal(t, "a", metas[0].GetBinlogPath())
-
-		metas, err = meta.getFieldBinlogMeta(segID, 200)
-		assert.NoError(t, err)
-		assert.Equal(t, 1, len(metas))
-		assert.Equal(t, "b", metas[0].GetBinlogPath())
-
-		metas, err = meta.getFieldBinlogMeta(segID, 300)
-		assert.NoError(t, err)
-		assert.Equal(t, 1, len(metas))
-		assert.Equal(t, "c", metas[0].GetBinlogPath())
-
-		fieldID2Path2 := map[UniqueID][]string{
-			100: {"aa"},
-			200: {"bb"},
-			300: {"cc"},
-		}
-
-		err = meta.SaveSegmentBinlogMetaTxn(segID, fieldID2Path2)
-		assert.NoError(t, err)
-
-		metas, err = meta.getSegmentBinlogMeta(segID)
-		assert.NoError(t, err)
-		assert.Equal(t, 6, len(metas))
-
-		paths := make([]string, 0, 6)
-		for _, meta := range metas {
-			paths = append(paths, meta.GetBinlogPath())
-		}
-
-		assert.ElementsMatch(t, []string{"a", "b", "c", "aa", "bb", "cc"}, paths)
-	})
-}

internal/datanode/data_node.go

@@ -185,7 +185,7 @@ func (node *DataNode) Init() error {
 	return nil
 }
 
-// NewDataSyncService adds a new dataSyncService to DataNode
+// NewDataSyncService adds a new dataSyncService for new dmlVchannel and starts dataSyncService.
 func (node *DataNode) NewDataSyncService(vchanPair *datapb.VchannelPair) error {
 	if _, ok := node.vchan2SyncService[vchanPair.GetDmlVchannelName()]; ok {
 		return nil
@@ -196,6 +196,7 @@ func (node *DataNode) NewDataSyncService(vchanPair *datapb.VchannelPair) error {
 
 	flushChan := make(chan *flushMsg, 100)
 	dataSyncService := newDataSyncService(node.ctx, flushChan, replica, alloc, node.msFactory, vchanPair)
+	// TODO metaService using timestamp in DescribeCollection
 	metaService := newMetaService(node.ctx, replica, node.masterService)
 	node.vchan2SyncService[vchanPair.GetDmlVchannelName()] = dataSyncService
 	node.vchan2FlushCh[vchanPair.GetDmlVchannelName()] = flushChan
@@ -206,18 +207,18 @@ func (node *DataNode) NewDataSyncService(vchanPair *datapb.VchannelPair) error {
 	return nil
 }
 
-// Start will update state to HEALTHY
+// Start will update DataNode state to HEALTHY
 func (node *DataNode) Start() error {
 	node.UpdateStateCode(internalpb.StateCode_Healthy)
 	return nil
 }
 
-// UpdateStateCode update datanode's state code
+// UpdateStateCode updates datanode's state code
 func (node *DataNode) UpdateStateCode(code internalpb.StateCode) {
 	node.State.Store(code)
 }
 
-// WatchDmChannels set insert channel names data node subscribs to.
+// WatchDmChannels create a new dataSyncService for every unique dmlVchannel name, ignore if dmlVchannel existed.
 func (node *DataNode) WatchDmChannels(ctx context.Context, in *datapb.WatchDmChannelsRequest) (*commonpb.Status, error) {
 	status := &commonpb.Status{
 		ErrorCode: commonpb.ErrorCode_UnexpectedError,
@@ -225,7 +226,7 @@ func (node *DataNode) WatchDmChannels(ctx context.Context, in *datapb.WatchDmCha
 
 	switch {
 	case node.State.Load() != internalpb.StateCode_Healthy:
-		status.Reason = fmt.Sprintf("DataNode %d not initializing!", node.NodeID)
+		status.Reason = fmt.Sprintf("DataNode %d not healthy, please re-send message", node.NodeID)
 		return status, errors.New(status.GetReason())
 
 	case len(in.GetVchannels()) == 0:
@@ -266,6 +267,8 @@ func (node *DataNode) getChannelName(segID UniqueID) string {
 }
 
 // FlushSegments packs flush messages into flowgraph through flushChan.
+//   If DataNode receives a valid segment to flush, new flush message for the segment should be ignored.
+//   So if receiving calls to flush segment A, DataNode should guarantee the segment to be flushed.
 func (node *DataNode) FlushSegments(ctx context.Context, req *datapb.FlushSegmentsRequest) (*commonpb.Status, error) {
 	log.Debug("FlushSegments ...", zap.Int("num", len(req.SegmentIDs)))
 	status := &commonpb.Status{
@@ -281,6 +284,7 @@ func (node *DataNode) FlushSegments(ctx context.Context, req *datapb.FlushSegmen
 		}
 		flushCh, ok := node.vchan2FlushCh[chanName]
 		if !ok {
+			// TODO restart DataNode or reshape vchan2FlushCh and vchan2SyncService
 			status.Reason = "DataNode abnormal!"
 			return status, errors.New(status.GetReason())
 		}

internal/datanode/data_node_test.go

@@ -202,6 +202,6 @@ func TestDataNode(t *testing.T) {
 		assert.NoError(t, err)
 	})
 
-	// <-node.ctx.Done()
+	<-node.ctx.Done()
 	node.Stop()
 }

internal/datanode/data_sync_service.go

@@ -13,15 +13,11 @@ package datanode
 
 import (
 	"context"
-	"time"
 
-	etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
 	"github.com/milvus-io/milvus/internal/log"
 	"github.com/milvus-io/milvus/internal/msgstream"
 	"github.com/milvus-io/milvus/internal/proto/datapb"
 	"github.com/milvus-io/milvus/internal/util/flowgraph"
-	"github.com/milvus-io/milvus/internal/util/retry"
-	"go.etcd.io/etcd/clientv3"
 
 	"go.uber.org/zap"
 )
@@ -74,24 +70,6 @@ func (dsService *dataSyncService) close() {
 
 func (dsService *dataSyncService) initNodes(vchanPair *datapb.VchannelPair) {
 	// TODO: add delete pipeline support
-	var kvClient *clientv3.Client
-	var err error
-	connectEtcdFn := func() error {
-		kvClient, err = clientv3.New(clientv3.Config{Endpoints: []string{Params.EtcdAddress}})
-		if err != nil {
-			return err
-		}
-		return nil
-	}
-	err = retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
-	if err != nil {
-		panic(err)
-	}
-
-	etcdKV := etcdkv.NewEtcdKV(kvClient, Params.MetaRootPath)
-	// New binlogMeta
-	mt, _ := NewBinlogMeta(etcdKV, dsService.idAllocator)
-
 	dsService.fg = flowgraph.NewTimeTickedFlowGraph(dsService.ctx)
 
 	m := map[string]interface{}{
@@ -100,7 +78,7 @@ func (dsService *dataSyncService) initNodes(vchanPair *datapb.VchannelPair) {
 		"PulsarBufSize":  1024,
 	}
 
-	err = dsService.msFactory.SetParams(m)
+	err := dsService.msFactory.SetParams(m)
 	if err != nil {
 		panic(err)
 	}
@@ -109,8 +87,8 @@ func (dsService *dataSyncService) initNodes(vchanPair *datapb.VchannelPair) {
 	var ddStreamNode Node = newDDInputNode(dsService.ctx, dsService.msFactory, vchanPair.GetDdlVchannelName(), vchanPair.GetDdlPosition())
 
 	var filterDmNode Node = newFilteredDmNode()
-	var ddNode Node = newDDNode(dsService.ctx, mt, dsService.flushChan, dsService.replica, dsService.idAllocator)
-	var insertBufferNode Node = newInsertBufferNode(dsService.ctx, mt, dsService.replica, dsService.msFactory, dsService.idAllocator)
+	var ddNode Node = newDDNode(dsService.ctx, dsService.flushChan, dsService.replica, dsService.idAllocator)
+	var insertBufferNode Node = newInsertBufferNode(dsService.ctx, dsService.replica, dsService.msFactory, dsService.idAllocator)
 	var gcNode Node = newGCNode(dsService.replica)
 
 	dsService.fg.AddNode(dmStreamNode)

internal/datanode/flow_graph_dd_node.go

@@ -44,9 +44,8 @@ type ddNode struct {
 	inFlushCh   <-chan *flushMsg
 	idAllocator allocatorInterface
 
-	kv         kv.BaseKV
-	replica    Replica
-	binlogMeta *binlogMeta
+	kv      kv.BaseKV
+	replica Replica
 }
 
 type ddData struct {
@@ -109,7 +108,6 @@ func (ddNode *ddNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
 			timestampMin: msMsg.TimestampMin(),
 			timestampMax: msMsg.TimestampMax(),
 		},
-		// flushMessages: make([]*flushMsg, 0),
 		gcRecord: &gcRecord{
 			collections: make([]UniqueID, 0),
 		},
@@ -158,7 +156,6 @@ func (ddNode *ddNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
 			go flush(collID, ddNode.flushMap, ddNode.kv, ddNode.idAllocator, binlogMetaCh)
 			go ddNode.flushComplete(binlogMetaCh, collID, fmsg.ddlFlushedCh)
 		} else {
-			// GOOSE TODO newest position
 			fmsg.ddlFlushedCh <- make([]*datapb.DDLBinlogMeta, 0)
 		}
 
@@ -182,14 +179,7 @@ func (ddNode *ddNode) flushComplete(binlogMetaCh <-chan *datapb.DDLBinlogMeta, c
 		ddlFlushedCh <- nil
 		return
 	}
-	//
-	// log.Debug(".. Saving ddl binlog meta ...")
-	// err := ddNode.binlogMeta.SaveDDLBinlogMetaTxn(collID, binlogMeta)
-	// if err != nil {
-	//     log.Error("Save binlog meta to etcd Wrong", zap.Error(err))
-	// }
 
-	// TODO  remove above
 	ddlFlushedCh <- []*datapb.DDLBinlogMeta{binlogMeta}
 }
 
@@ -197,7 +187,6 @@ func (ddNode *ddNode) flushComplete(binlogMetaCh <-chan *datapb.DDLBinlogMeta, c
 flush will
     generate binlogs for all buffer data in ddNode,
     store the generated binlogs to minIO/S3,
-    store the keys(paths to minIO/s3) of the binlogs to etcd. todo remove
 
 The keys of the binlogs are generated as below:
     ${tenant}/data_definition_log/${collection_id}/ts/${log_idx}
@@ -449,7 +438,7 @@ func (ddNode *ddNode) dropPartition(msg *msgstream.DropPartitionMsg) {
 		append(ddNode.ddBuffer.ddData[collectionID].eventTypes, storage.DropPartitionEventType)
 }
 
-func newDDNode(ctx context.Context, binlogMeta *binlogMeta, inFlushCh <-chan *flushMsg,
+func newDDNode(ctx context.Context, inFlushCh <-chan *flushMsg,
 	replica Replica, idAllocator allocatorInterface) *ddNode {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
@@ -488,7 +477,6 @@ func newDDNode(ctx context.Context, binlogMeta *binlogMeta, inFlushCh <-chan *fl
 		idAllocator: idAllocator,
 		kv:          minioKV,
 		replica:     replica,
-		binlogMeta:  binlogMeta,
 		flushMap:    &sync.Map{},
 	}
 }

internal/datanode/flow_graph_dd_node_test.go

@@ -44,14 +44,8 @@ func TestFlowGraphDDNode_Operate(t *testing.T) {
 	inFlushCh := make(chan *flushMsg, 10)
 	defer close(inFlushCh)
 
-	// testPath := "/test/datanode/root/meta"
-	// err := clearEtcd(testPath)
-	// require.NoError(t, err)
-	// Params.MetaRootPath = testPath
-
-	// Params.FlushDdBufferSize = 4
 	replica := newReplica()
-	ddNode := newDDNode(ctx, newBinlogMeta(), inFlushCh, replica, NewAllocatorFactory())
+	ddNode := newDDNode(ctx, inFlushCh, replica, NewAllocatorFactory())
 
 	collID := UniqueID(0)
 	collName := "col-test-0"

internal/datanode/flow_graph_insert_buffer_node.go

@@ -52,7 +52,6 @@ type insertBufferNode struct {
 	BaseNode
 	insertBuffer *insertBuffer
 	replica      Replica
-	flushMeta    *binlogMeta // GOOSE TODO remove
 	idAllocator  allocatorInterface
 	flushMap     sync.Map
 
@@ -525,8 +524,6 @@ func (ibNode *insertBufferNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
 
 	// iMsg is Flush() msg from dataservice
 	//   1. insertBuffer(not empty) -> binLogs -> minIO/S3
-	// for _, msg := range iMsg.flushMessages {
-	// for _, currentSegID := range msg.segmentIDs {
 	if iMsg.flushMessage != nil && ibNode.replica.hasSegment(iMsg.flushMessage.segmentID) {
 		currentSegID := iMsg.flushMessage.segmentID
 		log.Debug(". Receiving flush message", zap.Int64("segmentID", currentSegID))
@@ -697,10 +694,6 @@ func (ibNode *insertBufferNode) completeFlush(segID UniqueID, wait <-chan map[Un
 		return
 	}
 
-	// dmlFlushedCh <- true
-
-	// TODO Call DataService RPC SaveBinlogPaths
-	// TODO GetBufferedAutoFlushBinlogPaths
 	ibNode.replica.bufferAutoFlushBinlogPaths(segID, field2Path)
 	bufferField2Paths, err := ibNode.replica.getBufferPaths(segID)
 	if err != nil {
@@ -708,15 +701,6 @@ func (ibNode *insertBufferNode) completeFlush(segID UniqueID, wait <-chan map[Un
 		dmlFlushedCh <- nil
 	}
 
-	// GOOSE TODO remove the below
-	// log.Debug(".. Saving binlog paths to etcd ..", zap.Int("number of fields", len(field2Path)))
-	// err = ibNode.flushMeta.SaveSegmentBinlogMetaTxn(segID, bufferField2Paths)
-	// if err != nil {
-	//     log.Error("Flush failed ... cannot save binlog paths ..", zap.Error(err))
-	//     dmlFlushedCh <- nil
-	//     return
-	// }
-
 	binlogPaths := make([]*datapb.ID2PathList, 0, len(bufferField2Paths))
 	for k, paths := range bufferField2Paths {
 
@@ -732,28 +716,6 @@ func (ibNode *insertBufferNode) completeFlush(segID UniqueID, wait <-chan map[Un
 	ibNode.replica.setIsFlushed(segID)
 	ibNode.updateSegStatistics([]UniqueID{segID})
 
-	// msgPack := msgstream.MsgPack{}
-	// completeFlushMsg := internalpb.SegmentFlushCompletedMsg{
-	//     Base: &commonpb.MsgBase{
-	//         MsgType:   commonpb.MsgType_SegmentFlushDone,
-	//         MsgID:     0, // GOOSE TODO
-	//         Timestamp: 0, // GOOSE TODO
-	//         SourceID:  Params.NodeID,
-	//     },
-	//     SegmentID: segID,
-	// }
-	// var msg msgstream.TsMsg = &msgstream.FlushCompletedMsg{
-	//     BaseMsg: msgstream.BaseMsg{
-	//         HashValues: []uint32{0},
-	//     },
-	//     SegmentFlushCompletedMsg: completeFlushMsg,
-	// }
-	//
-	// msgPack.Msgs = append(msgPack.Msgs, msg)
-	// err = ibNode.completeFlushStream.Produce(&msgPack)
-	// if err != nil {
-	//     log.Error(".. Produce complete flush msg failed ..", zap.Error(err))
-	// }
 }
 
 func (ibNode *insertBufferNode) writeHardTimeTick(ts Timestamp) error {
@@ -846,8 +808,7 @@ func (ibNode *insertBufferNode) getCollectionandPartitionIDbySegID(segmentID Uni
 	return
 }
 
-func newInsertBufferNode(ctx context.Context, flushMeta *binlogMeta,
-	replica Replica, factory msgstream.Factory, idAllocator allocatorInterface) *insertBufferNode {
+func newInsertBufferNode(ctx context.Context, replica Replica, factory msgstream.Factory, idAllocator allocatorInterface) *insertBufferNode {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
 
@@ -905,8 +866,8 @@ func newInsertBufferNode(ctx context.Context, flushMeta *binlogMeta,
 		segmentStatisticsStream: segStatisticsMsgStream,
 		completeFlushStream:     completeFlushStream,
 		replica:                 replica,
-		flushMeta:               flushMeta,
-		flushMap:                sync.Map{},
-		idAllocator:             idAllocator,
+		// flushMeta:               flushMeta,
+		flushMap:    sync.Map{},
+		idAllocator: idAllocator,
 	}
 }

internal/datanode/flow_graph_insert_buffer_node_test.go

@@ -72,7 +72,7 @@ func TestFlowGraphInsertBufferNode_Operate(t *testing.T) {
 	err = msFactory.SetParams(m)
 	assert.Nil(t, err)
 
-	iBNode := newInsertBufferNode(ctx, newBinlogMeta(), replica, msFactory, NewAllocatorFactory())
+	iBNode := newInsertBufferNode(ctx, replica, msFactory, NewAllocatorFactory())
 
 	ddlFlushedCh := make(chan []*datapb.DDLBinlogMeta)
 	dmlFlushedCh := make(chan []*datapb.ID2PathList)
@@ -103,7 +103,6 @@ func genInsertMsg(ddlFlushedCh chan<- []*datapb.DDLBinlogMeta, dmlFlushedCh chan
 
 	var iMsg = &insertMsg{
 		insertMessages: make([]*msgstream.InsertMsg, 0),
-		// flushMessages:  make([]*flushMsg, 0),
 		timeRange: TimeRange{
 			timestampMin: timeRange.timestampMin,
 			timestampMax: timeRange.timestampMax,
@@ -128,7 +127,7 @@ func genInsertMsg(ddlFlushedCh chan<- []*datapb.DDLBinlogMeta, dmlFlushedCh chan
 
 }
 
-func TestFlushSegmentTxn(t *testing.T) {
+func TestFlushSegment(t *testing.T) {
 	idAllocMock := NewAllocatorFactory(1)
 	mockMinIO := memkv.NewMemoryKV()
 
@@ -140,7 +139,6 @@ func TestFlushSegmentTxn(t *testing.T) {
 
 	collMeta := genCollectionMeta(collectionID, "test_flush_segment_txn")
 	flushMap := sync.Map{}
-	flushMeta := newBinlogMeta()
 
 	finishCh := make(chan map[UniqueID]string)
 
@@ -175,7 +173,7 @@ func TestFlushSegmentTxn(t *testing.T) {
 		finishCh,
 		idAllocMock)
 
-	k, _ := flushMeta.genKey(false, collectionID, partitionID, segmentID, 0)
+	k, _ := idAllocMock.genKey(false, collectionID, partitionID, segmentID, 0)
 	key := path.Join(Params.StatsBinlogRootPath, k)
 	_, values, _ := mockMinIO.LoadWithPrefix(key)
 	assert.Equal(t, len(values), 1)

internal/datanode/flow_graph_message.go

@@ -69,8 +69,6 @@ type flushMsg struct {
 	timestamp    Timestamp
 	segmentID    UniqueID
 	collectionID UniqueID
-	// ddlFlushedCh chan<- bool
-	// dmlFlushedCh chan<- bool
 	ddlFlushedCh chan<- []*datapb.DDLBinlogMeta
 	dmlFlushedCh chan<- []*datapb.ID2PathList
 }

internal/datanode/flow_graph_msg_stream_input_node.go

@@ -25,7 +25,6 @@ func newDmInputNode(ctx context.Context, factory msgstream.Factory, vchannelName
 
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
-	// consumeChannels := Params.InsertChannelNames
 	consumeSubName := Params.MsgChannelSubName
 
 	insertStream, _ := factory.NewTtMsgStream(ctx)

internal/datanode/mock_test.go

@@ -26,7 +26,6 @@ import (
 	"go.uber.org/zap"
 
 	etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
-	memkv "github.com/milvus-io/milvus/internal/kv/mem"
 	"github.com/milvus-io/milvus/internal/log"
 	"github.com/milvus-io/milvus/internal/msgstream"
 	"github.com/milvus-io/milvus/internal/types"
@@ -136,13 +135,6 @@ func refreshChannelNames() {
 	Params.InsertChannelNames = makeNewChannelNames(Params.InsertChannelNames, suffix)
 }
 
-func newBinlogMeta() *binlogMeta {
-	kvMock := memkv.NewMemoryKV()
-	idAllocMock := NewAllocatorFactory(1)
-	mt, _ := NewBinlogMeta(kvMock, idAllocMock)
-	return mt
-}
-
 func clearEtcd(rootPath string) error {
 	etcdAddr := Params.EtcdAddress
 	log.Info("etcd tests address", zap.String("address", etcdAddr))

internal/datanode/param_table.go

@@ -59,10 +59,8 @@ type ParamTable struct {
 	MsgChannelSubName string
 
 	// --- ETCD ---
-	EtcdAddress         string
-	MetaRootPath        string
-	SegFlushMetaSubPath string // GOOSE TODO remove
-	DDLFlushMetaSubPath string // GOOSE TODO remove
+	EtcdAddress  string
+	MetaRootPath string
 
 	// --- MinIO ---
 	MinioAddress         string
@@ -109,8 +107,6 @@ func (p *ParamTable) Init() {
 		// --- ETCD ---
 		p.initEtcdAddress()
 		p.initMetaRootPath()
-		p.initSegFlushMetaSubPath() // GOOSE TODO remove
-		p.initDDLFlushMetaSubPath() // GOOSE TODO remove
 
 		// --- MinIO ---
 		p.initMinioAddress()
@@ -224,24 +220,6 @@ func (p *ParamTable) initMetaRootPath() {
 	p.MetaRootPath = path.Join(rootPath, subPath)
 }
 
-// GOOSE TODO remove
-func (p *ParamTable) initSegFlushMetaSubPath() {
-	subPath, err := p.Load("etcd.segFlushMetaSubPath")
-	if err != nil {
-		panic(err)
-	}
-	p.SegFlushMetaSubPath = subPath
-}
-
-// GOOSE TODO remove
-func (p *ParamTable) initDDLFlushMetaSubPath() {
-	subPath, err := p.Load("etcd.ddlFlushMetaSubPath")
-	if err != nil {
-		panic(err)
-	}
-	p.DDLFlushMetaSubPath = subPath
-}
-
 func (p *ParamTable) initMinioAddress() {
 	endpoint, err := p.Load("_MinioAddress")
 	if err != nil {

internal/datanode/param_table_test.go

@@ -90,16 +90,6 @@ func TestParamTable_DataNode(t *testing.T) {
 		log.Println("MetaRootPath:", path)
 	})
 
-	t.Run("Test SegFlushMetaSubPath", func(t *testing.T) {
-		path := Params.SegFlushMetaSubPath
-		log.Println("SegFlushMetaSubPath:", path)
-	})
-
-	t.Run("Test DDLFlushMetaSubPath", func(t *testing.T) {
-		path := Params.DDLFlushMetaSubPath
-		log.Println("DDLFlushMetaSubPath:", path)
-	})
-
 	t.Run("Test minioAccessKeyID", func(t *testing.T) {
 		id := Params.MinioAccessKeyID
 		log.Println("MinioAccessKeyID:", id)