From ff8ebd5f42c3d2177d45b338fc5ed3eb299d27c7 Mon Sep 17 00:00:00 2001
From: sunby <bingyi.sun@zilliz.com>
Date: Mon, 29 Mar 2021 15:14:51 +0800
Subject: [PATCH] Fix bug of estimating record size
Signed-off-by: sunby <bingyi.sun@zilliz.com>
---
internal/allocator/allocator.go | 13 ++++---------
internal/allocator/id.go | 7 +++----
internal/allocator/timestamp.go | 8 +++-----
internal/proxynode/segment.go | 7 +++----
internal/util/typeutil/schema.go | 16 ++++++++++++++--
5 files changed, 27 insertions(+), 24 deletions(-)
diff --git a/internal/allocator/allocator.go b/internal/allocator/allocator.go
index 6c827a044..ff0f23afd 100644
--- a/internal/allocator/allocator.go
+++ b/internal/allocator/allocator.go
@@ -13,9 +13,8 @@ const (
)
type Request interface {
- Wait()
+ Wait() error
Notify(error)
- IsValid() bool
}
type BaseRequest struct {
@@ -23,13 +22,9 @@ type BaseRequest struct {
Valid bool
}
-func (req *BaseRequest) Wait() {
+func (req *BaseRequest) Wait() error {
err := <-req.Done
- req.Valid = err == nil
-}
-
-func (req *BaseRequest) IsValid() bool {
- return req.Valid
+ return err
}
func (req *BaseRequest) Notify(err error) {
@@ -252,5 +247,5 @@ func (ta *Allocator) Close() {
func (ta *Allocator) CleanCache() {
req := &SyncRequest{BaseRequest: BaseRequest{Done: make(chan error), Valid: false}}
ta.ForceSyncChan <- req
- req.Wait()
+ _ = req.Wait()
}
diff --git a/internal/allocator/id.go b/internal/allocator/id.go
index 6588a5f56..5b681be0b 100644
--- a/internal/allocator/id.go
+++ b/internal/allocator/id.go
@@ -144,11 +144,10 @@ func (ia *IDAllocator) Alloc(count uint32) (UniqueID, UniqueID, error) {
req.count = count
ia.Reqs <- req
- req.Wait()
-
- if !req.IsValid() {
- return 0, 0, nil
+ if err := req.Wait(); err != nil {
+ return 0, 0, err
}
+
start, count := req.id, req.count
return start, start + int64(count), nil
}
diff --git a/internal/allocator/timestamp.go b/internal/allocator/timestamp.go
index 6fcd58f2d..c62acd428 100644
--- a/internal/allocator/timestamp.go
+++ b/internal/allocator/timestamp.go
@@ -2,7 +2,7 @@ package allocator
import (
"context"
- "errors"
+ "fmt"
"log"
"time"
@@ -144,10 +144,8 @@ func (ta *TimestampAllocator) Alloc(count uint32) ([]Timestamp, error) {
}
req.count = count
ta.Reqs <- req
- req.Wait()
-
- if !req.IsValid() {
- return nil, errors.New("alloc time stamp request failed")
+ if err := req.Wait(); err != nil {
+ return nil, fmt.Errorf("alloc time stamp request failed: %s", err)
}
start, count := req.timestamp, req.count
diff --git a/internal/proxynode/segment.go b/internal/proxynode/segment.go
index 65e58d52c..052bd09d5 100644
--- a/internal/proxynode/segment.go
+++ b/internal/proxynode/segment.go
@@ -355,10 +355,9 @@ func (sa *SegIDAssigner) GetSegmentID(collID UniqueID, partitionID UniqueID, cha
timestamp: ts,
}
sa.Reqs <- req
- req.Wait()
-
- if !req.IsValid() {
- return nil, errors.New("GetSegmentID Failed")
+ if err := req.Wait(); err != nil {
+ return nil, fmt.Errorf("GetSegmentID failed: %s", err)
}
+
return req.segInfo, nil
}
diff --git a/internal/util/typeutil/schema.go b/internal/util/typeutil/schema.go
index a5eb05bb0..5056a4867 100644
--- a/internal/util/typeutil/schema.go
+++ b/internal/util/typeutil/schema.go
@@ -20,14 +20,26 @@ func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
res += 8
case schemapb.DataType_String:
res += 125 // todo find a better way to estimate string type
- case schemapb.DataType_BinaryVector, schemapb.DataType_FloatVector:
+ case schemapb.DataType_BinaryVector:
for _, kv := range fs.TypeParams {
if kv.Key == "dim" {
v, err := strconv.Atoi(kv.Value)
if err != nil {
return -1, err
}
- res += v
+ res += v / 8
+ break
+ }
+ }
+ case schemapb.DataType_FloatVector:
+ for _, kv := range fs.TypeParams {
+ if kv.Key == "dim" {
+ v, err := strconv.Atoi(kv.Value)
+ if err != nil {
+ return -1, err
+ }
+ res += v * 4
+ break
}
}
}
--
GitLab