diff --git a/go.mod b/go.mod
index 74ed10b753decc4f0740af947abb8019a84b7ba0..adab48cd8c270cdd95ace9d3d04a6331ba34b922 100644
--- a/go.mod
+++ b/go.mod
@@ -33,7 +33,7 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/smartystreets/goconvey v1.7.2
github.com/stretchr/testify v1.7.5
- github.com/tidwall/btree v1.3.1
+ github.com/tidwall/btree v1.4.3
github.com/yireyun/go-queue v0.0.0-20220725040158-a4dd64810e1e
go.uber.org/zap v1.21.0
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10
diff --git a/go.sum b/go.sum
index 82649e0a36daf08e4d1d56cc59e2b69694535fcb..2b72851565a17d12d3f98620e2120dd7ce5c036a 100644
--- a/go.sum
+++ b/go.sum
@@ -456,6 +456,8 @@ github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324
github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/tidwall/btree v1.3.1 h1:636+tdVDs8Hjcf35Di260W2xCW4KuoXOKyk9QWOvCpA=
github.com/tidwall/btree v1.3.1/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE=
+github.com/tidwall/btree v1.4.3 h1:Lf5U/66bk0ftNppOBjVoy/AIPBrLMkheBp4NnSNiYOo=
+github.com/tidwall/btree v1.4.3/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE=
github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
diff --git a/pkg/txn/storage/txn/account.go b/pkg/txn/storage/txn/account.go
index a1db7617783aa0600f7b6bd4985df0421563ed1b..44b1ccffcba9199e6d561debe80bb4658b72a73b 100644
--- a/pkg/txn/storage/txn/account.go
+++ b/pkg/txn/storage/txn/account.go
@@ -15,7 +15,6 @@
package txnstorage
import (
- "github.com/google/uuid"
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog"
txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
)
@@ -39,8 +38,7 @@ func (m *MemHandler) ensureAccount(
if len(keys) == 0 {
// create one
db := DatabaseRow{
- ID: uuid.NewString(),
- NumberID: catalog.SystemDBID,
+ ID: txnengine.NewID(),
AccountID: accessInfo.AccountID,
Name: catalog.SystemDBName,
}
diff --git a/pkg/txn/storage/txn/batch.go b/pkg/txn/storage/txn/batch.go
index b01f60e5b77e76858da023047b3f11bb3e21d3c0..8840d02af71b79d5dc16eedeca966115783fa9bc 100644
--- a/pkg/txn/storage/txn/batch.go
+++ b/pkg/txn/storage/txn/batch.go
@@ -343,7 +343,7 @@ func vectorAt(vec *vector.Vector, i int) (value Nullable) {
}
- panic(fmt.Errorf("unknown column type: %v", vec.Typ))
+ panic(fmt.Sprintf("unknown column type: %v", vec.Typ))
}
func vectorAppend(vec *vector.Vector, value Nullable, heap *mheap.Mheap) {
diff --git a/pkg/txn/storage/txn/catalog.go b/pkg/txn/storage/txn/catalog.go
index 49a4cc19b8fcda57179f128d2cfc95c7881390e6..f653256f933319920cf3ea5af9736cb8797424c2 100644
--- a/pkg/txn/storage/txn/catalog.go
+++ b/pkg/txn/storage/txn/catalog.go
@@ -18,6 +18,7 @@ import (
"fmt"
"github.com/matrixorigin/matrixone/pkg/container/types"
+ "github.com/matrixorigin/matrixone/pkg/txn/storage/txn/memtable"
"github.com/matrixorigin/matrixone/pkg/vm/engine"
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog"
txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
@@ -35,15 +36,24 @@ var (
index_RowID = Text("row id")
)
+type (
+ DatabaseRowIter = Iter[ID, DatabaseRow]
+ RelationRowIter = Iter[ID, RelationRow]
+ AttributeRowIter = Iter[ID, AttributeRow]
+)
+
type DatabaseRow struct {
- ID string
- NumberID uint64
+ ID ID
AccountID uint32 // 0 is the sys account
Name string
}
-func (d DatabaseRow) Key() Text {
- return Text(d.ID)
+func (d DatabaseRow) Key() ID {
+ return d.ID
+}
+
+func (d DatabaseRow) Value() DatabaseRow {
+ return d
}
func (d DatabaseRow) Indexes() []Tuple {
@@ -61,8 +71,8 @@ func (d DatabaseRow) AttrByName(tx *Transaction, name string) (ret Nullable, err
if attr.Name != name {
continue
}
- if !typeMatch(ret.Value, attr.Type.Oid) {
- panic(fmt.Errorf("%s should be %v typed", name, attr.Type))
+ if !memtable.TypeMatch(ret.Value, attr.Type.Oid) {
+ panic(fmt.Sprintf("%s should be %v typed", name, attr.Type))
}
}
}()
@@ -74,17 +84,16 @@ func (d DatabaseRow) AttrByName(tx *Transaction, name string) (ret Nullable, err
case catalog.SystemDBAttr_CreateSQL:
ret.Value = ""
case catalog.SystemDBAttr_ID:
- ret.Value = d.NumberID
+ ret.Value = uint64(d.ID)
default:
- panic(fmt.Errorf("fixme: %s", name))
+ panic(fmt.Sprintf("fixme: %s", name))
}
return
}
type RelationRow struct {
- ID string
- NumberID uint64
- DatabaseID string
+ ID ID
+ DatabaseID ID
Name string
Type txnengine.RelationType
Comments string
@@ -95,14 +104,18 @@ type RelationRow struct {
handler *MemHandler
}
-func (r RelationRow) Key() Text {
- return Text(r.ID)
+func (r RelationRow) Key() ID {
+ return r.ID
+}
+
+func (r RelationRow) Value() RelationRow {
+ return r
}
func (r RelationRow) Indexes() []Tuple {
return []Tuple{
- {index_DatabaseID, Text(r.DatabaseID)},
- {index_DatabaseID_Name, Text(r.DatabaseID), Text(r.Name)},
+ {index_DatabaseID, r.DatabaseID},
+ {index_DatabaseID_Name, r.DatabaseID, Text(r.Name)},
}
}
@@ -114,22 +127,22 @@ func (r RelationRow) AttrByName(tx *Transaction, name string) (ret Nullable, err
if attr.Name != name {
continue
}
- if !typeMatch(ret.Value, attr.Type.Oid) {
- panic(fmt.Errorf("%s should be %v typed", name, attr.Type))
+ if !memtable.TypeMatch(ret.Value, attr.Type.Oid) {
+ panic(fmt.Sprintf("%s should be %v typed", name, attr.Type))
}
}
}()
switch name {
case catalog.SystemRelAttr_ID:
- ret.Value = r.NumberID
+ ret.Value = uint64(r.ID)
case catalog.SystemRelAttr_Name:
ret.Value = r.Name
case catalog.SystemRelAttr_DBName:
- if r.DatabaseID == "" {
+ if r.DatabaseID.IsEmpty() {
ret.Value = ""
return
}
- db, err := r.handler.databases.Get(tx, Text(r.DatabaseID))
+ db, err := r.handler.databases.Get(tx, r.DatabaseID)
if err != nil {
return ret, err
}
@@ -143,24 +156,24 @@ func (r RelationRow) AttrByName(tx *Transaction, name string) (ret Nullable, err
case catalog.SystemRelAttr_CreateSQL:
ret.Value = ""
case catalog.SystemRelAttr_DBID:
- if r.DatabaseID == "" {
+ if r.DatabaseID.IsEmpty() {
ret.Value = ""
return
}
- db, err := r.handler.databases.Get(tx, Text(r.DatabaseID))
+ db, err := r.handler.databases.Get(tx, r.DatabaseID)
if err != nil {
return ret, err
}
- ret.Value = db.NumberID
+ ret.Value = uint64(db.ID)
default:
- panic(fmt.Errorf("fixme: %s", name))
+ panic(fmt.Sprintf("fixme: %s", name))
}
return
}
type AttributeRow struct {
- ID string
- RelationID string
+ ID ID
+ RelationID ID
Order int
Nullable bool
engine.Attribute
@@ -168,16 +181,20 @@ type AttributeRow struct {
handler *MemHandler
}
-func (a AttributeRow) Key() Text {
- return Text(a.ID)
+func (a AttributeRow) Key() ID {
+ return a.ID
+}
+
+func (a AttributeRow) Value() AttributeRow {
+ return a
}
func (a AttributeRow) Indexes() []Tuple {
return []Tuple{
- {index_RelationID, Text(a.RelationID)},
- {index_RelationID_Name, Text(a.RelationID), Text(a.Name)},
- {index_RelationID_IsPrimary, Text(a.RelationID), Bool(a.Primary)},
- {index_RelationID_IsHidden, Text(a.RelationID), Bool(a.IsHidden)},
+ {index_RelationID, a.RelationID},
+ {index_RelationID_Name, a.RelationID, Text(a.Name)},
+ {index_RelationID_IsPrimary, a.RelationID, Bool(a.Primary)},
+ {index_RelationID_IsHidden, a.RelationID, Bool(a.IsHidden)},
}
}
@@ -189,28 +206,28 @@ func (a AttributeRow) AttrByName(tx *Transaction, name string) (ret Nullable, er
if attr.Name != name {
continue
}
- if !typeMatch(ret.Value, attr.Type.Oid) {
- panic(fmt.Errorf("%s should be %v typed", name, attr.Type))
+ if !memtable.TypeMatch(ret.Value, attr.Type.Oid) {
+ panic(fmt.Sprintf("%s should be %v typed", name, attr.Type))
}
}
}()
switch name {
case catalog.SystemColAttr_DBName:
- rel, err := a.handler.relations.Get(tx, Text(a.RelationID))
+ rel, err := a.handler.relations.Get(tx, a.RelationID)
if err != nil {
return ret, err
}
- if rel.DatabaseID == "" {
+ if rel.DatabaseID.IsEmpty() {
ret.Value = ""
return ret, nil
}
- db, err := a.handler.databases.Get(tx, Text(rel.DatabaseID))
+ db, err := a.handler.databases.Get(tx, rel.DatabaseID)
if err != nil {
return ret, err
}
ret.Value = db.Name
case catalog.SystemColAttr_RelName:
- rel, err := a.handler.relations.Get(tx, Text(a.RelationID))
+ rel, err := a.handler.relations.Get(tx, a.RelationID)
if err != nil {
return ret, err
}
@@ -250,24 +267,28 @@ func (a AttributeRow) AttrByName(tx *Transaction, name string) (ret Nullable, er
case catalog.SystemColAttr_IsHidden:
ret.Value = boolToInt8(a.IsHidden)
default:
- panic(fmt.Errorf("fixme: %s", name))
+ panic(fmt.Sprintf("fixme: %s", name))
}
return
}
type IndexRow struct {
- ID string
- RelationID string
+ ID ID
+ RelationID ID
engine.IndexTableDef
}
-func (i IndexRow) Key() Text {
- return Text(i.ID)
+func (i IndexRow) Key() ID {
+ return i.ID
+}
+
+func (i IndexRow) Value() IndexRow {
+ return i
}
func (i IndexRow) Indexes() []Tuple {
return []Tuple{
- {index_RelationID, Text(i.RelationID)},
- {index_RelationID_Name, Text(i.RelationID), Text(i.Name)},
+ {index_RelationID, i.RelationID},
+ {index_RelationID_Name, i.RelationID, Text(i.Name)},
}
}
diff --git a/pkg/txn/storage/txn/catalog_handler.go b/pkg/txn/storage/txn/catalog_handler.go
index e4e284a61b80bfdb82697a022cc62b14e231bc3f..fdadec460ec8bf297b5380088ecdb48790875b36 100644
--- a/pkg/txn/storage/txn/catalog_handler.go
+++ b/pkg/txn/storage/txn/catalog_handler.go
@@ -20,11 +20,13 @@ import (
"sync"
"github.com/google/uuid"
+ "github.com/matrixorigin/matrixone/pkg/common/moerr"
"github.com/matrixorigin/matrixone/pkg/container/batch"
"github.com/matrixorigin/matrixone/pkg/container/nulls"
"github.com/matrixorigin/matrixone/pkg/container/vector"
"github.com/matrixorigin/matrixone/pkg/pb/timestamp"
"github.com/matrixorigin/matrixone/pkg/pb/txn"
+ "github.com/matrixorigin/matrixone/pkg/txn/storage/txn/memtable"
"github.com/matrixorigin/matrixone/pkg/vm/engine"
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog"
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/moengine"
@@ -36,11 +38,11 @@ import (
// CatalogHandler handles read-only requests for catalog
type CatalogHandler struct {
upstream *MemHandler
- dbID string
- sysRelationIDs map[string]string
+ dbID ID
+ sysRelationIDs map[ID]string
iterators struct {
sync.Mutex
- Map map[string]any // id -> Iterator
+ Map map[ID]any // id -> Iterator
}
}
@@ -50,16 +52,16 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
handler := &CatalogHandler{
upstream: upstream,
- sysRelationIDs: make(map[string]string),
+ sysRelationIDs: make(map[ID]string),
}
- handler.iterators.Map = make(map[string]any)
+ handler.iterators.Map = make(map[ID]any)
now := Time{
Timestamp: timestamp.Timestamp{
PhysicalTime: math.MinInt,
},
}
- tx := NewTransaction(uuid.NewString(), now, SnapshotIsolation)
+ tx := memtable.NewTransaction(uuid.NewString(), now, memtable.SnapshotIsolation)
defer func() {
if err := tx.Commit(); err != nil {
panic(err)
@@ -68,8 +70,7 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
// database
db := DatabaseRow{
- ID: uuid.NewString(),
- NumberID: catalog.SystemDBID,
+ ID: ID(catalog.SystemDBID),
AccountID: 0,
Name: catalog.SystemDBName,
}
@@ -80,8 +81,7 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
// relations
databasesRelRow := RelationRow{
- ID: uuid.NewString(),
- NumberID: catalog.SystemTable_DB_ID,
+ ID: ID(catalog.SystemTable_DB_ID),
DatabaseID: db.ID,
Name: catalog.SystemTable_DB_Name,
Type: txnengine.RelationTable,
@@ -92,8 +92,7 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
handler.sysRelationIDs[databasesRelRow.ID] = databasesRelRow.Name
tablesRelRow := RelationRow{
- ID: uuid.NewString(),
- NumberID: catalog.SystemTable_Table_ID,
+ ID: ID(catalog.SystemTable_Table_ID),
DatabaseID: db.ID,
Name: catalog.SystemTable_Table_Name,
Type: txnengine.RelationTable,
@@ -104,8 +103,7 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
handler.sysRelationIDs[tablesRelRow.ID] = tablesRelRow.Name
attributesRelRow := RelationRow{
- ID: uuid.NewString(),
- NumberID: catalog.SystemTable_Columns_ID,
+ ID: ID(catalog.SystemTable_Columns_ID),
DatabaseID: db.ID,
Name: catalog.SystemTable_Columns_Name,
Type: txnengine.RelationTable,
@@ -127,7 +125,7 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
continue
}
row := AttributeRow{
- ID: uuid.NewString(),
+ ID: txnengine.NewID(),
RelationID: databasesRelRow.ID,
Order: i,
Nullable: true,
@@ -148,7 +146,7 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
continue
}
row := AttributeRow{
- ID: uuid.NewString(),
+ ID: txnengine.NewID(),
RelationID: tablesRelRow.ID,
Order: i,
Nullable: true,
@@ -169,7 +167,7 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
continue
}
row := AttributeRow{
- ID: uuid.NewString(),
+ ID: txnengine.NewID(),
RelationID: attributesRelRow.ID,
Order: i,
Nullable: true,
@@ -184,10 +182,9 @@ func NewCatalogHandler(upstream *MemHandler) *CatalogHandler {
}
func (c *CatalogHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableDefReq, resp *txnengine.AddTableDefResp) (err error) {
- if name, ok := c.sysRelationIDs[req.TableID]; ok {
+ if _, ok := c.sysRelationIDs[req.TableID]; ok {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = fmt.Sprintf("%s is system table", name)
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
return c.upstream.HandleAddTableDef(meta, req, resp)
}
@@ -204,20 +201,20 @@ func (c *CatalogHandler) HandleCloseTableIter(meta txn.TxnMeta, req txnengine.Cl
if ok {
defer logReq("catalog", req, meta, resp, &err)()
switch v := v.(type) {
- case *Iter[Text, DatabaseRow]:
+ case *DatabaseRowIter:
if err := v.TableIter.Close(); err != nil {
return err
}
- case *Iter[Text, RelationRow]:
+ case *RelationRowIter:
if err := v.TableIter.Close(); err != nil {
return err
}
- case *Iter[Text, AttributeRow]:
+ case *AttributeRowIter:
if err := v.TableIter.Close(); err != nil {
return err
}
default:
- panic(fmt.Errorf("fixme: %T", v))
+ panic(fmt.Sprintf("fixme: %T", v))
}
c.iterators.Lock()
delete(c.iterators.Map, req.IterID)
@@ -246,8 +243,7 @@ func (c *CatalogHandler) HandleCreateDatabase(meta txn.TxnMeta, req txnengine.Cr
if req.Name == catalog.SystemDBName {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = req.Name + " is system database"
- return nil
+ return moerr.NewDBAlreadyExists(req.Name)
}
return c.upstream.HandleCreateDatabase(meta, req, resp)
}
@@ -257,19 +253,17 @@ func (c *CatalogHandler) HandleCreateRelation(meta txn.TxnMeta, req txnengine.Cr
}
func (c *CatalogHandler) HandleDelTableDef(meta txn.TxnMeta, req txnengine.DelTableDefReq, resp *txnengine.DelTableDefResp) (err error) {
- if name, ok := c.sysRelationIDs[req.TableID]; ok {
+ if _, ok := c.sysRelationIDs[req.TableID]; ok {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = fmt.Sprintf("%s is system table", name)
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
return c.upstream.HandleDelTableDef(meta, req, resp)
}
func (c *CatalogHandler) HandleDelete(meta txn.TxnMeta, req txnengine.DeleteReq, resp *txnengine.DeleteResp) (err error) {
- if name, ok := c.sysRelationIDs[req.TableID]; ok {
+ if _, ok := c.sysRelationIDs[req.TableID]; ok {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = fmt.Sprintf("%s is system table", name)
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
return c.upstream.HandleDelete(meta, req, resp)
}
@@ -283,8 +277,7 @@ func (c *CatalogHandler) HandleDeleteDatabase(meta txn.TxnMeta, req txnengine.De
if req.Name == catalog.SystemDBName {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = fmt.Sprintf("%s is system database", req.Name)
- return nil
+ return moerr.NewBadDB(req.Name)
}
return c.upstream.HandleDeleteDatabase(meta, req, resp)
}
@@ -294,8 +287,7 @@ func (c *CatalogHandler) HandleDeleteRelation(meta txn.TxnMeta, req txnengine.De
for _, name := range c.sysRelationIDs {
if req.Name == name {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = "can't delete this system table"
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.Name)
}
}
}
@@ -344,7 +336,7 @@ func (c *CatalogHandler) HandleNewTableIter(meta txn.TxnMeta, req txnengine.NewT
attrsMap := make(map[string]*AttributeRow)
if err := c.upstream.iterRelationAttributes(
tx, req.TableID,
- func(_ Text, row *AttributeRow) error {
+ func(_ ID, row *AttributeRow) error {
attrsMap[row.Name] = row
return nil
},
@@ -356,30 +348,30 @@ func (c *CatalogHandler) HandleNewTableIter(meta txn.TxnMeta, req txnengine.NewT
switch name {
case catalog.SystemTable_DB_Name:
tableIter := c.upstream.databases.NewIter(tx)
- iter = &Iter[Text, DatabaseRow]{
+ iter = &DatabaseRowIter{
TableIter: tableIter,
AttrsMap: attrsMap,
nextFunc: tableIter.First,
}
case catalog.SystemTable_Table_Name:
tableIter := c.upstream.relations.NewIter(tx)
- iter = &Iter[Text, RelationRow]{
+ iter = &RelationRowIter{
TableIter: tableIter,
AttrsMap: attrsMap,
nextFunc: tableIter.First,
}
case catalog.SystemTable_Columns_Name:
tableIter := c.upstream.attributes.NewIter(tx)
- iter = &Iter[Text, AttributeRow]{
+ iter = &AttributeRowIter{
TableIter: tableIter,
AttrsMap: attrsMap,
nextFunc: tableIter.First,
}
default:
- panic(fmt.Errorf("fixme: %s", name))
+ panic(fmt.Sprintf("fixme: %s", name))
}
- id := uuid.NewString()
+ id := txnengine.NewID()
resp.IterID = id
c.iterators.Lock()
c.iterators.Map[id] = iter
@@ -443,7 +435,7 @@ func (c *CatalogHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, res
switch iter := v.(type) {
- case *Iter[Text, DatabaseRow]:
+ case *DatabaseRowIter:
for i, name := range req.ColNames {
b.Vecs[i] = vector.New(iter.AttrsMap[name].Type)
}
@@ -465,7 +457,7 @@ func (c *CatalogHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, res
}
}
- case *Iter[Text, RelationRow]:
+ case *RelationRowIter:
for i, name := range req.ColNames {
b.Vecs[i] = vector.New(iter.AttrsMap[name].Type)
}
@@ -488,7 +480,7 @@ func (c *CatalogHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, res
}
}
- case *Iter[Text, AttributeRow]:
+ case *AttributeRowIter:
for i, name := range req.ColNames {
b.Vecs[i] = vector.New(iter.AttrsMap[name].Type)
}
@@ -515,7 +507,7 @@ func (c *CatalogHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, res
}
default:
- panic(fmt.Errorf("fixme: %T", v))
+ panic(fmt.Sprintf("fixme: %T", v))
}
if rows > 0 {
@@ -541,25 +533,25 @@ func (c *CatalogHandler) HandleStartRecovery(ch chan txn.TxnMeta) {
}
func (c *CatalogHandler) HandleTruncate(meta txn.TxnMeta, req txnengine.TruncateReq, resp *txnengine.TruncateResp) (err error) {
- if name, ok := c.sysRelationIDs[req.TableID]; ok {
+ if _, ok := c.sysRelationIDs[req.TableID]; ok {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = fmt.Sprintf("%s is system table", name)
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
return c.upstream.HandleTruncate(meta, req, resp)
}
func (c *CatalogHandler) HandleUpdate(meta txn.TxnMeta, req txnengine.UpdateReq, resp *txnengine.UpdateResp) (err error) {
- if name, ok := c.sysRelationIDs[req.TableID]; ok {
+ if _, ok := c.sysRelationIDs[req.TableID]; ok {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = fmt.Sprintf("%s is system table", name)
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
return c.upstream.HandleUpdate(meta, req, resp)
}
func (c *CatalogHandler) HandleWrite(meta txn.TxnMeta, req txnengine.WriteReq, resp *txnengine.WriteResp) (err error) {
- if name, ok := c.sysRelationIDs[req.TableID]; ok {
+ if _, ok := c.sysRelationIDs[req.TableID]; ok {
defer logReq("catalog", req, meta, resp, &err)()
- resp.ErrResp.Why = fmt.Sprintf("%s is system table", name)
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
err = c.upstream.HandleWrite(meta, req, resp)
return
diff --git a/pkg/txn/storage/txn/catalog_handler_test.go b/pkg/txn/storage/txn/catalog_handler_test.go
index 1e94309ef7e81d6b052f0e01d25717fd5b3d26e3..c848019173b1f508b290b344e2ac7e7f8d1ccda4 100644
--- a/pkg/txn/storage/txn/catalog_handler_test.go
+++ b/pkg/txn/storage/txn/catalog_handler_test.go
@@ -24,6 +24,7 @@ import (
"github.com/matrixorigin/matrixone/pkg/pb/txn"
"github.com/matrixorigin/matrixone/pkg/testutil"
"github.com/matrixorigin/matrixone/pkg/txn/clock"
+ "github.com/matrixorigin/matrixone/pkg/txn/storage/txn/memtable"
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/catalog"
txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
"github.com/stretchr/testify/assert"
@@ -41,7 +42,7 @@ func TestCatalogHandler(t *testing.T) {
NewCatalogHandler(
NewMemHandler(
testutil.NewMheap(),
- Serializable,
+ memtable.Serializable,
clock,
),
),
@@ -60,7 +61,7 @@ func TestCatalogHandler(t *testing.T) {
}
// system db
- var dbID string
+ var dbID ID
{
var resp txnengine.OpenDatabaseResp
err = handler.HandleOpenDatabase(meta, txnengine.OpenDatabaseReq{
@@ -82,7 +83,7 @@ func TestCatalogHandler(t *testing.T) {
}
// mo_database
- var tableID string
+ var tableID ID
{
var resp txnengine.OpenRelationResp
err = handler.HandleOpenRelation(meta, txnengine.OpenRelationReq{
@@ -93,7 +94,7 @@ func TestCatalogHandler(t *testing.T) {
tableID = resp.ID
assert.NotEmpty(t, tableID)
}
- var iterID string
+ var iterID ID
{
var resp txnengine.NewTableIterResp
err = handler.HandleNewTableIter(meta, txnengine.NewTableIterReq{
diff --git a/pkg/txn/storage/txn/data.go b/pkg/txn/storage/txn/data.go
index f59989b0316ef8f8191e85fbabc664b7adebbc29..0769e4788462a5fabc759c324f014fd7492e5117 100644
--- a/pkg/txn/storage/txn/data.go
+++ b/pkg/txn/storage/txn/data.go
@@ -20,30 +20,37 @@ import (
)
type DataKey struct {
- tableID string
+ tableID ID
primaryKey Tuple
}
func (d DataKey) Less(than DataKey) bool {
- if d.tableID < than.tableID {
+ if d.tableID.Less(than.tableID) {
return true
}
- if d.tableID > than.tableID {
+ if than.tableID.Less(d.tableID) {
return false
}
return d.primaryKey.Less(than.primaryKey)
}
+// use AttributeRow.Order as index
+type DataValue = []Nullable
+
type DataRow struct {
- key DataKey
- indexes []Tuple
- attributes map[string]Nullable // attribute id -> nullable value
+ key DataKey
+ value DataValue
+ indexes []Tuple
}
func (a DataRow) Key() DataKey {
return a.key
}
+func (a DataRow) Value() DataValue {
+ return a.value
+}
+
func (a DataRow) Indexes() []Tuple {
return a.indexes
}
@@ -52,33 +59,32 @@ func (a *DataRow) String() string {
buf := new(strings.Builder)
buf.WriteString("DataRow{")
buf.WriteString(fmt.Sprintf("key: %+v", a.key))
- for key, value := range a.attributes {
- buf.WriteString(fmt.Sprintf(", %s: %+v", key, value))
+ for _, attr := range a.value {
+ buf.WriteString(fmt.Sprintf(", %+v", attr))
}
buf.WriteString("}")
return buf.String()
}
func NewDataRow(
- tableID string,
+ tableID ID,
indexes []Tuple,
) *DataRow {
return &DataRow{
key: DataKey{
tableID: tableID,
},
- attributes: make(map[string]Nullable),
- indexes: indexes,
+ indexes: indexes,
}
}
type NamedDataRow struct {
- Row *DataRow
+ Value DataValue
AttrsMap map[string]*AttributeRow
}
var _ NamedRow = new(NamedDataRow)
func (n *NamedDataRow) AttrByName(tx *Transaction, name string) (Nullable, error) {
- return n.Row.attributes[n.AttrsMap[name].ID], nil
+ return n.Value[n.AttrsMap[name].Order], nil
}
diff --git a/pkg/txn/storage/txn/frontend_test.go b/pkg/txn/storage/txn/frontend_test.go
index c39fd1342e48a3fc264bebe0d09acccabdddf674..f8c1740562238f9def450a4cd572a68bd0fcfbf7 100644
--- a/pkg/txn/storage/txn/frontend_test.go
+++ b/pkg/txn/storage/txn/frontend_test.go
@@ -26,6 +26,7 @@ import (
logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice"
"github.com/matrixorigin/matrixone/pkg/testutil"
"github.com/matrixorigin/matrixone/pkg/txn/clock"
+ "github.com/matrixorigin/matrixone/pkg/txn/storage/txn/memtable"
txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
"github.com/matrixorigin/matrixone/pkg/vm/mempool"
"github.com/matrixorigin/matrixone/pkg/vm/mheap"
@@ -35,7 +36,6 @@ import (
)
func TestFrontend(t *testing.T) {
- t.Skip("Skip because of error handling refactor work.")
ctx, cancel := context.WithTimeout(
context.Background(),
time.Minute,
@@ -85,7 +85,7 @@ func TestFrontend(t *testing.T) {
}, math.MaxInt)
storage, err := NewMemoryStorage(
heap,
- SnapshotIsolation,
+ memtable.SnapshotIsolation,
clock,
)
assert.Nil(t, err)
diff --git a/pkg/txn/storage/txn/handler.go b/pkg/txn/storage/txn/handler.go
index 2dae212dd9272be927f7b683361383d6b47836fd..2dd3d62d99ca7c585d3f61153224c870c01424ff 100644
--- a/pkg/txn/storage/txn/handler.go
+++ b/pkg/txn/storage/txn/handler.go
@@ -15,6 +15,7 @@
package txnstorage
import (
+ apipb "github.com/matrixorigin/matrixone/pkg/pb/api"
"github.com/matrixorigin/matrixone/pkg/pb/timestamp"
"github.com/matrixorigin/matrixone/pkg/pb/txn"
txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
@@ -176,7 +177,7 @@ type Handler interface {
HandleGetLogTail(
meta txn.TxnMeta,
- req txnengine.GetLogTailReq,
- resp *txnengine.GetLogTailResp,
+ req apipb.SyncLogTailReq,
+ resp *apipb.SyncLogTailResp,
) error
}
diff --git a/pkg/txn/storage/txn/log_tail.go b/pkg/txn/storage/txn/log_tail.go
index d3f80c652ae48bb7384bbda6d5d4424e7a46e044..c5782c5b54aa661ec86a682ac80e1bc97c4c189e 100644
--- a/pkg/txn/storage/txn/log_tail.go
+++ b/pkg/txn/storage/txn/log_tail.go
@@ -19,31 +19,32 @@ import (
"errors"
"math"
+ "github.com/matrixorigin/matrixone/pkg/common/moerr"
"github.com/matrixorigin/matrixone/pkg/container/batch"
"github.com/matrixorigin/matrixone/pkg/container/vector"
apipb "github.com/matrixorigin/matrixone/pkg/pb/api"
"github.com/matrixorigin/matrixone/pkg/pb/timestamp"
"github.com/matrixorigin/matrixone/pkg/pb/txn"
- txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
+ "github.com/matrixorigin/matrixone/pkg/txn/storage/txn/memtable"
)
type LogTailEntry = apipb.Entry
-func (m *MemHandler) HandleGetLogTail(meta txn.TxnMeta, req txnengine.GetLogTailReq, resp *txnengine.GetLogTailResp) (err error) {
+func (m *MemHandler) HandleGetLogTail(meta txn.TxnMeta, req apipb.SyncLogTailReq, resp *apipb.SyncLogTailResp) (err error) {
+ tableID := ID(req.Table.TbId)
// tx
tx := m.getTx(meta)
// table and db infos
- tableRow, err := m.relations.Get(tx, Text(req.TableID))
+ tableRow, err := m.relations.Get(tx, tableID)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrRelationNotFound.ID = req.TableID
- return nil
+ return moerr.NewInternalError("invalid relation id %v", tableID)
}
return err
}
- dbRow, err := m.databases.Get(tx, Text(tableRow.DatabaseID))
+ dbRow, err := m.databases.Get(tx, tableRow.DatabaseID)
if err != nil {
return err
}
@@ -52,14 +53,14 @@ func (m *MemHandler) HandleGetLogTail(meta txn.TxnMeta, req txnengine.GetLogTail
from := timestamp.Timestamp{
PhysicalTime: math.MinInt64,
}
- if req.Request.CnHave != nil {
- from = *req.Request.CnHave
+ if req.CnHave != nil {
+ from = *req.CnHave
}
to := timestamp.Timestamp{
PhysicalTime: math.MaxInt64,
}
- if req.Request.CnWant != nil {
- to = *req.Request.CnWant
+ if req.CnWant != nil {
+ to = *req.CnWant
}
fromTime := Time{
Timestamp: from,
@@ -69,21 +70,21 @@ func (m *MemHandler) HandleGetLogTail(meta txn.TxnMeta, req txnengine.GetLogTail
}
// attributes
- rows, err := m.attributes.IndexRows(tx, Tuple{
+ attrs, err := m.attributes.Index(tx, Tuple{
index_RelationID,
- Text(req.TableID),
+ tableID,
})
if err != nil {
return err
}
attrsMap := make(map[string]*AttributeRow)
- insertNames := make([]string, 0, len(rows))
- deleteNames := make([]string, 0, len(rows))
- for _, row := range rows {
- attrsMap[row.Name] = row
- insertNames = append(insertNames, row.Name)
- if row.Primary || row.IsRowId {
- deleteNames = append(deleteNames, row.Name)
+ insertNames := make([]string, 0, len(attrs))
+ deleteNames := make([]string, 0, len(attrs))
+ for _, attr := range attrs {
+ attrsMap[attr.Value.Name] = attr.Value
+ insertNames = append(insertNames, attr.Value.Name)
+ if attr.Value.Primary || attr.Value.IsRowId {
+ deleteNames = append(deleteNames, attr.Value.Name)
}
}
@@ -99,33 +100,32 @@ func (m *MemHandler) HandleGetLogTail(meta txn.TxnMeta, req txnengine.GetLogTail
// iter
// we don't use m.data.NewIter because we want to see deleted rows
- iter := m.data.rows.Iter()
- defer iter.Release()
- tableKey := &PhysicalRow[DataKey, DataRow]{
+ iter := m.data.NewPhysicalIter()
+ defer iter.Close()
+ tableKey := &memtable.PhysicalRow[DataKey, DataValue]{
Key: DataKey{
- tableID: req.TableID,
+ tableID: tableID,
primaryKey: Tuple{},
},
}
for ok := iter.Seek(tableKey); ok; ok = iter.Next() {
physicalRow := iter.Item()
- if physicalRow.Key.tableID != req.TableID {
+ if physicalRow.Key.tableID != tableID {
break
}
- values := physicalRow.Values
- values.RLock()
- for i := len(values.Values) - 1; i >= 0; i-- {
- value := values.Values[i]
+ physicalRow.Versions.RLock()
+ for i := len(physicalRow.Versions.List) - 1; i >= 0; i-- {
+ value := physicalRow.Versions.List[i]
if value.LockTx != nil &&
- value.LockTx.State.Load() == Committed &&
+ value.LockTx.State.Load() == memtable.Committed &&
value.LockTime.After(fromTime) &&
value.LockTime.Before(toTime) {
// committed delete
namedRow := &NamedDataRow{
- Row: value.Value,
+ Value: *value.Value,
AttrsMap: attrsMap,
}
if err := appendNamedRow(tx, m.mheap, deleteBatch, namedRow); err != nil {
@@ -133,12 +133,12 @@ func (m *MemHandler) HandleGetLogTail(meta txn.TxnMeta, req txnengine.GetLogTail
}
break
- } else if value.BornTx.State.Load() == Committed &&
+ } else if value.BornTx.State.Load() == memtable.Committed &&
value.BornTime.After(fromTime) &&
value.BornTime.Before(toTime) {
// committed insert
namedRow := &NamedDataRow{
- Row: value.Value,
+ Value: *value.Value,
AttrsMap: attrsMap,
}
if err := appendNamedRow(tx, m.mheap, insertBatch, namedRow); err != nil {
@@ -149,7 +149,7 @@ func (m *MemHandler) HandleGetLogTail(meta txn.TxnMeta, req txnengine.GetLogTail
}
}
- values.RUnlock()
+ physicalRow.Versions.RUnlock()
}
@@ -158,27 +158,27 @@ func (m *MemHandler) HandleGetLogTail(meta txn.TxnMeta, req txnengine.GetLogTail
{
EntryType: apipb.Entry_Insert,
Bat: toPBBatch(insertBatch),
- TableId: tableRow.NumberID,
+ TableId: uint64(tableRow.ID),
TableName: tableRow.Name,
- DatabaseId: dbRow.NumberID,
+ DatabaseId: uint64(dbRow.ID),
DatabaseName: dbRow.Name,
},
{
EntryType: apipb.Entry_Delete,
Bat: toPBBatch(deleteBatch),
- TableId: tableRow.NumberID,
+ TableId: uint64(tableRow.ID),
TableName: tableRow.Name,
- DatabaseId: dbRow.NumberID,
+ DatabaseId: uint64(dbRow.ID),
DatabaseName: dbRow.Name,
},
}
- resp.Response.Commands = entries
+ resp.Commands = entries
return nil
}
-func (c *CatalogHandler) HandleGetLogTail(meta txn.TxnMeta, req txnengine.GetLogTailReq, resp *txnengine.GetLogTailResp) (err error) {
+func (c *CatalogHandler) HandleGetLogTail(meta txn.TxnMeta, req apipb.SyncLogTailReq, resp *apipb.SyncLogTailResp) (err error) {
return c.upstream.HandleGetLogTail(meta, req, resp)
}
diff --git a/pkg/txn/storage/txn/log_tail_test.go b/pkg/txn/storage/txn/log_tail_test.go
index de9c895664299109be5490a721ea830eeafee10e..a3d69b6d83834d1f45ef044398e7dfcdb3281603 100644
--- a/pkg/txn/storage/txn/log_tail_test.go
+++ b/pkg/txn/storage/txn/log_tail_test.go
@@ -18,6 +18,7 @@ import (
"context"
"testing"
+ apipb "github.com/matrixorigin/matrixone/pkg/pb/api"
"github.com/matrixorigin/matrixone/pkg/pb/timestamp"
"github.com/matrixorigin/matrixone/pkg/pb/txn"
txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
@@ -46,14 +47,16 @@ func testLogTail(
// test get log tail
{
- resp := testRead[txnengine.GetLogTailResp](
+ resp, err := testRead[apipb.SyncLogTailResp](
t, s, txnMeta,
txnengine.OpGetLogTail,
- txnengine.GetLogTailReq{
+ apipb.SyncLogTailReq{
+ Table: &apipb.TableID{},
//TODO args
},
)
//TODO asserts
+ _ = err
_ = resp
}
}
diff --git a/pkg/txn/storage/txn/mem_handler.go b/pkg/txn/storage/txn/mem_handler.go
index 90bd2e66595d7a1c8822b98c578844951eb0ffa6..ec8f5a5b38ee45e69051c61c7cc5c6713422f095 100644
--- a/pkg/txn/storage/txn/mem_handler.go
+++ b/pkg/txn/storage/txn/mem_handler.go
@@ -15,14 +15,15 @@
package txnstorage
import (
+ crand "crypto/rand"
"database/sql"
+ "encoding/binary"
"errors"
"fmt"
- "math/rand"
"sort"
"sync"
- "github.com/google/uuid"
+ "github.com/matrixorigin/matrixone/pkg/common/moerr"
"github.com/matrixorigin/matrixone/pkg/container/batch"
"github.com/matrixorigin/matrixone/pkg/container/nulls"
"github.com/matrixorigin/matrixone/pkg/container/types"
@@ -31,6 +32,7 @@ import (
"github.com/matrixorigin/matrixone/pkg/pb/timestamp"
"github.com/matrixorigin/matrixone/pkg/pb/txn"
"github.com/matrixorigin/matrixone/pkg/txn/clock"
+ "github.com/matrixorigin/matrixone/pkg/txn/storage/txn/memtable"
"github.com/matrixorigin/matrixone/pkg/vm/engine"
txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
"github.com/matrixorigin/matrixone/pkg/vm/mheap"
@@ -39,13 +41,13 @@ import (
type MemHandler struct {
// catalog
- databases *Table[Text, DatabaseRow]
- relations *Table[Text, RelationRow]
- attributes *Table[Text, AttributeRow]
- indexes *Table[Text, IndexRow]
+ databases *memtable.Table[ID, DatabaseRow, DatabaseRow]
+ relations *memtable.Table[ID, RelationRow, RelationRow]
+ attributes *memtable.Table[ID, AttributeRow, AttributeRow]
+ indexes *memtable.Table[ID, IndexRow, IndexRow]
// data
- data *Table[DataKey, DataRow]
+ data *memtable.Table[DataKey, DataValue, DataRow]
// transactions
transactions struct {
@@ -58,7 +60,7 @@ type MemHandler struct {
iterators struct {
sync.Mutex
// iterator id -> iterator
- Map map[string]*Iter[DataKey, DataRow]
+ Map map[ID]*Iter[DataKey, DataValue]
}
// misc
@@ -68,14 +70,16 @@ type MemHandler struct {
}
type Iter[
- K Ordered[K],
- R Row[K],
+ K memtable.Ordered[K],
+ V any,
] struct {
- TableIter *TableIter[K, R]
- TableID string
+ TableIter *memtable.TableIter[K, V]
+ TableID ID
AttrsMap map[string]*AttributeRow
Expr *plan.Expr
nextFunc func() bool
+ ReadTime Time
+ Tx *Transaction
}
func NewMemHandler(
@@ -84,17 +88,17 @@ func NewMemHandler(
clock clock.Clock,
) *MemHandler {
h := &MemHandler{
- databases: NewTable[Text, DatabaseRow](),
- relations: NewTable[Text, RelationRow](),
- attributes: NewTable[Text, AttributeRow](),
- data: NewTable[DataKey, DataRow](),
- indexes: NewTable[Text, IndexRow](),
+ databases: memtable.NewTable[ID, DatabaseRow, DatabaseRow](),
+ relations: memtable.NewTable[ID, RelationRow, RelationRow](),
+ attributes: memtable.NewTable[ID, AttributeRow, AttributeRow](),
+ indexes: memtable.NewTable[ID, IndexRow, IndexRow](),
+ data: memtable.NewTable[DataKey, DataValue, DataRow](),
mheap: mheap,
defaultIsolationPolicy: defaultIsolationPolicy,
clock: clock,
}
h.transactions.Map = make(map[string]*Transaction)
- h.iterators.Map = make(map[string]*Iter[DataKey, DataRow])
+ h.iterators.Map = make(map[ID]*Iter[DataKey, DataValue])
return h
}
@@ -106,7 +110,7 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
maxAttributeOrder := 0
if err := m.iterRelationAttributes(
tx, req.TableID,
- func(_ Text, row *AttributeRow) error {
+ func(_ ID, row *AttributeRow) error {
if row.Order > maxAttributeOrder {
maxAttributeOrder = row.Order
}
@@ -120,10 +124,9 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
case *engine.CommentDef:
// update comments
- row, err := m.relations.Get(tx, Text(req.TableID))
+ row, err := m.relations.Get(tx, req.TableID)
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrResp.ID = req.TableID
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
if err != nil {
return err
@@ -135,10 +138,9 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
case *engine.PartitionDef:
// update
- row, err := m.relations.Get(tx, Text(req.TableID))
+ row, err := m.relations.Get(tx, req.TableID)
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrResp.ID = req.TableID
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
if err != nil {
return err
@@ -150,10 +152,9 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
case *engine.ViewDef:
// update
- row, err := m.relations.Get(tx, Text(req.TableID))
+ row, err := m.relations.Get(tx, req.TableID)
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrResp.ID = req.TableID
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
if err != nil {
return err
@@ -166,21 +167,20 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
case *engine.AttributeDef:
// add attribute
// check existence
- keys, err := m.attributes.Index(tx, Tuple{
+ entries, err := m.attributes.Index(tx, Tuple{
index_RelationID_Name,
- Text(req.TableID),
+ req.TableID,
Text(def.Attr.Name),
})
if err != nil {
return err
}
- if len(keys) > 0 {
- resp.ErrResp.ErrExisted = true
- return nil
+ if len(entries) > 0 {
+ return moerr.NewDuplicate()
}
// insert
attrRow := AttributeRow{
- ID: uuid.NewString(),
+ ID: txnengine.NewID(),
RelationID: req.TableID,
Order: maxAttributeOrder + 1,
Nullable: true, //TODO fix
@@ -193,21 +193,20 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
case *engine.IndexTableDef:
// add index
// check existence
- keys, err := m.indexes.Index(tx, Tuple{
+ entries, err := m.indexes.Index(tx, Tuple{
index_RelationID_Name,
- Text(req.TableID),
+ req.TableID,
Text(def.Name),
})
if err != nil {
return err
}
- if len(keys) > 0 {
- resp.ErrResp.ErrExisted = true
- return nil
+ if len(entries) > 0 {
+ return moerr.NewDuplicate()
}
// insert
idxRow := IndexRow{
- ID: uuid.NewString(),
+ ID: txnengine.NewID(),
RelationID: req.TableID,
IndexTableDef: *def,
}
@@ -217,10 +216,9 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
case *engine.PropertiesDef:
// update properties
- row, err := m.relations.Get(tx, Text(req.TableID))
+ row, err := m.relations.Get(tx, req.TableID)
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrResp.ID = req.TableID
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
for _, prop := range def.Properties {
row.Properties[prop.Key] = prop.Value
@@ -233,7 +231,7 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
// set primary index
if err := m.iterRelationAttributes(
tx, req.TableID,
- func(_ Text, row *AttributeRow) error {
+ func(_ ID, row *AttributeRow) error {
isPrimary := false
for _, name := range def.Names {
if name == row.Name {
@@ -255,7 +253,7 @@ func (m *MemHandler) HandleAddTableDef(meta txn.TxnMeta, req txnengine.AddTableD
}
default:
- return fmt.Errorf("unknown table def: %T", req.Def)
+ panic(fmt.Sprintf("unknown table def: %T", req.Def))
}
@@ -267,8 +265,7 @@ func (m *MemHandler) HandleCloseTableIter(meta txn.TxnMeta, req txnengine.CloseT
defer m.iterators.Unlock()
iter, ok := m.iterators.Map[req.IterID]
if !ok {
- resp.ErrResp.ID = req.IterID
- return nil
+ return moerr.NewInternalError("no such iter: %v", req.IterID)
}
delete(m.iterators.Map, req.IterID)
if err := iter.TableIter.Close(); err != nil {
@@ -280,7 +277,7 @@ func (m *MemHandler) HandleCloseTableIter(meta txn.TxnMeta, req txnengine.CloseT
func (m *MemHandler) HandleCreateDatabase(meta txn.TxnMeta, req txnengine.CreateDatabaseReq, resp *txnengine.CreateDatabaseResp) error {
tx := m.getTx(meta)
- keys, err := m.databases.Index(tx, Tuple{
+ entries, err := m.databases.Index(tx, Tuple{
index_AccountID_Name,
Uint(req.AccessInfo.AccountID),
Text(req.Name),
@@ -288,15 +285,13 @@ func (m *MemHandler) HandleCreateDatabase(meta txn.TxnMeta, req txnengine.Create
if err != nil {
return err
}
- if len(keys) > 0 {
- resp.ErrResp.ErrExisted = true
- return nil
+ if len(entries) > 0 {
+ return moerr.NewDBAlreadyExists(req.Name)
}
- id := uuid.NewString()
+ id := txnengine.NewID()
err = m.databases.Insert(tx, DatabaseRow{
ID: id,
- NumberID: rand.Uint64(),
AccountID: req.AccessInfo.AccountID,
Name: req.Name,
})
@@ -312,11 +307,10 @@ func (m *MemHandler) HandleCreateRelation(meta txn.TxnMeta, req txnengine.Create
tx := m.getTx(meta)
// validate database id
- if req.DatabaseID != "" {
- _, err := m.databases.Get(tx, Text(req.DatabaseID))
+ if !req.DatabaseID.IsEmpty() {
+ _, err := m.databases.Get(tx, req.DatabaseID)
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrResp.ID = req.DatabaseID
- return nil
+ return moerr.NewNoDB()
}
if err != nil {
return err
@@ -324,23 +318,21 @@ func (m *MemHandler) HandleCreateRelation(meta txn.TxnMeta, req txnengine.Create
}
// check existence
- keys, err := m.relations.Index(tx, Tuple{
+ entries, err := m.relations.Index(tx, Tuple{
index_DatabaseID_Name,
- Text(req.DatabaseID),
+ req.DatabaseID,
Text(req.Name),
})
if err != nil {
return err
}
- if len(keys) > 0 {
- resp.ErrResp.ErrExisted = true
- return nil
+ if len(entries) > 0 {
+ return moerr.NewTableAlreadyExists(req.Name)
}
// row
row := RelationRow{
- ID: uuid.NewString(),
- NumberID: rand.Uint64(),
+ ID: txnengine.NewID(),
DatabaseID: req.DatabaseID,
Name: req.Name,
Type: req.Type,
@@ -378,7 +370,7 @@ func (m *MemHandler) HandleCreateRelation(meta txn.TxnMeta, req txnengine.Create
primaryColumnNames = def.Names
default:
- panic(fmt.Errorf("unknown table def: %T", def))
+ panic(fmt.Sprintf("unknown table def: %T", def))
}
}
@@ -406,7 +398,7 @@ func (m *MemHandler) HandleCreateRelation(meta txn.TxnMeta, req txnengine.Create
attr.Primary = isPrimary
}
attrRow := AttributeRow{
- ID: uuid.NewString(),
+ ID: txnengine.NewID(),
RelationID: row.ID,
Order: i + 1,
Nullable: true, //TODO fix
@@ -420,7 +412,7 @@ func (m *MemHandler) HandleCreateRelation(meta txn.TxnMeta, req txnengine.Create
// insert relation indexes
for _, idx := range relIndexes {
idxRow := IndexRow{
- ID: uuid.NewString(),
+ ID: txnengine.NewID(),
RelationID: row.ID,
IndexTableDef: idx,
}
@@ -446,10 +438,9 @@ func (m *MemHandler) HandleDelTableDef(meta txn.TxnMeta, req txnengine.DelTableD
case *engine.CommentDef:
// del comments
- row, err := m.relations.Get(tx, Text(req.TableID))
+ row, err := m.relations.Get(tx, req.TableID)
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrResp.ID = req.TableID
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
if err != nil {
return err
@@ -461,42 +452,42 @@ func (m *MemHandler) HandleDelTableDef(meta txn.TxnMeta, req txnengine.DelTableD
case *engine.AttributeDef:
// delete attribute
- keys, err := m.attributes.Index(tx, Tuple{
+ entries, err := m.attributes.Index(tx, Tuple{
index_RelationID_Name,
- Text(req.TableID),
+ req.TableID,
Text(def.Attr.Name),
})
if err != nil {
return err
}
- for _, key := range keys {
- if err := m.attributes.Delete(tx, key); err != nil {
+ for _, entry := range entries {
+ if err := m.attributes.Delete(tx, entry.Key); err != nil {
return err
}
+ //TODO update DataValue
}
case *engine.IndexTableDef:
// delete index
- keys, err := m.indexes.Index(tx, Tuple{
+ entries, err := m.indexes.Index(tx, Tuple{
index_RelationID_Name,
- Text(req.TableID),
+ req.TableID,
Text(def.Name),
})
if err != nil {
return err
}
- for _, key := range keys {
- if err := m.indexes.Delete(tx, key); err != nil {
+ for _, entry := range entries {
+ if err := m.indexes.Delete(tx, entry.Key); err != nil {
return err
}
}
case *engine.PropertiesDef:
// delete properties
- row, err := m.relations.Get(tx, Text(req.TableID))
+ row, err := m.relations.Get(tx, req.TableID)
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrResp.ID = req.TableID
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
for _, prop := range def.Properties {
delete(row.Properties, prop.Key)
@@ -509,7 +500,7 @@ func (m *MemHandler) HandleDelTableDef(meta txn.TxnMeta, req txnengine.DelTableD
// delete primary index
if err := m.iterRelationAttributes(
tx, req.TableID,
- func(key Text, row *AttributeRow) error {
+ func(key ID, row *AttributeRow) error {
if !row.Primary {
return nil
}
@@ -524,7 +515,7 @@ func (m *MemHandler) HandleDelTableDef(meta txn.TxnMeta, req txnengine.DelTableD
}
default:
- return fmt.Errorf("unknown table def: %T", req.Def)
+ panic(fmt.Sprintf("unknown table def: %T", req.Def))
}
@@ -540,19 +531,19 @@ func (m *MemHandler) HandleDelete(meta txn.TxnMeta, req txnengine.DeleteReq, res
for i := 0; i < reqVecLen; i++ {
value := vectorAt(req.Vector, i)
rowID := value.Value.(types.Rowid)
- keys, err := m.data.Index(tx, Tuple{
- index_RowID, typeConv(rowID),
+ entries, err := m.data.Index(tx, Tuple{
+ index_RowID, memtable.ToOrdered(rowID),
})
if err != nil {
return err
}
- if len(keys) == 0 {
+ if len(entries) == 0 {
continue
}
- if len(keys) != 1 {
+ if len(entries) != 1 {
panic("impossible")
}
- if err := m.data.Delete(tx, keys[0]); err != nil {
+ if err := m.data.Delete(tx, entries[0].Key); err != nil {
return err
}
}
@@ -560,21 +551,21 @@ func (m *MemHandler) HandleDelete(meta txn.TxnMeta, req txnengine.DeleteReq, res
}
// by primary keys
- rows, err := m.attributes.IndexRows(tx, Tuple{
+ entries, err := m.attributes.Index(tx, Tuple{
index_RelationID_IsPrimary,
- Text(req.TableID),
+ req.TableID,
Bool(true),
})
if err != nil {
return err
}
- if len(rows) == 1 && rows[0].Name == req.ColumnName {
+ if len(entries) == 1 && entries[0].Value.Name == req.ColumnName {
// by primary key
for i := 0; i < reqVecLen; i++ {
value := vectorAt(req.Vector, i)
key := DataKey{
tableID: req.TableID,
- primaryKey: Tuple{typeConv(value.Value)},
+ primaryKey: Tuple{memtable.ToOrdered(value.Value)},
}
if err := m.data.Delete(tx, key); err != nil {
return err
@@ -584,26 +575,25 @@ func (m *MemHandler) HandleDelete(meta txn.TxnMeta, req txnengine.DeleteReq, res
}
// by non-primary key, slow but works
- rows, err = m.attributes.IndexRows(tx, Tuple{
+ entries, err = m.attributes.Index(tx, Tuple{
index_RelationID_Name,
- Text(req.TableID),
+ req.TableID,
Text(req.ColumnName),
})
if err != nil {
return err
}
- if len(rows) != 1 {
- resp.ErrResp.Name = req.ColumnName
- return nil
+ if len(entries) != 1 {
+ return moerr.NewInternalError("wrong column name: %s", req.ColumnName)
}
- attrID := rows[0].ID
+ attrIndex := entries[0].Value.Order
iter := m.data.NewIter(tx)
defer iter.Close()
tableKey := DataKey{
tableID: req.TableID,
}
for ok := iter.Seek(tableKey); ok; ok = iter.Next() {
- key, row, err := iter.Read()
+ key, dataValue, err := iter.Read()
if err != nil {
return err
}
@@ -612,11 +602,11 @@ func (m *MemHandler) HandleDelete(meta txn.TxnMeta, req txnengine.DeleteReq, res
}
for i := 0; i < reqVecLen; i++ {
value := vectorAt(req.Vector, i)
- attrInRow, ok := (*row).attributes[attrID]
- if !ok {
+ if attrIndex >= len(*dataValue) {
// attr not in row
continue
}
+ attrInRow := (*dataValue)[attrIndex]
if value.Equal(attrInRow) {
if err := m.data.Delete(tx, key); err != nil {
return err
@@ -631,7 +621,7 @@ func (m *MemHandler) HandleDelete(meta txn.TxnMeta, req txnengine.DeleteReq, res
func (m *MemHandler) HandleDeleteDatabase(meta txn.TxnMeta, req txnengine.DeleteDatabaseReq, resp *txnengine.DeleteDatabaseResp) error {
tx := m.getTx(meta)
- rows, err := m.databases.IndexRows(tx, Tuple{
+ entries, err := m.databases.Index(tx, Tuple{
index_AccountID_Name,
Uint(req.AccessInfo.AccountID),
Text(req.Name),
@@ -639,53 +629,53 @@ func (m *MemHandler) HandleDeleteDatabase(meta txn.TxnMeta, req txnengine.Delete
if err != nil {
return err
}
- if len(rows) == 0 {
- resp.ErrResp.Name = req.Name
- return nil
+ if len(entries) == 0 {
+ return moerr.NewNoDB()
}
- for _, row := range rows {
- if err := m.databases.Delete(tx, row.Key()); err != nil {
+ for _, entry := range entries {
+ if err := m.databases.Delete(tx, entry.Key); err != nil {
return err
}
- if err := m.deleteRelationsByDBID(tx, row.ID); err != nil {
+ if err := m.deleteRelationsByDBID(tx, entry.Value.ID); err != nil {
return err
}
- resp.ID = row.ID
+ resp.ID = entry.Value.ID
}
return nil
}
-func (m *MemHandler) deleteRelationsByDBID(tx *Transaction, dbID string) error {
- rows, err := m.relations.IndexRows(tx, Tuple{
+func (m *MemHandler) deleteRelationsByDBID(tx *Transaction, dbID ID) error {
+ entries, err := m.relations.Index(tx, Tuple{
index_DatabaseID,
- Text(dbID),
+ dbID,
})
if err != nil {
return err
}
- for _, row := range rows {
- if err := m.relations.Delete(tx, row.Key()); err != nil {
+ for _, entry := range entries {
+ if err := m.relations.Delete(tx, entry.Key); err != nil {
return err
}
- if err := m.deleteAttributesByRelationID(tx, row.ID); err != nil {
+ if err := m.deleteAttributesByRelationID(tx, entry.Value.ID); err != nil {
return err
}
+ //TODO delete data
}
return nil
}
-func (m *MemHandler) deleteAttributesByRelationID(tx *Transaction, relationID string) error {
- keys, err := m.attributes.Index(tx, Tuple{
+func (m *MemHandler) deleteAttributesByRelationID(tx *Transaction, relationID ID) error {
+ entries, err := m.attributes.Index(tx, Tuple{
index_RelationID,
- Text(relationID),
+ relationID,
})
if err != nil {
return err
}
- for _, key := range keys {
- if err := m.attributes.Delete(tx, key); err != nil {
+ for _, entry := range entries {
+ if err := m.attributes.Delete(tx, entry.Key); err != nil {
return err
}
}
@@ -694,37 +684,37 @@ func (m *MemHandler) deleteAttributesByRelationID(tx *Transaction, relationID st
func (m *MemHandler) HandleDeleteRelation(meta txn.TxnMeta, req txnengine.DeleteRelationReq, resp *txnengine.DeleteRelationResp) error {
tx := m.getTx(meta)
- rows, err := m.relations.IndexRows(tx, Tuple{
+ entries, err := m.relations.Index(tx, Tuple{
index_DatabaseID_Name,
- Text(req.DatabaseID),
+ req.DatabaseID,
Text(req.Name),
})
if err != nil {
return err
}
- if len(rows) == 0 {
+ if len(entries) == 0 {
// the caller expects no error if table not exist
//resp.ErrNotFound.Name = req.Name
return nil
}
- if len(rows) != 1 {
+ if len(entries) != 1 {
panic("impossible")
}
- row := rows[0]
- if err := m.relations.Delete(tx, row.Key()); err != nil {
+ entry := entries[0]
+ if err := m.relations.Delete(tx, entry.Key); err != nil {
return err
}
- if err := m.deleteAttributesByRelationID(tx, row.ID); err != nil {
+ if err := m.deleteAttributesByRelationID(tx, entry.Value.ID); err != nil {
return err
}
- resp.ID = row.ID
+ resp.ID = entry.Value.ID
return nil
}
func (m *MemHandler) HandleGetDatabases(meta txn.TxnMeta, req txnengine.GetDatabasesReq, resp *txnengine.GetDatabasesResp) error {
tx := m.getTx(meta)
- rows, err := m.databases.IndexRows(tx, Tuple{
+ entries, err := m.databases.Index(tx, Tuple{
index_AccountID,
Uint(req.AccessInfo.AccountID),
})
@@ -732,8 +722,8 @@ func (m *MemHandler) HandleGetDatabases(meta txn.TxnMeta, req txnengine.GetDatab
return err
}
- for _, row := range rows {
- resp.Names = append(resp.Names, row.Name)
+ for _, entry := range entries {
+ resp.Names = append(resp.Names, entry.Value.Name)
}
return nil
@@ -741,31 +731,31 @@ func (m *MemHandler) HandleGetDatabases(meta txn.TxnMeta, req txnengine.GetDatab
func (m *MemHandler) HandleGetPrimaryKeys(meta txn.TxnMeta, req txnengine.GetPrimaryKeysReq, resp *txnengine.GetPrimaryKeysResp) error {
tx := m.getTx(meta)
- rows, err := m.attributes.IndexRows(tx, Tuple{
+ entries, err := m.attributes.Index(tx, Tuple{
index_RelationID_IsPrimary,
- Text(req.TableID),
+ req.TableID,
Bool(true),
})
if err != nil {
return err
}
- for _, row := range rows {
- resp.Attrs = append(resp.Attrs, &row.Attribute)
+ for _, entry := range entries {
+ resp.Attrs = append(resp.Attrs, &entry.Value.Attribute)
}
return nil
}
func (m *MemHandler) HandleGetRelations(meta txn.TxnMeta, req txnengine.GetRelationsReq, resp *txnengine.GetRelationsResp) error {
tx := m.getTx(meta)
- rows, err := m.relations.IndexRows(tx, Tuple{
+ entries, err := m.relations.Index(tx, Tuple{
index_DatabaseID,
- Text(req.DatabaseID),
+ req.DatabaseID,
})
if err != nil {
return err
}
- for _, row := range rows {
- resp.Names = append(resp.Names, row.Name)
+ for _, entry := range entries {
+ resp.Names = append(resp.Names, entry.Value.Name)
}
return nil
}
@@ -773,7 +763,7 @@ func (m *MemHandler) HandleGetRelations(meta txn.TxnMeta, req txnengine.GetRelat
func (m *MemHandler) HandleGetTableDefs(meta txn.TxnMeta, req txnengine.GetTableDefsReq, resp *txnengine.GetTableDefsResp) error {
tx := m.getTx(meta)
- relRow, err := m.relations.Get(tx, Text(req.TableID))
+ relRow, err := m.relations.Get(tx, req.TableID)
if errors.Is(err, sql.ErrNoRows) {
// the caller expects no error if table not exist
//resp.ErrTableNotFound.ID = req.TableID
@@ -810,7 +800,7 @@ func (m *MemHandler) HandleGetTableDefs(meta txn.TxnMeta, req txnengine.GetTable
var attrRows []*AttributeRow
if err := m.iterRelationAttributes(
tx, req.TableID,
- func(key Text, row *AttributeRow) error {
+ func(key ID, row *AttributeRow) error {
if row.IsHidden {
return nil
}
@@ -842,14 +832,14 @@ func (m *MemHandler) HandleGetTableDefs(meta txn.TxnMeta, req txnengine.GetTable
// indexes
{
- rows, err := m.indexes.IndexRows(tx, Tuple{
- index_RelationID, Text(req.TableID),
+ entries, err := m.indexes.Index(tx, Tuple{
+ index_RelationID, req.TableID,
})
if err != nil {
return err
}
- for _, row := range rows {
- resp.Defs = append(resp.Defs, &row.IndexTableDef)
+ for _, entry := range entries {
+ resp.Defs = append(resp.Defs, &entry.Value.IndexTableDef)
}
}
@@ -870,16 +860,16 @@ func (m *MemHandler) HandleGetTableDefs(meta txn.TxnMeta, req txnengine.GetTable
func (m *MemHandler) HandleGetHiddenKeys(meta txn.TxnMeta, req txnengine.GetHiddenKeysReq, resp *txnengine.GetHiddenKeysResp) error {
tx := m.getTx(meta)
- rows, err := m.attributes.IndexRows(tx, Tuple{
+ entries, err := m.attributes.Index(tx, Tuple{
index_RelationID_IsHidden,
- Text(req.TableID),
+ req.TableID,
Bool(true),
})
if err != nil {
return err
}
- for _, row := range rows {
- resp.Attrs = append(resp.Attrs, &row.Attribute)
+ for _, entry := range entries {
+ resp.Attrs = append(resp.Attrs, &entry.Value.Attribute)
}
return nil
}
@@ -891,7 +881,7 @@ func (m *MemHandler) HandleNewTableIter(meta txn.TxnMeta, req txnengine.NewTable
attrsMap := make(map[string]*AttributeRow)
if err := m.iterRelationAttributes(
tx, req.TableID,
- func(_ Text, row *AttributeRow) error {
+ func(_ ID, row *AttributeRow) error {
attrsMap[row.Name] = row
return nil
},
@@ -899,7 +889,7 @@ func (m *MemHandler) HandleNewTableIter(meta txn.TxnMeta, req txnengine.NewTable
return err
}
- iter := &Iter[DataKey, DataRow]{
+ iter := &Iter[DataKey, DataValue]{
TableIter: tableIter,
TableID: req.TableID,
AttrsMap: attrsMap,
@@ -910,11 +900,13 @@ func (m *MemHandler) HandleNewTableIter(meta txn.TxnMeta, req txnengine.NewTable
}
return tableIter.Seek(tableKey)
},
+ ReadTime: tx.Time,
+ Tx: tx,
}
m.iterators.Lock()
defer m.iterators.Unlock()
- id := uuid.NewString()
+ id := txnengine.NewID()
resp.IterID = id
m.iterators.Map[id] = iter
@@ -924,7 +916,7 @@ func (m *MemHandler) HandleNewTableIter(meta txn.TxnMeta, req txnengine.NewTable
func (m *MemHandler) HandleOpenDatabase(meta txn.TxnMeta, req txnengine.OpenDatabaseReq, resp *txnengine.OpenDatabaseResp) error {
tx := m.getTx(meta)
- rows, err := m.databases.IndexRows(tx, Tuple{
+ entries, err := m.databases.Index(tx, Tuple{
index_AccountID_Name,
Uint(req.AccessInfo.AccountID),
Text(req.Name),
@@ -933,31 +925,37 @@ func (m *MemHandler) HandleOpenDatabase(meta txn.TxnMeta, req txnengine.OpenData
return err
}
- for _, row := range rows {
- resp.ID = row.ID
+ for _, entry := range entries {
+ resp.ID = entry.Value.ID
+ resp.Name = entry.Value.Name
return nil
}
- resp.ErrResp.Name = req.Name
- return nil
+ return moerr.NewNoDB()
}
func (m *MemHandler) HandleOpenRelation(meta txn.TxnMeta, req txnengine.OpenRelationReq, resp *txnengine.OpenRelationResp) error {
tx := m.getTx(meta)
- rows, err := m.relations.IndexRows(tx, Tuple{
+ entries, err := m.relations.Index(tx, Tuple{
index_DatabaseID_Name,
- Text(req.DatabaseID),
+ req.DatabaseID,
Text(req.Name),
})
if err != nil {
return err
}
- for _, row := range rows {
- resp.ID = row.ID
- resp.Type = row.Type
- return nil
+ if len(entries) == 0 {
+ return moerr.NewNoSuchTable(req.DatabaseName, req.Name)
+ }
+ entry := entries[0]
+ resp.ID = entry.Value.ID
+ resp.Type = entry.Value.Type
+ resp.RelationName = entry.Value.Name
+ db, err := m.databases.Get(tx, entry.Value.DatabaseID)
+ if err != nil {
+ return err
}
- resp.ErrResp.Name = req.Name
+ resp.DatabaseName = db.Name
return nil
}
@@ -968,8 +966,7 @@ func (m *MemHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, resp *t
iter, ok := m.iterators.Map[req.IterID]
if !ok {
m.iterators.Unlock()
- resp.ErrResp.ID = req.IterID
- return nil
+ return moerr.NewInternalError("no such iter: %v", req.IterID)
}
m.iterators.Unlock()
@@ -987,18 +984,18 @@ func (m *MemHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, resp *t
maxRows := 4096
type Row struct {
- Value *DataRow
- PhysicalRow *PhysicalRow[DataKey, DataRow]
+ Value *DataValue
+ PhysicalRow *memtable.PhysicalRow[DataKey, DataValue]
}
var rows []Row
for ok := fn(); ok; ok = iter.TableIter.Next() {
item := iter.TableIter.Item()
- row, err := item.Values.Read(iter.TableIter.readTime, iter.TableIter.tx)
+ value, err := item.Read(iter.ReadTime, iter.Tx)
if err != nil {
return err
}
- if row.key.tableID != iter.TableID {
+ if item.Key.tableID != iter.TableID {
break
}
@@ -1008,7 +1005,7 @@ func (m *MemHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, resp *t
}
rows = append(rows, Row{
- Value: row,
+ Value: value,
PhysicalRow: item,
})
if len(rows) >= maxRows {
@@ -1026,7 +1023,7 @@ func (m *MemHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, resp *t
tx := m.getTx(meta)
for _, row := range rows {
namedRow := &NamedDataRow{
- Row: row.Value,
+ Value: *row.Value,
AttrsMap: iter.AttrsMap,
}
if err := appendNamedRow(tx, m.mheap, b, namedRow); err != nil {
@@ -1047,10 +1044,9 @@ func (m *MemHandler) HandleRead(meta txn.TxnMeta, req txnengine.ReadReq, resp *t
func (m *MemHandler) HandleTruncate(meta txn.TxnMeta, req txnengine.TruncateReq, resp *txnengine.TruncateResp) error {
tx := m.getTx(meta)
- _, err := m.relations.Get(tx, Text(req.TableID))
+ _, err := m.relations.Get(tx, req.TableID)
if errors.Is(err, sql.ErrNoRows) {
- resp.ErrResp.ID = req.TableID
- return nil
+ return moerr.NewNoSuchTable(req.DatabaseName, req.TableName)
}
iter := m.data.NewIter(tx)
defer iter.Close()
@@ -1078,8 +1074,9 @@ func (m *MemHandler) HandleUpdate(meta txn.TxnMeta, req txnengine.UpdateReq, res
if err := m.rangeBatchPhysicalRows(
tx,
req.TableID,
+ req.DatabaseName,
+ req.TableName,
req.Batch,
- &resp.ErrResp,
func(
row *DataRow,
rowID types.Rowid,
@@ -1102,8 +1099,9 @@ func (m *MemHandler) HandleWrite(meta txn.TxnMeta, req txnengine.WriteReq, resp
if err := m.rangeBatchPhysicalRows(
tx,
req.TableID,
+ req.DatabaseName,
+ req.TableName,
req.Batch,
- &resp.ErrResp,
func(
row *DataRow,
rowID types.Rowid,
@@ -1122,9 +1120,10 @@ func (m *MemHandler) HandleWrite(meta txn.TxnMeta, req txnengine.WriteReq, resp
func (m *MemHandler) rangeBatchPhysicalRows(
tx *Transaction,
- tableID string,
+ tableID ID,
+ dbName string,
+ tableName string,
b *batch.Batch,
- resp *txnengine.ErrorResp,
fn func(
*DataRow,
types.Rowid,
@@ -1135,7 +1134,7 @@ func (m *MemHandler) rangeBatchPhysicalRows(
nameToAttrs := make(map[string]*AttributeRow)
if err := m.iterRelationAttributes(
tx, tableID,
- func(_ Text, row *AttributeRow) error {
+ func(_ ID, row *AttributeRow) error {
nameToAttrs[row.Name] = row
return nil
},
@@ -1144,8 +1143,7 @@ func (m *MemHandler) rangeBatchPhysicalRows(
}
if len(nameToAttrs) == 0 {
- resp.ID = tableID
- return nil
+ return moerr.NewNoSuchTable(dbName, tableName)
}
// iter
@@ -1160,10 +1158,15 @@ func (m *MemHandler) rangeBatchPhysicalRows(
physicalRow := NewDataRow(
tableID,
[]Tuple{
- {index_RowID, typeConv(rowID)},
+ {index_RowID, memtable.ToOrdered(rowID)},
},
)
- physicalRow.attributes[nameToAttrs[rowIDColumnName].ID] = Nullable{
+ physicalRow.value = make(DataValue, 0, len(nameToAttrs))
+ idx := nameToAttrs[rowIDColumnName].Order
+ for idx >= len(physicalRow.value) {
+ physicalRow.value = append(physicalRow.value, Nullable{})
+ }
+ physicalRow.value[idx] = Nullable{
Value: rowID,
}
@@ -1172,19 +1175,29 @@ func (m *MemHandler) rangeBatchPhysicalRows(
attr, ok := nameToAttrs[name]
if !ok {
- return fmt.Errorf("unknown attr: %s", name)
+ panic(fmt.Sprintf("unknown attr: %s", name))
}
if attr.Primary {
- physicalRow.key.primaryKey = append(physicalRow.key.primaryKey, typeConv(col.Value))
+ physicalRow.key.primaryKey = append(
+ physicalRow.key.primaryKey,
+ memtable.ToOrdered(col.Value),
+ )
}
- physicalRow.attributes[attr.ID] = col
+ idx := attr.Order
+ for idx >= len(physicalRow.value) {
+ physicalRow.value = append(physicalRow.value, Nullable{})
+ }
+ physicalRow.value[idx] = col
}
// use row id as primary key if no primary key is provided
if len(physicalRow.key.primaryKey) == 0 {
- physicalRow.key.primaryKey = append(physicalRow.key.primaryKey, typeConv(rowID))
+ physicalRow.key.primaryKey = append(
+ physicalRow.key.primaryKey,
+ memtable.ToOrdered(rowID),
+ )
}
if err := fn(physicalRow, rowID); err != nil {
@@ -1202,7 +1215,7 @@ func (m *MemHandler) getTx(meta txn.TxnMeta) *Transaction {
defer m.transactions.Unlock()
tx, ok := m.transactions.Map[id]
if !ok {
- tx = NewTransaction(
+ tx = memtable.NewTransaction(
id,
Time{
Timestamp: meta.SnapshotTS,
@@ -1253,18 +1266,18 @@ func (m *MemHandler) HandleStartRecovery(ch chan txn.TxnMeta) {
func (m *MemHandler) iterRelationAttributes(
tx *Transaction,
- relationID string,
- fn func(key Text, row *AttributeRow) error,
+ relationID ID,
+ fn func(key ID, row *AttributeRow) error,
) error {
- rows, err := m.attributes.IndexRows(tx, Tuple{
+ entries, err := m.attributes.Index(tx, Tuple{
index_RelationID,
- Text(relationID),
+ relationID,
})
if err != nil {
return err
}
- for _, row := range rows {
- if err := fn(row.Key(), row); err != nil {
+ for _, entry := range entries {
+ if err := fn(entry.Key, entry.Value); err != nil {
return err
}
}
@@ -1295,3 +1308,12 @@ func (m *MemHandler) HandleTableStats(meta txn.TxnMeta, req txnengine.TableStats
return nil
}
+
+func newRowID() types.Rowid {
+ var rowid types.Rowid
+ err := binary.Read(crand.Reader, binary.LittleEndian, &rowid)
+ if err != nil {
+ panic(err)
+ }
+ return rowid
+}
diff --git a/pkg/txn/storage/txn/mem_handler_test.go b/pkg/txn/storage/txn/mem_handler_test.go
index 7ebd346f8e66934db37eccd7f73338f85b0a8b26..3c48055eeb5c09f7f09998dcdd574d2c59c06cd9 100644
--- a/pkg/txn/storage/txn/mem_handler_test.go
+++ b/pkg/txn/storage/txn/mem_handler_test.go
@@ -21,6 +21,7 @@ import (
"github.com/matrixorigin/matrixone/pkg/testutil"
"github.com/matrixorigin/matrixone/pkg/txn/clock"
+ "github.com/matrixorigin/matrixone/pkg/txn/storage/txn/memtable"
)
func TestMemHandler(t *testing.T) {
@@ -28,7 +29,7 @@ func TestMemHandler(t *testing.T) {
return New(
NewMemHandler(
testutil.NewMheap(),
- Serializable,
+ memtable.Serializable,
clock.NewHLCClock(func() int64 {
return time.Now().UnixNano()
}, math.MaxInt64),
diff --git a/pkg/txn/storage/txn/atomic.go b/pkg/txn/storage/txn/memtable/atomic.go
similarity index 97%
rename from pkg/txn/storage/txn/atomic.go
rename to pkg/txn/storage/txn/memtable/atomic.go
index 92cac6fd416bd7b918badb4664ae743abe338bea..7d7a3ec035037149b19f5ff8e2d54976a2da75a0 100644
--- a/pkg/txn/storage/txn/atomic.go
+++ b/pkg/txn/storage/txn/memtable/atomic.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
import "sync/atomic"
diff --git a/pkg/txn/storage/txn/memtable/bench_test.go b/pkg/txn/storage/txn/memtable/bench_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..a0a9524f6ee4a2c0671f397b24a7b4abfd2ca683
--- /dev/null
+++ b/pkg/txn/storage/txn/memtable/bench_test.go
@@ -0,0 +1,59 @@
+// Copyright 2022 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memtable
+
+import (
+ "testing"
+
+ "github.com/google/uuid"
+)
+
+func BenchmarkTable(b *testing.B) {
+ tx := NewTransaction(uuid.NewString(), Time{}, SnapshotIsolation)
+ table := NewTable[Int, int, TestRow]()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ key := Int(i)
+ tx.Time.Timestamp.PhysicalTime++
+ if err := table.Delete(tx, key); err != nil {
+ b.Fatal(err)
+ }
+ row := TestRow{
+ key: key,
+ value: i,
+ }
+ tx.Time.Timestamp.PhysicalTime++
+ if err := table.Insert(tx, row); err != nil {
+ b.Fatal(err)
+ }
+ tx.Time.Timestamp.PhysicalTime++
+ p, err := table.Get(tx, key)
+ if err != nil {
+ b.Fatal(err)
+ }
+ if *p != i {
+ b.Fatal()
+ }
+ entries, err := table.Index(tx, Tuple{
+ Text("foo"), Int(i),
+ })
+ if err != nil {
+ b.Fatal(err)
+ }
+ if len(entries) != 1 {
+ b.Fatal()
+ }
+ }
+}
diff --git a/pkg/txn/storage/txn/isolation.go b/pkg/txn/storage/txn/memtable/isolation.go
similarity index 97%
rename from pkg/txn/storage/txn/isolation.go
rename to pkg/txn/storage/txn/memtable/isolation.go
index f789ab571f6871d6b583c72293766482b63e52f0..54d78e8ec9ac352e927eefb0ad0183d332123ad6 100644
--- a/pkg/txn/storage/txn/isolation.go
+++ b/pkg/txn/storage/txn/memtable/isolation.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
type IsolationPolicy struct {
Read ReadPolicy
diff --git a/pkg/txn/storage/txn/nullable.go b/pkg/txn/storage/txn/memtable/nullable.go
similarity index 91%
rename from pkg/txn/storage/txn/nullable.go
rename to pkg/txn/storage/txn/memtable/nullable.go
index a786033d8c5015483436a43a8fe9715b2791bb50..bd19a78c5a144bd0fe440e062bcd771ac497f962 100644
--- a/pkg/txn/storage/txn/nullable.go
+++ b/pkg/txn/storage/txn/memtable/nullable.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
import (
"bytes"
@@ -34,7 +34,7 @@ func (n Nullable) Equal(n2 Nullable) bool {
if ok {
return bytes.Equal(bsA, bsB)
}
- panic(fmt.Errorf("type not the same: %T %T", n.Value, n2.Value))
+ panic(fmt.Sprintf("type not the same: %T %T", n.Value, n2.Value))
}
return n.Value == n2.Value
}
diff --git a/pkg/txn/storage/txn/memtable/physical_iter.go b/pkg/txn/storage/txn/memtable/physical_iter.go
new file mode 100644
index 0000000000000000000000000000000000000000..cac18516f1d7eff079c063947b8ea297a424b24a
--- /dev/null
+++ b/pkg/txn/storage/txn/memtable/physical_iter.go
@@ -0,0 +1,35 @@
+// Copyright 2022 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memtable
+
+import "github.com/tidwall/btree"
+
+type TablePhysicalIter[
+ K Ordered[K],
+ V any,
+] struct {
+ btree.GenericIter[*PhysicalRow[K, V]]
+}
+
+func (t *Table[K, V, R]) NewPhysicalIter() *TablePhysicalIter[K, V] {
+ return &TablePhysicalIter[K, V]{
+ GenericIter: t.rows.Iter(),
+ }
+}
+
+func (t *TablePhysicalIter[K, V]) Close() error {
+ t.GenericIter.Release()
+ return nil
+}
diff --git a/pkg/txn/storage/txn/mvcc.go b/pkg/txn/storage/txn/memtable/physical_row.go
similarity index 60%
rename from pkg/txn/storage/txn/mvcc.go
rename to pkg/txn/storage/txn/memtable/physical_row.go
index 5af7448a744cb5ae0374aae9160a40d672be8163..4cbaeee4b082679c7741e05f84ff43a2ca9272c9 100644
--- a/pkg/txn/storage/txn/mvcc.go
+++ b/pkg/txn/storage/txn/memtable/physical_row.go
@@ -12,24 +12,34 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
import (
"database/sql"
"fmt"
"io"
"sync"
+ "time"
"github.com/matrixorigin/matrixone/pkg/common/moerr"
+ txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
)
-type MVCC[T any] struct {
- //TODO use lock-free linked list
- sync.RWMutex
- Values []*MVCCValue[T]
+type PhysicalRow[
+ K Ordered[K],
+ V any,
+] struct {
+ Key K
+ LastUpdate *Atomic[time.Time]
+ Versions struct {
+ sync.RWMutex
+ List []*Version[V]
+ //TODO version GC
+ }
}
-type MVCCValue[T any] struct {
+type Version[T any] struct {
+ ID ID
BornTx *Transaction
BornTime Time
LockTx *Transaction
@@ -39,15 +49,23 @@ type MVCCValue[T any] struct {
// Read reads the visible value from Values
// readTime's logical time should be monotonically increasing in one transaction to reflect commands order
-func (m *MVCC[T]) Read(now Time, tx *Transaction) (*T, error) {
+func (p *PhysicalRow[K, V]) Read(now Time, tx *Transaction) (value *V, err error) {
+ version, err := p.readVersion(now, tx)
+ if version != nil {
+ value = version.Value
+ }
+ return
+}
+
+func (p *PhysicalRow[K, V]) readVersion(now Time, tx *Transaction) (*Version[V], error) {
if tx.State.Load() != Active {
panic("should not call Read")
}
- m.RLock()
- defer m.RUnlock()
- for i := len(m.Values) - 1; i >= 0; i-- {
- value := m.Values[i]
+ p.Versions.RLock()
+ defer p.Versions.RUnlock()
+ for i := len(p.Versions.List) - 1; i >= 0; i-- {
+ value := p.Versions.List[i]
if value.Visible(now, tx.ID) {
switch tx.IsolationPolicy.Read {
case ReadCommitted:
@@ -59,10 +77,10 @@ func (m *MVCC[T]) Read(now Time, tx *Transaction) (*T, error) {
case ReadNoStale:
// BornTx must be committed to be visible here
if value.BornTx.ID != tx.ID && value.BornTime.After(tx.BeginTime) {
- return value.Value, moerr.NewTxnReadConflict("%s %s", tx.ID, value.BornTx.ID)
+ return value, moerr.NewTxnReadConflict("%s %s", tx.ID, value.BornTx.ID)
}
}
- return value.Value, nil
+ return value, nil
}
}
@@ -70,15 +88,15 @@ func (m *MVCC[T]) Read(now Time, tx *Transaction) (*T, error) {
}
// ReadVisible reads a committed value despite the tx's isolation policy
-func (m *MVCC[T]) ReadVisible(now Time, tx *Transaction) (*MVCCValue[T], error) {
+func (p *PhysicalRow[K, V]) ReadVisible(now Time, tx *Transaction) (*Version[V], error) {
if tx.State.Load() != Active {
panic("should not call Read")
}
- m.RLock()
- defer m.RUnlock()
- for i := len(m.Values) - 1; i >= 0; i-- {
- value := m.Values[i]
+ p.Versions.RLock()
+ defer p.Versions.RUnlock()
+ for i := len(p.Versions.List) - 1; i >= 0; i-- {
+ value := p.Versions.List[i]
if value.Visible(now, tx.ID) {
return value, nil
}
@@ -87,39 +105,39 @@ func (m *MVCC[T]) ReadVisible(now Time, tx *Transaction) (*MVCCValue[T], error)
return nil, sql.ErrNoRows
}
-func (m *MVCCValue[T]) Visible(now Time, txID string) bool {
+func (v *Version[T]) Visible(now Time, txID string) bool {
// the following algorithm is from https://momjian.us/main/writings/pgsql/mvcc.pdf
// "[Mike Olson] says 17 march 1993: the tests in this routine are correct; if you think they’re not, you’re wrongand you should think about it again. i know, it happened to me."
// inserted by current tx
- if m.BornTx.ID == txID {
+ if v.BornTx.ID == txID {
// inserted before the read time
- if m.BornTime.Before(now) {
+ if v.BornTime.Before(now) {
// not been deleted
- if m.LockTx == nil {
+ if v.LockTx == nil {
return true
}
// deleted by current tx after the read time
- if m.LockTx.ID == txID && m.LockTime.After(now) {
+ if v.LockTx.ID == txID && v.LockTime.After(now) {
return true
}
}
}
// inserted by a committed tx
- if m.BornTx.State.Load() == Committed {
+ if v.BornTx.State.Load() == Committed {
// not been deleted
- if m.LockTx == nil {
+ if v.LockTx == nil {
// for isolation levels stricter than read-committed, instead of checking timestamps here, let the caller do it.
return true
}
// being deleted by current tx after the read time
- if m.LockTx.ID == txID && m.LockTime.After(now) {
+ if v.LockTx.ID == txID && v.LockTime.After(now) {
return true
}
// deleted by another tx but not committed
- if m.LockTx.ID != txID && m.LockTx.State.Load() != Committed {
+ if v.LockTx.ID != txID && v.LockTx.State.Load() != Committed {
return true
}
}
@@ -127,16 +145,16 @@ func (m *MVCCValue[T]) Visible(now Time, txID string) bool {
return false
}
-func (m *MVCC[T]) Insert(now Time, tx *Transaction, value *T) error {
+func (p *PhysicalRow[K, V]) Insert(now Time, tx *Transaction, value *V, callbacks ...any) error {
if tx.State.Load() != Active {
panic("should not call Insert")
}
- m.Lock()
- defer m.Unlock()
+ p.Versions.Lock()
+ defer p.Versions.Unlock()
- for i := len(m.Values) - 1; i >= 0; i-- {
- value := m.Values[i]
+ for i := len(p.Versions.List) - 1; i >= 0; i-- {
+ value := p.Versions.List[i]
if value.Visible(now, tx.ID) {
if value.LockTx != nil && value.LockTx.State.Load() != Aborted {
// locked by active or committed tx
@@ -148,25 +166,36 @@ func (m *MVCC[T]) Insert(now Time, tx *Transaction, value *T) error {
}
}
- m.Values = append(m.Values, &MVCCValue[T]{
+ id := txnengine.NewID()
+ p.Versions.List = append(p.Versions.List, &Version[V]{
+ ID: id,
BornTx: tx,
BornTime: now,
Value: value,
})
+ for _, callback := range callbacks {
+ switch callback := callback.(type) {
+ case func(ID): // version id
+ callback(id)
+ default:
+ panic(fmt.Sprintf("unknown type: %T", callback))
+ }
+ }
+
return nil
}
-func (m *MVCC[T]) Delete(now Time, tx *Transaction) error {
+func (p *PhysicalRow[K, V]) Delete(now Time, tx *Transaction) error {
if tx.State.Load() != Active {
panic("should not call Delete")
}
- m.Lock()
- defer m.Unlock()
+ p.Versions.Lock()
+ defer p.Versions.Unlock()
- for i := len(m.Values) - 1; i >= 0; i-- {
- value := m.Values[i]
+ for i := len(p.Versions.List) - 1; i >= 0; i-- {
+ value := p.Versions.List[i]
if value.Visible(now, tx.ID) {
if value.LockTx != nil && value.LockTx.State.Load() != Aborted {
return moerr.NewTxnWriteConflict("%s %s", tx.ID, value.LockTx.ID)
@@ -183,30 +212,45 @@ func (m *MVCC[T]) Delete(now Time, tx *Transaction) error {
return sql.ErrNoRows
}
-func (m *MVCC[T]) Update(now Time, tx *Transaction, newValue *T) error {
+func (p *PhysicalRow[K, V]) Update(now Time, tx *Transaction, newValue *V, callbacks ...any) error {
if tx.State.Load() != Active {
panic("should not call Update")
}
- m.Lock()
- defer m.Unlock()
+ p.Versions.Lock()
+ defer p.Versions.Unlock()
- for i := len(m.Values) - 1; i >= 0; i-- {
- value := m.Values[i]
+ for i := len(p.Versions.List) - 1; i >= 0; i-- {
+ value := p.Versions.List[i]
if value.Visible(now, tx.ID) {
+
if value.LockTx != nil && value.LockTx.State.Load() != Aborted {
return moerr.NewTxnWriteConflict("%s %s", tx.ID, value.LockTx.ID)
}
+
if value.BornTx.ID != tx.ID && value.BornTime.After(tx.BeginTime) {
return moerr.NewTxnWriteConflict("%s %s", tx.ID, value.BornTx.ID)
}
+
value.LockTx = tx
value.LockTime = now
- m.Values = append(m.Values, &MVCCValue[T]{
+ id := txnengine.NewID()
+ p.Versions.List = append(p.Versions.List, &Version[V]{
+ ID: id,
BornTx: tx,
BornTime: now,
Value: newValue,
})
+
+ for _, callback := range callbacks {
+ switch callback := callback.(type) {
+ case func(ID): // version id
+ callback(id)
+ default:
+ panic(fmt.Sprintf("unknown type: %T", callback))
+ }
+ }
+
return nil
}
}
@@ -214,8 +258,10 @@ func (m *MVCC[T]) Update(now Time, tx *Transaction, newValue *T) error {
return sql.ErrNoRows
}
-func (m *MVCC[T]) dump(w io.Writer) {
- for _, value := range m.Values {
+func (p *PhysicalRow[K, V]) dump(w io.Writer) {
+ p.Versions.RLock()
+ defer p.Versions.RUnlock()
+ for _, value := range p.Versions.List {
fmt.Fprintf(w, "born tx %s, born time %s, value %v",
value.BornTx.ID,
value.BornTime.String(),
diff --git a/pkg/txn/storage/txn/mvcc_test.go b/pkg/txn/storage/txn/memtable/physical_row_test.go
similarity index 68%
rename from pkg/txn/storage/txn/mvcc_test.go
rename to pkg/txn/storage/txn/memtable/physical_row_test.go
index 075e8f5aca81f74a10e0bdc347133625e0eec0df..237be2648e35f8f24fce49526a995bfb283b8f14 100644
--- a/pkg/txn/storage/txn/mvcc_test.go
+++ b/pkg/txn/storage/txn/memtable/physical_row_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
import (
"database/sql"
@@ -25,13 +25,13 @@ import (
"github.com/stretchr/testify/assert"
)
-func testMVCC(
+func testPhysicalRow(
t *testing.T,
isolationPolicy IsolationPolicy,
) {
// new
- m := new(MVCC[int])
+ m := new(PhysicalRow[Int, int])
m.dump(io.Discard)
// time
@@ -53,20 +53,20 @@ func testMVCC(
n := 1
err := m.Insert(now, tx1, &n)
assert.Nil(t, err)
- assert.Equal(t, 1, len(m.Values))
- assert.Equal(t, tx1, m.Values[0].BornTx)
- assert.Equal(t, now, m.Values[0].BornTime)
- assert.Nil(t, m.Values[0].LockTx)
- assert.True(t, m.Values[0].LockTime.IsZero())
+ assert.Equal(t, 1, len(m.Versions.List))
+ assert.Equal(t, tx1, m.Versions.List[0].BornTx)
+ assert.Equal(t, now, m.Versions.List[0].BornTime)
+ assert.Nil(t, m.Versions.List[0].LockTx)
+ assert.True(t, m.Versions.List[0].LockTime.IsZero())
n2 := 2
err = m.Insert(now, tx2, &n2)
assert.Nil(t, err)
- assert.Equal(t, 2, len(m.Values))
- assert.Equal(t, tx2, m.Values[1].BornTx)
- assert.Equal(t, now, m.Values[1].BornTime)
- assert.Nil(t, m.Values[1].LockTx)
- assert.True(t, m.Values[1].LockTime.IsZero())
+ assert.Equal(t, 2, len(m.Versions.List))
+ assert.Equal(t, tx2, m.Versions.List[1].BornTx)
+ assert.Equal(t, now, m.Versions.List[1].BornTime)
+ assert.Nil(t, m.Versions.List[1].LockTx)
+ assert.True(t, m.Versions.List[1].LockTime.IsZero())
// not readable now
res, err := m.Read(now, tx1)
@@ -93,9 +93,9 @@ func testMVCC(
// delete
err = m.Delete(now, tx1)
assert.Nil(t, err)
- assert.Equal(t, 2, len(m.Values))
- assert.Equal(t, tx1, m.Values[0].LockTx)
- assert.Equal(t, now, m.Values[0].LockTime)
+ assert.Equal(t, 2, len(m.Versions.List))
+ assert.Equal(t, tx1, m.Versions.List[0].LockTx)
+ assert.Equal(t, now, m.Versions.List[0].LockTime)
// not readable now by current tx
res, err = m.Read(now, tx1)
@@ -110,9 +110,9 @@ func testMVCC(
err = m.Delete(now, tx2)
assert.Nil(t, err)
- assert.Equal(t, 2, len(m.Values))
- assert.Equal(t, tx2, m.Values[1].LockTx)
- assert.Equal(t, now, m.Values[1].LockTime)
+ assert.Equal(t, 2, len(m.Versions.List))
+ assert.Equal(t, tx2, m.Versions.List[1].LockTx)
+ assert.Equal(t, now, m.Versions.List[1].LockTime)
res, err = m.Read(now, tx2)
assert.Equal(t, sql.ErrNoRows, err)
@@ -124,20 +124,20 @@ func testMVCC(
n3 := 3
err = m.Insert(now, tx1, &n3)
assert.Nil(t, err)
- assert.Equal(t, 3, len(m.Values))
- assert.Equal(t, tx1, m.Values[2].BornTx)
- assert.Equal(t, now, m.Values[2].BornTime)
- assert.Nil(t, m.Values[2].LockTx)
- assert.True(t, m.Values[2].LockTime.IsZero())
+ assert.Equal(t, 3, len(m.Versions.List))
+ assert.Equal(t, tx1, m.Versions.List[2].BornTx)
+ assert.Equal(t, now, m.Versions.List[2].BornTime)
+ assert.Nil(t, m.Versions.List[2].LockTx)
+ assert.True(t, m.Versions.List[2].LockTime.IsZero())
n4 := 4
err = m.Insert(now, tx2, &n4)
assert.Nil(t, err)
- assert.Equal(t, 4, len(m.Values))
- assert.Equal(t, tx2, m.Values[3].BornTx)
- assert.Equal(t, now, m.Values[3].BornTime)
- assert.Nil(t, m.Values[3].LockTx)
- assert.True(t, m.Values[3].LockTime.IsZero())
+ assert.Equal(t, 4, len(m.Versions.List))
+ assert.Equal(t, tx2, m.Versions.List[3].BornTx)
+ assert.Equal(t, now, m.Versions.List[3].BornTime)
+ assert.Nil(t, m.Versions.List[3].LockTx)
+ assert.True(t, m.Versions.List[3].LockTime.IsZero())
tick()
@@ -145,13 +145,13 @@ func testMVCC(
n5 := 5
err = m.Update(now, tx1, &n5)
assert.Nil(t, err)
- assert.Equal(t, 5, len(m.Values))
- assert.Equal(t, tx1, m.Values[2].LockTx)
- assert.Equal(t, now, m.Values[2].LockTime)
- assert.Equal(t, tx1, m.Values[4].BornTx)
- assert.Equal(t, now, m.Values[4].BornTime)
- assert.Nil(t, m.Values[4].LockTx)
- assert.True(t, m.Values[4].LockTime.IsZero())
+ assert.Equal(t, 5, len(m.Versions.List))
+ assert.Equal(t, tx1, m.Versions.List[2].LockTx)
+ assert.Equal(t, now, m.Versions.List[2].LockTime)
+ assert.Equal(t, tx1, m.Versions.List[4].BornTx)
+ assert.Equal(t, now, m.Versions.List[4].BornTime)
+ assert.Nil(t, m.Versions.List[4].LockTx)
+ assert.True(t, m.Versions.List[4].LockTime.IsZero())
// commit tx1
err = tx1.Commit()
@@ -176,7 +176,7 @@ func testMVCC(
assert.Equal(t, 5, *res)
assert.True(t, moerr.IsMoErrCode(err, moerr.ErrTxnReadConflict))
default:
- panic(fmt.Errorf("not handle: %v", isolationPolicy.Read))
+ panic(fmt.Sprintf("not handle: %v", isolationPolicy.Read))
}
// write stale conflict
@@ -217,18 +217,18 @@ func testMVCC(
assert.True(t, moerr.IsMoErrCode(err, moerr.ErrTxnWriteConflict))
}
-func TestMVCC(t *testing.T) {
+func TestPhysicalRow(t *testing.T) {
t.Run("read committed", func(t *testing.T) {
- testMVCC(t, IsolationPolicy{
+ testPhysicalRow(t, IsolationPolicy{
Read: ReadCommitted,
})
})
t.Run("snapshot isolation", func(t *testing.T) {
- testMVCC(t, SnapshotIsolation)
+ testPhysicalRow(t, SnapshotIsolation)
})
t.Run("serializable", func(t *testing.T) {
- testMVCC(t, Serializable)
+ testPhysicalRow(t, Serializable)
})
}
diff --git a/pkg/txn/storage/txn/memtable/table.go b/pkg/txn/storage/txn/memtable/table.go
new file mode 100644
index 0000000000000000000000000000000000000000..2ab532be30fd5e0a3e4752255c74f2fe3c267281
--- /dev/null
+++ b/pkg/txn/storage/txn/memtable/table.go
@@ -0,0 +1,331 @@
+// Copyright 2022 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memtable
+
+import (
+ "database/sql"
+ "errors"
+ "sync"
+ "time"
+
+ "github.com/matrixorigin/matrixone/pkg/common/moerr"
+ "github.com/tidwall/btree"
+)
+
+type Table[
+ K Ordered[K],
+ V any,
+ R Row[K, V],
+] struct {
+ sync.Mutex
+ rows *btree.BTreeG[*PhysicalRow[K, V]]
+ rowsHint *btree.PathHint
+ index *btree.BTreeG[*IndexEntry[K, V]]
+ indexHint *btree.PathHint
+ writeSets map[*Transaction]map[*PhysicalRow[K, V]]struct{}
+}
+
+type Row[K any, V any] interface {
+ Key() K
+ Value() V
+ Indexes() []Tuple
+}
+
+type NamedRow interface {
+ AttrByName(tx *Transaction, name string) (Nullable, error)
+}
+
+type Ordered[To any] interface {
+ Less(to To) bool
+}
+
+type IndexEntry[
+ K Ordered[K],
+ V any,
+] struct {
+ Index Tuple
+ Key K
+ VersionID ID
+ Value *V
+}
+
+func NewTable[
+ K Ordered[K],
+ V any,
+ R Row[K, V],
+]() *Table[K, V, R] {
+ return &Table[K, V, R]{
+ rows: btree.NewBTreeG(func(a, b *PhysicalRow[K, V]) bool {
+ return a.Key.Less(b.Key)
+ }),
+ rowsHint: new(btree.PathHint),
+ index: btree.NewBTreeG(func(a, b *IndexEntry[K, V]) bool {
+ if a.Index.Less(b.Index) {
+ return true
+ }
+ if b.Index.Less(a.Index) {
+ return false
+ }
+ if a.Key.Less(b.Key) {
+ return true
+ }
+ if b.Key.Less(a.Key) {
+ return false
+ }
+ return a.VersionID.Less(b.VersionID)
+ }),
+ indexHint: new(btree.PathHint),
+ writeSets: make(map[*Transaction]map[*PhysicalRow[K, V]]struct{}),
+ }
+}
+
+func (t *Table[K, V, R]) Insert(
+ tx *Transaction,
+ row R,
+) error {
+ key := row.Key()
+ physicalRow := t.getOrSetRowByKey(key)
+
+ existed, err := physicalRow.ReadVisible(tx.Time, tx)
+ if errors.Is(err, sql.ErrNoRows) {
+ err = nil
+ }
+ if err != nil {
+ return err
+ }
+ if existed != nil {
+ return moerr.NewPrimaryKeyDuplicated(key)
+ }
+
+ value := row.Value()
+ if err := physicalRow.Insert(
+ tx.Time, tx, &value,
+ func(versionID ID) {
+ for _, index := range row.Indexes() {
+ t.index.SetHint(&IndexEntry[K, V]{
+ Index: index,
+ Key: key,
+ VersionID: versionID,
+ Value: &value,
+ }, t.indexHint)
+ }
+ },
+ ); err != nil {
+ return err
+ }
+ physicalRow.LastUpdate.Store(time.Now())
+
+ t.setCommitter(tx, physicalRow)
+
+ tx.Time.Tick()
+ return nil
+}
+
+func (t *Table[K, V, R]) Update(
+ tx *Transaction,
+ row R,
+) error {
+ key := row.Key()
+ physicalRow := t.getOrSetRowByKey(key)
+
+ value := row.Value()
+ if err := physicalRow.Update(
+ tx.Time, tx, &value,
+ func(versionID ID) {
+ for _, index := range row.Indexes() {
+ t.index.SetHint(&IndexEntry[K, V]{
+ Index: index,
+ Key: key,
+ VersionID: versionID,
+ Value: &value,
+ }, t.indexHint)
+ }
+ },
+ ); err != nil {
+ return err
+ }
+ physicalRow.LastUpdate.Store(time.Now())
+
+ t.setCommitter(tx, physicalRow)
+
+ tx.Time.Tick()
+ return nil
+}
+
+func (t *Table[K, V, R]) Delete(
+ tx *Transaction,
+ key K,
+) error {
+ physicalRow := t.getRowByKey(key)
+
+ if physicalRow == nil {
+ return nil
+ }
+ if err := physicalRow.Delete(tx.Time, tx); err != nil {
+ return err
+ }
+ physicalRow.LastUpdate.Store(time.Now())
+
+ t.setCommitter(tx, physicalRow)
+
+ tx.Time.Tick()
+ return nil
+}
+
+func (t *Table[K, V, R]) Get(
+ tx *Transaction,
+ key K,
+) (
+ value *V,
+ err error,
+) {
+ physicalRow := t.getRowByKey(key)
+ if physicalRow == nil {
+ err = sql.ErrNoRows
+ return
+ }
+ value, err = physicalRow.Read(tx.Time, tx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (t *Table[K, V, R]) getRowByKey(key K) *PhysicalRow[K, V] {
+ pivot := &PhysicalRow[K, V]{
+ Key: key,
+ }
+ row, _ := t.rows.GetHint(pivot, t.rowsHint)
+ return row
+}
+
+func (t *Table[K, V, R]) getOrSetRowByKey(key K) *PhysicalRow[K, V] {
+ pivot := &PhysicalRow[K, V]{
+ Key: key,
+ }
+ if row, _ := t.rows.GetHint(pivot, t.rowsHint); row != nil {
+ return row
+ }
+ return t.getOrSetRowByKeySlow(pivot)
+}
+
+func (t *Table[K, V, R]) getOrSetRowByKeySlow(pivot *PhysicalRow[K, V]) *PhysicalRow[K, V] {
+ t.Lock()
+ defer t.Unlock()
+ if row, _ := t.rows.GetHint(pivot, t.rowsHint); row != nil {
+ return row
+ }
+ pivot.LastUpdate = NewAtomic(time.Now())
+ t.rows.SetHint(pivot, t.rowsHint)
+ return pivot
+}
+
+func (t *Table[K, V, R]) Index(tx *Transaction, index Tuple) (entries []*IndexEntry[K, V], err error) {
+ pivot := &IndexEntry[K, V]{
+ Index: index,
+ }
+ iter := t.index.Copy().Iter()
+ defer iter.Release()
+ for ok := iter.Seek(pivot); ok; ok = iter.Next() {
+ item := iter.Item()
+ if index.Less(item.Index) {
+ break
+ }
+ if item.Index.Less(index) {
+ break
+ }
+
+ physicalRow := t.getRowByKey(item.Key)
+ if physicalRow == nil {
+ continue
+ }
+ currentVersion, err := physicalRow.readVersion(tx.Time, tx)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ continue
+ }
+ return nil, err
+ }
+ if currentVersion.ID == item.VersionID {
+ entries = append(entries, item)
+ }
+ }
+ return
+}
+
+func (t *Table[K, V, R]) setCommitter(tx *Transaction, row *PhysicalRow[K, V]) {
+ tx.committers[t] = struct{}{}
+ t.Lock()
+ defer t.Unlock()
+ set, ok := t.writeSets[tx]
+ if !ok {
+ set = make(map[*PhysicalRow[K, V]]struct{})
+ t.writeSets[tx] = set
+ }
+ set[row] = struct{}{}
+}
+
+func (t *Table[K, V, R]) CommitTx(tx *Transaction) error {
+ t.Lock()
+ set := t.writeSets[tx]
+ t.Unlock()
+ defer func() {
+ t.Lock()
+ delete(t.writeSets, tx)
+ t.Unlock()
+ }()
+
+ for physicalRow := range set {
+
+ // verify the latest committed operation is done by the tx
+ var err error
+ physicalRow.Versions.RLock()
+ for i := len(physicalRow.Versions.List) - 1; i >= 0; i-- {
+ version := physicalRow.Versions.List[i]
+
+ // locked by another committed tx after tx begin
+ if version.LockTx != nil &&
+ version.LockTx.State.Load() == Committed &&
+ version.LockTx.ID != tx.ID &&
+ version.LockTime.After(tx.BeginTime) {
+ err = moerr.NewPrimaryKeyDuplicated(physicalRow.Key)
+ break
+ }
+
+ // born in another committed tx after tx begin
+ if version.BornTx.State.Load() == Committed &&
+ version.BornTx.ID != tx.ID &&
+ version.BornTime.After(tx.BeginTime) {
+ err = moerr.NewPrimaryKeyDuplicated(physicalRow.Key)
+ break
+ }
+
+ }
+ physicalRow.Versions.RUnlock()
+
+ if err != nil {
+ return err
+ }
+
+ }
+
+ return nil
+}
+
+func (t *Table[K, V, R]) AbortTx(tx *Transaction) {
+ t.Lock()
+ delete(t.writeSets, tx)
+ t.Unlock()
+}
diff --git a/pkg/txn/storage/txn/table_iter.go b/pkg/txn/storage/txn/memtable/table_iter.go
similarity index 68%
rename from pkg/txn/storage/txn/table_iter.go
rename to pkg/txn/storage/txn/memtable/table_iter.go
index 50a2ba2b2017827239ca87472ddc110a4933e08d..68e5d2bfa3a379c88ba2bcae9dbb5162956d9ca0 100644
--- a/pkg/txn/storage/txn/table_iter.go
+++ b/pkg/txn/storage/txn/memtable/table_iter.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
import (
"github.com/tidwall/btree"
@@ -20,19 +20,19 @@ import (
type TableIter[
K Ordered[K],
- R Row[K],
+ V any,
] struct {
tx *Transaction
- iter btree.GenericIter[*PhysicalRow[K, R]]
+ iter btree.GenericIter[*PhysicalRow[K, V]]
readTime Time
}
-func (t *Table[K, R]) NewIter(
+func (t *Table[K, V, R]) NewIter(
tx *Transaction,
) (
- iter *TableIter[K, R],
+ iter *TableIter[K, V],
) {
- iter = &TableIter[K, R]{
+ iter = &TableIter[K, V]{
tx: tx,
iter: t.rows.Copy().Iter(),
readTime: tx.Time,
@@ -40,27 +40,27 @@ func (t *Table[K, R]) NewIter(
return
}
-func (t *TableIter[K, R]) Read() (key K, row *R, err error) {
+func (t *TableIter[K, V]) Read() (key K, value *V, err error) {
physicalRow := t.iter.Item()
key = physicalRow.Key
- row, err = physicalRow.Values.Read(t.readTime, t.tx)
+ value, err = physicalRow.Read(t.readTime, t.tx)
if err != nil {
return
}
return
}
-func (t *TableIter[K, R]) Item() (row *PhysicalRow[K, R]) {
+func (t *TableIter[K, V]) Item() (row *PhysicalRow[K, V]) {
return t.iter.Item()
}
-func (t *TableIter[K, R]) Next() bool {
+func (t *TableIter[K, V]) Next() bool {
for {
if ok := t.iter.Next(); !ok {
return false
}
// skip unreadable values
- value, _ := t.iter.Item().Values.Read(t.readTime, t.tx)
+ value, _ := t.iter.Item().Read(t.readTime, t.tx)
if value == nil {
continue
}
@@ -68,13 +68,13 @@ func (t *TableIter[K, R]) Next() bool {
}
}
-func (t *TableIter[K, R]) First() bool {
+func (t *TableIter[K, V]) First() bool {
if ok := t.iter.First(); !ok {
return false
}
for {
// skip unreadable values
- value, _ := t.iter.Item().Values.Read(t.readTime, t.tx)
+ value, _ := t.iter.Item().Read(t.readTime, t.tx)
if value == nil {
if ok := t.iter.Next(); !ok {
return false
@@ -85,8 +85,8 @@ func (t *TableIter[K, R]) First() bool {
}
}
-func (t *TableIter[K, R]) Seek(key K) bool {
- pivot := &PhysicalRow[K, R]{
+func (t *TableIter[K, V]) Seek(key K) bool {
+ pivot := &PhysicalRow[K, V]{
Key: key,
}
if ok := t.iter.Seek(pivot); !ok {
@@ -94,7 +94,7 @@ func (t *TableIter[K, R]) Seek(key K) bool {
}
for {
// skip unreadable values
- value, _ := t.iter.Item().Values.Read(t.readTime, t.tx)
+ value, _ := t.iter.Item().Read(t.readTime, t.tx)
if value == nil {
if ok := t.iter.Next(); !ok {
return false
@@ -105,7 +105,7 @@ func (t *TableIter[K, R]) Seek(key K) bool {
}
}
-func (t *TableIter[K, R]) Close() error {
+func (t *TableIter[K, V]) Close() error {
t.iter.Release()
return nil
}
diff --git a/pkg/txn/storage/txn/table_test.go b/pkg/txn/storage/txn/memtable/table_test.go
similarity index 85%
rename from pkg/txn/storage/txn/table_test.go
rename to pkg/txn/storage/txn/memtable/table_test.go
index c6445dbec07640dbbd21ae1a21e4bcc54610a922..bda5cbcabfa963da059100c87fec10c2a0f4099f 100644
--- a/pkg/txn/storage/txn/table_test.go
+++ b/pkg/txn/storage/txn/memtable/table_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
import (
"testing"
@@ -31,6 +31,10 @@ func (t TestRow) Key() Int {
return t.key
}
+func (t TestRow) Value() int {
+ return t.value
+}
+
func (t TestRow) Indexes() []Tuple {
return []Tuple{
{Text("foo"), Int(t.value)},
@@ -39,7 +43,7 @@ func (t TestRow) Indexes() []Tuple {
func TestTable(t *testing.T) {
- table := NewTable[Int, TestRow]()
+ table := NewTable[Int, int, TestRow]()
tx := NewTransaction("1", Time{}, Serializable)
row := TestRow{key: 42, value: 1}
@@ -50,7 +54,7 @@ func TestTable(t *testing.T) {
// get
r, err := table.Get(tx, Int(42))
assert.Nil(t, err)
- assert.Equal(t, &row, r)
+ assert.Equal(t, 1, *r)
// update
row.value = 2
@@ -59,20 +63,21 @@ func TestTable(t *testing.T) {
r, err = table.Get(tx, Int(42))
assert.Nil(t, err)
- assert.Equal(t, &row, r)
+ assert.Equal(t, 2, *r)
// index
- keys, err := table.Index(tx, Tuple{
+ entries, err := table.Index(tx, Tuple{
Text("foo"), Int(1),
})
assert.Nil(t, err)
- assert.Equal(t, 0, len(keys))
- keys, err = table.Index(tx, Tuple{
+ assert.Equal(t, 0, len(entries))
+ entries, err = table.Index(tx, Tuple{
Text("foo"), Int(2),
})
assert.Nil(t, err)
- assert.Equal(t, 1, len(keys))
- assert.Equal(t, Int(42), keys[0])
+ assert.Equal(t, 1, len(entries))
+ assert.Equal(t, Int(42), entries[0].Key)
+ assert.Equal(t, 2, *entries[0].Value)
// delete
err = table.Delete(tx, Int(42))
@@ -82,7 +87,7 @@ func TestTable(t *testing.T) {
func TestTableIsolation(t *testing.T) {
- table := NewTable[Int, TestRow]()
+ table := NewTable[Int, int, TestRow]()
tx1 := NewTransaction("1", Time{
Timestamp: timestamp.Timestamp{
diff --git a/pkg/txn/storage/txn/time.go b/pkg/txn/storage/txn/memtable/time.go
similarity index 98%
rename from pkg/txn/storage/txn/time.go
rename to pkg/txn/storage/txn/memtable/time.go
index 1bdb32a158a0b5885e588fce90d525d0a198fd04..4331669c909a88b0986f48ceacfc76e7475e6c51 100644
--- a/pkg/txn/storage/txn/time.go
+++ b/pkg/txn/storage/txn/memtable/time.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
import (
"fmt"
diff --git a/pkg/txn/storage/txn/transaction.go b/pkg/txn/storage/txn/memtable/transaction.go
similarity index 98%
rename from pkg/txn/storage/txn/transaction.go
rename to pkg/txn/storage/txn/memtable/transaction.go
index 5d3f72ced798f56c81529b0baed753f9ccab47a8..8058c8ac9317936e605a2fef077d3946bb38b6f7 100644
--- a/pkg/txn/storage/txn/transaction.go
+++ b/pkg/txn/storage/txn/memtable/transaction.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
type Transaction struct {
ID string
diff --git a/pkg/txn/storage/txn/tuple.go b/pkg/txn/storage/txn/memtable/tuple.go
similarity index 89%
rename from pkg/txn/storage/txn/tuple.go
rename to pkg/txn/storage/txn/memtable/tuple.go
index 7668350a8aa1eb39fca33adc3cff98812b7b6f68..1cd5540f223ed8085a0d6cd864da4f1a5787950c 100644
--- a/pkg/txn/storage/txn/tuple.go
+++ b/pkg/txn/storage/txn/memtable/tuple.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package txnstorage
+package memtable
import "fmt"
@@ -78,8 +78,16 @@ func (t Tuple) Less(than Tuple) bool {
return false
}
+ case ID:
+ key2 := than[i].(ID)
+ if key.Less(key2) {
+ return true
+ } else if key2.Less(key) {
+ return false
+ }
+
default:
- panic(fmt.Errorf("unknown key type: %T", key))
+ panic(fmt.Sprintf("unknown key type: %T", key))
}
}
// equal
diff --git a/pkg/txn/storage/txn/memtable/types.go b/pkg/txn/storage/txn/memtable/types.go
new file mode 100644
index 0000000000000000000000000000000000000000..62b15967cc72f4f952e091112171b7e33532338b
--- /dev/null
+++ b/pkg/txn/storage/txn/memtable/types.go
@@ -0,0 +1,175 @@
+// Copyright 2022 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memtable
+
+import (
+ "bytes"
+ "fmt"
+
+ "github.com/matrixorigin/matrixone/pkg/container/types"
+ txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
+)
+
+type ID = txnengine.ID
+
+type Text string
+
+func (t Text) Less(than Text) bool {
+ return t < than
+}
+
+type Bool bool
+
+func (b Bool) Less(than Bool) bool {
+ return bool(!b && than)
+}
+
+type Int int64
+
+func (i Int) Less(than Int) bool {
+ return i < than
+}
+
+type Uint int64
+
+func (i Uint) Less(than Uint) bool {
+ return i < than
+}
+
+type Float float64
+
+func (f Float) Less(than Float) bool {
+ return f < than
+}
+
+type Bytes []byte
+
+func (b Bytes) Less(than Bytes) bool {
+ return bytes.Compare(b, than) < 0
+}
+
+func ToOrdered(v any) any {
+ if v == nil {
+ panic("should not be nil")
+ }
+ switch v := v.(type) {
+ case bool:
+ return Bool(v)
+ case int:
+ return Int(v)
+ case int8:
+ return Int(v)
+ case int16:
+ return Int(v)
+ case int32:
+ return Int(v)
+ case int64:
+ return Int(v)
+ case uint:
+ return Uint(v)
+ case uint8:
+ return Uint(v)
+ case uint16:
+ return Uint(v)
+ case uint32:
+ return Uint(v)
+ case uint64:
+ return Uint(v)
+ case float32:
+ return Float(v)
+ case float64:
+ return Float(v)
+ case []byte:
+ return Bytes(v)
+ case types.Date:
+ return Int(v)
+ case types.Datetime:
+ return Int(v)
+ case types.Timestamp:
+ return Int(v)
+ case types.Decimal64:
+ return Bytes(v[:])
+ case types.Decimal128:
+ return Bytes(v[:])
+ case types.TS:
+ return Bytes(v[:])
+ case types.Rowid:
+ return Bytes(v[:])
+ case types.Uuid:
+ return Bytes(v[:])
+ case ID:
+ return v
+ default:
+ panic(fmt.Sprintf("unknown type: %T", v))
+ }
+}
+
+func TypeMatch(v any, typ types.T) bool {
+ if v == nil {
+ panic("should not be nil")
+ }
+ var ok bool
+ switch typ {
+ case types.T_bool:
+ _, ok = v.(bool)
+ case types.T_int8:
+ _, ok = v.(int8)
+ case types.T_int16:
+ _, ok = v.(int16)
+ case types.T_int32:
+ _, ok = v.(int32)
+ case types.T_int64:
+ _, ok = v.(int64)
+ case types.T_uint8:
+ _, ok = v.(uint8)
+ case types.T_uint16:
+ _, ok = v.(uint16)
+ case types.T_uint32:
+ _, ok = v.(uint32)
+ case types.T_uint64:
+ _, ok = v.(uint64)
+ case types.T_float32:
+ _, ok = v.(float32)
+ case types.T_float64:
+ _, ok = v.(float64)
+ case types.T_decimal64:
+ _, ok = v.(types.Decimal64)
+ case types.T_decimal128:
+ _, ok = v.(types.Decimal128)
+ case types.T_date:
+ _, ok = v.(types.Date)
+ case types.T_time:
+ _, ok = v.(types.TimeType)
+ case types.T_datetime:
+ _, ok = v.(types.Datetime)
+ case types.T_timestamp:
+ _, ok = v.(types.Timestamp)
+ case types.T_interval:
+ _, ok = v.(types.IntervalType)
+ case types.T_char:
+ _, ok = v.(string)
+ case types.T_varchar:
+ _, ok = v.(string)
+ case types.T_json:
+ _, ok = v.(string)
+ case types.T_blob:
+ _, ok = v.(string)
+ case types.T_uuid:
+ _, ok = v.(string)
+ default:
+ panic(fmt.Sprintf("fixme: %v", typ))
+ }
+ return ok
+}
diff --git a/pkg/txn/storage/txn/storage_test.go b/pkg/txn/storage/txn/storage_test.go
index 7c6dc0b012dcac89ed04a6b8425eba8812b00f45..e1774f035a367baeeadf03109f98c3bf031b3e48 100644
--- a/pkg/txn/storage/txn/storage_test.go
+++ b/pkg/txn/storage/txn/storage_test.go
@@ -20,6 +20,7 @@ import (
"encoding/gob"
"testing"
+ "github.com/matrixorigin/matrixone/pkg/common/moerr"
"github.com/matrixorigin/matrixone/pkg/container/batch"
"github.com/matrixorigin/matrixone/pkg/container/types"
"github.com/matrixorigin/matrixone/pkg/container/vector"
@@ -54,73 +55,75 @@ func testDatabase(
// open database
{
- resp := testRead[txnengine.OpenDatabaseResp](
+ _, err := testRead[txnengine.OpenDatabaseResp](
t, s, txnMeta,
txnengine.OpOpenDatabase,
txnengine.OpenDatabaseReq{
Name: "foo",
},
)
- assert.Equal(t, "foo", resp.ErrResp.Name)
+ assert.True(t, moerr.IsMoErrCode(err, moerr.ErrNoDB))
}
// create database
{
- resp := testWrite[txnengine.CreateDatabaseResp](
+ resp, err := testWrite[txnengine.CreateDatabaseResp](
t, s, txnMeta,
txnengine.OpCreateDatabase,
txnengine.CreateDatabaseReq{
Name: "foo",
},
)
- assert.Equal(t, false, resp.ErrResp.ErrExisted)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.ID)
}
// get databases
{
- resp := testRead[txnengine.GetDatabasesResp](
+ resp, err := testRead[txnengine.GetDatabasesResp](
t, s, txnMeta,
txnengine.OpGetDatabases,
txnengine.GetDatabasesReq{},
)
+ assert.Nil(t, err)
assert.Equal(t, 1, len(resp.Names))
assert.Equal(t, "foo", resp.Names[0])
}
// open database
- var dbID string
+ var dbID ID
{
- resp := testRead[txnengine.OpenDatabaseResp](
+ resp, err := testRead[txnengine.OpenDatabaseResp](
t, s, txnMeta,
txnengine.OpOpenDatabase,
txnengine.OpenDatabaseReq{
Name: "foo",
},
)
- assert.Equal(t, "", resp.ErrResp.Name)
+ assert.Nil(t, err)
assert.NotNil(t, resp.ID)
dbID = resp.ID
// delete database
defer func() {
{
- resp := testWrite[txnengine.DeleteDatabaseResp](
+ resp, err := testWrite[txnengine.DeleteDatabaseResp](
t, s, txnMeta,
txnengine.OpDeleteDatabase,
txnengine.DeleteDatabaseReq{
Name: "foo",
},
)
- assert.Equal(t, "", resp.ErrResp.Name)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.ID)
}
{
- resp := testRead[txnengine.GetDatabasesResp](
+ resp, err := testRead[txnengine.GetDatabasesResp](
t, s, txnMeta,
txnengine.OpGetDatabases,
txnengine.GetDatabasesReq{},
)
+ assert.Nil(t, err)
assert.Equal(t, 0, len(resp.Names))
}
}()
@@ -128,7 +131,7 @@ func testDatabase(
// open relation
{
- resp := testRead[txnengine.OpenRelationResp](
+ _, err := testRead[txnengine.OpenRelationResp](
t, s, txnMeta,
txnengine.OpOpenRelation,
txnengine.OpenRelationReq{
@@ -136,12 +139,12 @@ func testDatabase(
Name: "table",
},
)
- assert.Equal(t, "table", resp.ErrResp.Name)
+ assert.True(t, moerr.IsMoErrCode(err, moerr.ErrNoSuchTable))
}
// create relation
{
- resp := testWrite[txnengine.CreateRelationResp](
+ resp, err := testWrite[txnengine.CreateRelationResp](
t, s, txnMeta,
txnengine.OpCreateRelation,
txnengine.CreateRelationReq{
@@ -166,27 +169,28 @@ func testDatabase(
},
},
)
- assert.Equal(t, false, resp.ErrResp.ErrExisted)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.ID)
}
// get relations
{
- resp := testRead[txnengine.GetRelationsResp](
+ resp, err := testRead[txnengine.GetRelationsResp](
t, s, txnMeta,
txnengine.OpGetRelations,
txnengine.GetRelationsReq{
DatabaseID: dbID,
},
)
+ assert.Nil(t, err)
assert.Equal(t, 1, len(resp.Names))
assert.Equal(t, "table", resp.Names[0])
}
// open relation
- var relID string
+ var relID ID
{
- resp := testRead[txnengine.OpenRelationResp](
+ resp, err := testRead[txnengine.OpenRelationResp](
t, s, txnMeta,
txnengine.OpOpenRelation,
txnengine.OpenRelationReq{
@@ -194,7 +198,7 @@ func testDatabase(
Name: "table",
},
)
- assert.Equal(t, "", resp.ErrResp.Name)
+ assert.Nil(t, err)
assert.NotNil(t, resp.ID)
relID = resp.ID
assert.Equal(t, txnengine.RelationTable, resp.Type)
@@ -203,15 +207,14 @@ func testDatabase(
// get relation defs
{
- resp := testRead[txnengine.GetTableDefsResp](
+ resp, err := testRead[txnengine.GetTableDefsResp](
t, s, txnMeta,
txnengine.OpGetTableDefs,
txnengine.GetTableDefsReq{
TableID: relID,
},
)
- assert.Empty(t, resp.ErrResp.ID)
- assert.Empty(t, resp.ErrResp.Name)
+ assert.Nil(t, err)
assert.Equal(t, 3, len(resp.Defs))
}
@@ -239,7 +242,7 @@ func testDatabase(
bat.Vecs[0] = colA
bat.Vecs[1] = colB
bat.InitZsOne(5)
- resp := testWrite[txnengine.WriteResp](
+ _, err := testWrite[txnengine.WriteResp](
t, s, txnMeta,
txnengine.OpWrite,
txnengine.WriteReq{
@@ -247,26 +250,25 @@ func testDatabase(
Batch: bat,
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
}
// read
- var iterID string
+ var iterID ID
{
- resp := testRead[txnengine.NewTableIterResp](
+ resp, err := testRead[txnengine.NewTableIterResp](
t, s, txnMeta,
txnengine.OpNewTableIter,
txnengine.NewTableIterReq{
TableID: relID,
},
)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.IterID)
- assert.Empty(t, resp.ErrResp)
iterID = resp.IterID
}
{
- resp := testRead[txnengine.ReadResp](
+ resp, err := testRead[txnengine.ReadResp](
t, s, txnMeta,
txnengine.OpRead,
txnengine.ReadReq{
@@ -274,8 +276,7 @@ func testDatabase(
ColNames: []string{"a", "b"},
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
assert.NotNil(t, resp.Batch)
assert.Equal(t, 5, resp.Batch.Length())
}
@@ -291,7 +292,7 @@ func testDatabase(
1,
},
)
- resp := testWrite[txnengine.DeleteResp](
+ _, err := testWrite[txnengine.DeleteResp](
t, s, txnMeta,
txnengine.OpDelete,
txnengine.DeleteReq{
@@ -300,25 +301,24 @@ func testDatabase(
Vector: colA,
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
}
// read after delete
{
- resp := testRead[txnengine.NewTableIterResp](
+ resp, err := testRead[txnengine.NewTableIterResp](
t, s, txnMeta,
txnengine.OpNewTableIter,
txnengine.NewTableIterReq{
TableID: relID,
},
)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.IterID)
- assert.Empty(t, resp.ErrResp)
iterID = resp.IterID
}
{
- resp := testRead[txnengine.ReadResp](
+ resp, err := testRead[txnengine.ReadResp](
t, s, txnMeta,
txnengine.OpRead,
txnengine.ReadReq{
@@ -326,8 +326,7 @@ func testDatabase(
ColNames: []string{"a", "b"},
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
assert.NotNil(t, resp.Batch)
assert.Equal(t, 4, resp.Batch.Length())
}
@@ -343,7 +342,7 @@ func testDatabase(
8,
},
)
- resp := testWrite[txnengine.DeleteResp](
+ _, err := testWrite[txnengine.DeleteResp](
t, s, txnMeta,
txnengine.OpDelete,
txnengine.DeleteReq{
@@ -352,25 +351,24 @@ func testDatabase(
Vector: colB,
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
}
// read after delete
{
- resp := testRead[txnengine.NewTableIterResp](
+ resp, err := testRead[txnengine.NewTableIterResp](
t, s, txnMeta,
txnengine.OpNewTableIter,
txnengine.NewTableIterReq{
TableID: relID,
},
)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.IterID)
- assert.Empty(t, resp.ErrResp)
iterID = resp.IterID
}
{
- resp := testRead[txnengine.ReadResp](
+ resp, err := testRead[txnengine.ReadResp](
t, s, txnMeta,
txnengine.OpRead,
txnengine.ReadReq{
@@ -378,8 +376,7 @@ func testDatabase(
ColNames: []string{"a", "b"},
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
assert.NotNil(t, resp.Batch)
assert.Equal(t, 3, resp.Batch.Length())
}
@@ -408,7 +405,7 @@ func testDatabase(
bat.Vecs[0] = colA
bat.Vecs[1] = colB
bat.InitZsOne(1)
- resp := testWrite[txnengine.WriteResp](
+ _, err := testWrite[txnengine.WriteResp](
t, s, txnMeta,
txnengine.OpWrite,
txnengine.WriteReq{
@@ -416,13 +413,12 @@ func testDatabase(
Batch: bat,
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
}
// delete relation
{
- resp := testWrite[txnengine.DeleteRelationResp](
+ resp, err := testWrite[txnengine.DeleteRelationResp](
t, s, txnMeta,
txnengine.OpDeleteRelation,
txnengine.DeleteRelationReq{
@@ -430,23 +426,24 @@ func testDatabase(
Name: "table",
},
)
- assert.Equal(t, "", resp.ErrResp.Name)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.ID)
}
{
- resp := testRead[txnengine.GetRelationsResp](
+ resp, err := testRead[txnengine.GetRelationsResp](
t, s, txnMeta,
txnengine.OpGetRelations,
txnengine.GetRelationsReq{
DatabaseID: dbID,
},
)
+ assert.Nil(t, err)
assert.Equal(t, 0, len(resp.Names))
}
// new relation without primary key
{
- resp := testWrite[txnengine.CreateRelationResp](
+ resp, err := testWrite[txnengine.CreateRelationResp](
t, s, txnMeta,
txnengine.OpCreateRelation,
txnengine.CreateRelationReq{
@@ -469,7 +466,7 @@ func testDatabase(
},
},
)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.ID)
relID = resp.ID
}
@@ -498,7 +495,7 @@ func testDatabase(
bat.Vecs[0] = colA
bat.Vecs[1] = colB
bat.InitZsOne(5)
- resp := testWrite[txnengine.WriteResp](
+ _, err := testWrite[txnengine.WriteResp](
t, s, txnMeta,
txnengine.OpWrite,
txnengine.WriteReq{
@@ -506,8 +503,7 @@ func testDatabase(
Batch: bat,
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
}
// delete by primary key
@@ -521,7 +517,7 @@ func testDatabase(
1,
},
)
- resp := testWrite[txnengine.DeleteResp](
+ _, err := testWrite[txnengine.DeleteResp](
t, s, txnMeta,
txnengine.OpDelete,
txnengine.DeleteReq{
@@ -530,25 +526,24 @@ func testDatabase(
Vector: colA,
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
}
// read after delete
{
- resp := testRead[txnengine.NewTableIterResp](
+ resp, err := testRead[txnengine.NewTableIterResp](
t, s, txnMeta,
txnengine.OpNewTableIter,
txnengine.NewTableIterReq{
TableID: relID,
},
)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.IterID)
- assert.Empty(t, resp.ErrResp)
iterID = resp.IterID
}
{
- resp := testRead[txnengine.ReadResp](
+ resp, err := testRead[txnengine.ReadResp](
t, s, txnMeta,
txnengine.OpRead,
txnengine.ReadReq{
@@ -556,8 +551,7 @@ func testDatabase(
ColNames: []string{"a", "b"},
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
assert.NotNil(t, resp.Batch)
assert.Equal(t, 4, resp.Batch.Length())
}
@@ -573,7 +567,7 @@ func testDatabase(
8,
},
)
- resp := testWrite[txnengine.DeleteResp](
+ _, err := testWrite[txnengine.DeleteResp](
t, s, txnMeta,
txnengine.OpDelete,
txnengine.DeleteReq{
@@ -582,26 +576,25 @@ func testDatabase(
Vector: colB,
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
}
// read after delete
{
- resp := testRead[txnengine.NewTableIterResp](
+ resp, err := testRead[txnengine.NewTableIterResp](
t, s, txnMeta,
txnengine.OpNewTableIter,
txnengine.NewTableIterReq{
TableID: relID,
},
)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.IterID)
- assert.Empty(t, resp.ErrResp)
iterID = resp.IterID
}
var rowIDs *vector.Vector
{
- resp := testRead[txnengine.ReadResp](
+ resp, err := testRead[txnengine.ReadResp](
t, s, txnMeta,
txnengine.OpRead,
txnengine.ReadReq{
@@ -609,8 +602,7 @@ func testDatabase(
ColNames: []string{"a", "b", rowIDColumnName},
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
assert.NotNil(t, resp.Batch)
assert.Equal(t, 3, resp.Batch.Length())
rowIDs = resp.Batch.Vecs[2]
@@ -618,7 +610,7 @@ func testDatabase(
// delete by row id
{
- resp := testWrite[txnengine.DeleteResp](
+ _, err := testWrite[txnengine.DeleteResp](
t, s, txnMeta,
txnengine.OpDelete,
txnengine.DeleteReq{
@@ -627,25 +619,24 @@ func testDatabase(
Vector: rowIDs,
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
}
// read after delete
{
- resp := testRead[txnengine.NewTableIterResp](
+ resp, err := testRead[txnengine.NewTableIterResp](
t, s, txnMeta,
txnengine.OpNewTableIter,
txnengine.NewTableIterReq{
TableID: relID,
},
)
+ assert.Nil(t, err)
assert.NotEmpty(t, resp.IterID)
- assert.Empty(t, resp.ErrResp)
iterID = resp.IterID
}
{
- resp := testRead[txnengine.ReadResp](
+ resp, err := testRead[txnengine.ReadResp](
t, s, txnMeta,
txnengine.OpRead,
txnengine.ReadReq{
@@ -653,8 +644,7 @@ func testDatabase(
ColNames: []string{"a", "b", rowIDColumnName},
},
)
- assert.Empty(t, resp.ErrResp)
- assert.Empty(t, resp.ErrResp)
+ assert.Nil(t, err)
assert.Nil(t, resp.Batch)
}
@@ -675,14 +665,17 @@ func testRead[
req Req,
) (
resp Resp,
+ err error,
) {
buf := new(bytes.Buffer)
- err := gob.NewEncoder(buf).Encode(req)
+ err = gob.NewEncoder(buf).Encode(req)
assert.Nil(t, err)
res, err := s.Read(context.TODO(), txnMeta, op, buf.Bytes())
- assert.Nil(t, err)
+ if err != nil {
+ return
+ }
data, err := res.Read()
assert.Nil(t, err)
@@ -703,14 +696,17 @@ func testWrite[
req Req,
) (
resp Resp,
+ err error,
) {
buf := new(bytes.Buffer)
- err := gob.NewEncoder(buf).Encode(req)
+ err = gob.NewEncoder(buf).Encode(req)
assert.Nil(t, err)
data, err := s.Write(context.TODO(), txnMeta, op, buf.Bytes())
- assert.Nil(t, err)
+ if err != nil {
+ return
+ }
err = gob.NewDecoder(bytes.NewReader(data)).Decode(&resp)
assert.Nil(t, err)
diff --git a/pkg/txn/storage/txn/table.go b/pkg/txn/storage/txn/table.go
deleted file mode 100644
index d328ed980c6fd64e732b0a9d514b5d700bbe62eb..0000000000000000000000000000000000000000
--- a/pkg/txn/storage/txn/table.go
+++ /dev/null
@@ -1,328 +0,0 @@
-// Copyright 2022 Matrix Origin
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package txnstorage
-
-import (
- "database/sql"
- "errors"
- "sync"
- "time"
-
- "github.com/matrixorigin/matrixone/pkg/common/moerr"
- "github.com/tidwall/btree"
-)
-
-type Table[
- K Ordered[K],
- R Row[K],
-] struct {
- sync.Mutex
- rows *btree.Generic[*PhysicalRow[K, R]]
- index *btree.Generic[IndexEntry[K, R]]
- writeSets map[*Transaction]map[*PhysicalRow[K, R]]struct{}
-}
-
-type Row[K any] interface {
- Key() K
- Indexes() []Tuple
-}
-
-type NamedRow interface {
- AttrByName(tx *Transaction, name string) (Nullable, error)
-}
-
-type PhysicalRow[
- K Ordered[K],
- R Row[K],
-] struct {
- Key K
- LastUpdate *Atomic[time.Time]
- Values *MVCC[R]
-}
-
-type Ordered[To any] interface {
- Less(to To) bool
-}
-
-type IndexEntry[K Ordered[K], R Row[K]] struct {
- Index Tuple
- Key K
- Row *R
-}
-
-func NewTable[
- K Ordered[K],
- R Row[K],
-]() *Table[K, R] {
- return &Table[K, R]{
- rows: btree.NewGeneric(func(a, b *PhysicalRow[K, R]) bool {
- return a.Key.Less(b.Key)
- }),
- index: btree.NewGeneric(func(a, b IndexEntry[K, R]) bool {
- if a.Index.Less(b.Index) {
- return true
- }
- if b.Index.Less(a.Index) {
- return false
- }
- return a.Key.Less(b.Key)
- }),
- writeSets: make(map[*Transaction]map[*PhysicalRow[K, R]]struct{}),
- }
-}
-
-func (t *Table[K, R]) Insert(
- tx *Transaction,
- row R,
-) error {
- key := row.Key()
- t.Lock()
- physicalRow := t.getOrSetRowByKey(key)
- t.Unlock()
-
- existed, err := physicalRow.Values.ReadVisible(tx.Time, tx)
- if errors.Is(err, sql.ErrNoRows) {
- err = nil
- }
- if err != nil {
- return err
- }
- if existed != nil {
- return moerr.NewPrimaryKeyDuplicated(key)
- }
-
- if err := physicalRow.Values.Insert(tx.Time, tx, &row); err != nil {
- return err
- }
- physicalRow.LastUpdate.Store(time.Now())
- for _, index := range row.Indexes() {
- t.index.Set(IndexEntry[K, R]{
- Index: index,
- Key: key,
- Row: &row,
- })
- }
-
- t.setCommitter(tx, physicalRow)
-
- tx.Time.Tick()
- return nil
-}
-
-func (t *Table[K, R]) Update(
- tx *Transaction,
- row R,
-) error {
- key := row.Key()
- t.Lock()
- physicalRow := t.getOrSetRowByKey(key)
- t.Unlock()
-
- if err := physicalRow.Values.Update(tx.Time, tx, &row); err != nil {
- return err
- }
- physicalRow.LastUpdate.Store(time.Now())
- for _, index := range row.Indexes() {
- t.index.Set(IndexEntry[K, R]{
- Index: index,
- Key: key,
- Row: &row,
- })
- }
-
- //TODO
- //t.setCommitter(tx, physicalRow)
-
- tx.Time.Tick()
- return nil
-}
-
-func (t *Table[K, R]) Delete(
- tx *Transaction,
- key K,
-) error {
- t.Lock()
- physicalRow := t.getRowByKey(key)
- t.Unlock()
-
- if physicalRow == nil {
- return nil
- }
- if err := physicalRow.Values.Delete(tx.Time, tx); err != nil {
- return err
- }
- physicalRow.LastUpdate.Store(time.Now())
-
- //TODO
- //t.setCommitter(tx, physicalRow)
-
- tx.Time.Tick()
- return nil
-}
-
-func (t *Table[K, R]) Get(
- tx *Transaction,
- key K,
-) (
- row *R,
- err error,
-) {
- t.Lock()
- physicalRow := t.getRowByKey(key)
- t.Unlock()
- if physicalRow == nil {
- err = sql.ErrNoRows
- return
- }
- mvccValues := physicalRow.Values
- row, err = mvccValues.Read(tx.Time, tx)
- if err != nil {
- return
- }
- return
-}
-
-func (t *Table[K, R]) getRowByKey(key K) *PhysicalRow[K, R] {
- pivot := &PhysicalRow[K, R]{
- Key: key,
- }
- row, _ := t.rows.Get(pivot)
- return row
-}
-
-func (t *Table[K, R]) getOrSetRowByKey(key K) *PhysicalRow[K, R] {
- pivot := &PhysicalRow[K, R]{
- Key: key,
- }
- row, ok := t.rows.Get(pivot)
- if !ok {
- row = pivot
- row.Values = new(MVCC[R])
- row.LastUpdate = NewAtomic(time.Now())
- t.rows.Set(row)
- }
- return row
-}
-
-func (t *Table[K, R]) Index(tx *Transaction, index Tuple) (keys []K, err error) {
- pivot := IndexEntry[K, R]{
- Index: index,
- }
- iter := t.index.Copy().Iter()
- defer iter.Release()
- for ok := iter.Seek(pivot); ok; ok = iter.Next() {
- item := iter.Item()
- if index.Less(item.Index) {
- break
- }
- if item.Index.Less(index) {
- break
- }
- cur, err := t.Get(tx, item.Key)
- if err != nil {
- if errors.Is(err, sql.ErrNoRows) {
- continue
- }
- return nil, err
- }
- if cur == item.Row {
- keys = append(keys, item.Key)
- }
- }
- return
-}
-
-func (t *Table[K, R]) IndexRows(tx *Transaction, index Tuple) (rows []*R, err error) {
- pivot := IndexEntry[K, R]{
- Index: index,
- }
- iter := t.index.Copy().Iter()
- defer iter.Release()
- for ok := iter.Seek(pivot); ok; ok = iter.Next() {
- item := iter.Item()
- if index.Less(item.Index) {
- break
- }
- if item.Index.Less(index) {
- break
- }
- cur, err := t.Get(tx, item.Key)
- if err != nil {
- if errors.Is(err, sql.ErrNoRows) {
- continue
- }
- return nil, err
- }
- if cur == item.Row {
- rows = append(rows, cur)
- }
- }
- return
-}
-
-func (t *Table[K, R]) setCommitter(tx *Transaction, row *PhysicalRow[K, R]) {
- tx.committers[t] = struct{}{}
- t.Lock()
- set, ok := t.writeSets[tx]
- if !ok {
- set = make(map[*PhysicalRow[K, R]]struct{})
- t.writeSets[tx] = set
- }
- set[row] = struct{}{}
- t.Unlock()
-}
-
-func (t *Table[K, R]) CommitTx(tx *Transaction) error {
- t.Lock()
- set := t.writeSets[tx]
- t.Unlock()
- defer func() {
- t.Lock()
- delete(t.writeSets, tx)
- t.Unlock()
- }()
-
- for physicalRow := range set {
- values := physicalRow.Values
-
- var err error
- values.RLock()
- for i := len(values.Values) - 1; i >= 0; i-- {
- value := values.Values[i]
-
- if value.Visible(tx.Time, tx.ID) &&
- value.BornTx.ID != tx.ID &&
- value.BornTime.After(tx.BeginTime) {
- err = moerr.NewPrimaryKeyDuplicated(physicalRow.Key)
- break
- }
-
- }
- values.RUnlock()
-
- if err != nil {
- return err
- }
-
- }
-
- return nil
-}
-
-func (t *Table[K, R]) AbortTx(tx *Transaction) {
- t.Lock()
- delete(t.writeSets, tx)
- t.Unlock()
-}
diff --git a/pkg/txn/storage/txn/types.go b/pkg/txn/storage/txn/types.go
index c8c48b2117e4ca46c69d04bbce00e7efc8853a17..ae783147176c177667e14c12f8809f9f35ce000e 100644
--- a/pkg/txn/storage/txn/types.go
+++ b/pkg/txn/storage/txn/types.go
@@ -15,161 +15,27 @@
package txnstorage
import (
- "bytes"
- "crypto/rand"
- "encoding/binary"
- "fmt"
-
- "github.com/matrixorigin/matrixone/pkg/container/types"
+ "github.com/matrixorigin/matrixone/pkg/txn/storage/txn/memtable"
+ txnengine "github.com/matrixorigin/matrixone/pkg/vm/engine/txn"
)
-type Text string
-
-func (t Text) Less(than Text) bool {
- return t < than
-}
-
-type Bool bool
-
-func (b Bool) Less(than Bool) bool {
- return bool(!b && than)
-}
-
-type Int int64
-
-func (i Int) Less(than Int) bool {
- return i < than
-}
-
-type Uint int64
-
-func (i Uint) Less(than Uint) bool {
- return i < than
-}
-
-type Float float64
-
-func (f Float) Less(than Float) bool {
- return f < than
-}
-
-type Bytes []byte
-
-func (b Bytes) Less(than Bytes) bool {
- return bytes.Compare(b, than) < 0
-}
-
-func typeConv(v any) any {
- if v == nil {
- panic("should not be nil")
- }
- switch v := v.(type) {
- case bool:
- return Bool(v)
- case int:
- return Int(v)
- case int8:
- return Int(v)
- case int16:
- return Int(v)
- case int32:
- return Int(v)
- case int64:
- return Int(v)
- case uint:
- return Uint(v)
- case uint8:
- return Uint(v)
- case uint16:
- return Uint(v)
- case uint32:
- return Uint(v)
- case uint64:
- return Uint(v)
- case float32:
- return Float(v)
- case float64:
- return Float(v)
- case []byte:
- return Bytes(v)
- case types.Date:
- return Int(v)
- case types.Datetime:
- return Int(v)
- case types.Timestamp:
- return Int(v)
- case types.Decimal64:
- return Bytes(v[:])
- case types.Decimal128:
- return Bytes(v[:])
- case types.TS:
- return Bytes(v[:])
- case types.Rowid:
- return Bytes(v[:])
- case types.Uuid:
- return Bytes(v[:])
- default:
- panic(fmt.Errorf("unknown type: %T", v))
- }
-}
+type (
+ ID = txnengine.ID
+ IsolationPolicy = memtable.IsolationPolicy
+ Nullable = memtable.Nullable
+ Transaction = memtable.Transaction
+ Tuple = memtable.Tuple
+ Text = memtable.Text
+ Uint = memtable.Uint
+ Bool = memtable.Bool
+ Time = memtable.Time
+ NamedRow = memtable.NamedRow
+)
-func typeMatch(v any, typ types.T) bool {
- if v == nil {
- panic("should not be nil")
- }
- var ok bool
- switch typ {
- case types.T_bool:
- _, ok = v.(bool)
- case types.T_int8:
- _, ok = v.(int8)
- case types.T_int16:
- _, ok = v.(int16)
- case types.T_int32:
- _, ok = v.(int32)
- case types.T_int64:
- _, ok = v.(int64)
- case types.T_uint8:
- _, ok = v.(uint8)
- case types.T_uint16:
- _, ok = v.(uint16)
- case types.T_uint32:
- _, ok = v.(uint32)
- case types.T_uint64:
- _, ok = v.(uint64)
- case types.T_float32:
- _, ok = v.(float32)
- case types.T_float64:
- _, ok = v.(float64)
- case types.T_decimal64:
- _, ok = v.(types.Decimal64)
- case types.T_decimal128:
- _, ok = v.(types.Decimal128)
- case types.T_date:
- _, ok = v.(types.Date)
- case types.T_time:
- _, ok = v.(types.TimeType)
- case types.T_datetime:
- _, ok = v.(types.Datetime)
- case types.T_timestamp:
- _, ok = v.(types.Timestamp)
- case types.T_interval:
- _, ok = v.(types.IntervalType)
- case types.T_char:
- _, ok = v.(string)
- case types.T_varchar:
- _, ok = v.(string)
- case types.T_json:
- _, ok = v.(string)
- case types.T_blob:
- _, ok = v.(string)
- case types.T_uuid:
- _, ok = v.(string)
- default:
- panic(fmt.Errorf("fixme: %v", typ))
- }
- return ok
-}
+var (
+ SnapshotIsolation = memtable.SnapshotIsolation
+ Serializable = memtable.Serializable
+)
func boolToInt8(b bool) int8 {
if b {
@@ -177,12 +43,3 @@ func boolToInt8(b bool) int8 {
}
return 0
}
-
-func newRowID() types.Rowid {
- var rowid types.Rowid
- err := binary.Read(rand.Reader, binary.LittleEndian, &rowid)
- if err != nil {
- panic(err)
- }
- return rowid
-}
diff --git a/pkg/vm/engine/txn/database.go b/pkg/vm/engine/txn/database.go
index 7b6c125e763dd92f8334f5b043572c8f26ce72ad..e70c0844099d5db8348d79265f6d9a523196fbde 100644
--- a/pkg/vm/engine/txn/database.go
+++ b/pkg/vm/engine/txn/database.go
@@ -24,10 +24,10 @@ import (
)
type Database struct {
+ id ID
+ name string
engine *Engine
txnOperator client.TxnOperator
-
- id string
}
var _ engine.Database = new(Database)
@@ -41,10 +41,11 @@ func (d *Database) Create(ctx context.Context, relName string, defs []engine.Tab
d.engine.allNodesShards,
OpCreateRelation,
CreateRelationReq{
- DatabaseID: d.id,
- Type: RelationTable,
- Name: strings.ToLower(relName),
- Defs: defs,
+ DatabaseID: d.id,
+ DatabaseName: d.name,
+ Type: RelationTable,
+ Name: strings.ToLower(relName),
+ Defs: defs,
},
)
if err != nil {
@@ -63,8 +64,9 @@ func (d *Database) Delete(ctx context.Context, relName string) error {
d.engine.allNodesShards,
OpDeleteRelation,
DeleteRelationReq{
- DatabaseID: d.id,
- Name: strings.ToLower(relName),
+ DatabaseID: d.id,
+ DatabaseName: d.name,
+ Name: strings.ToLower(relName),
},
)
if err != nil {
@@ -87,8 +89,9 @@ func (d *Database) Relation(ctx context.Context, relName string) (engine.Relatio
d.engine.firstNodeShard,
OpOpenRelation,
OpenRelationReq{
- DatabaseID: d.id,
- Name: strings.ToLower(relName),
+ DatabaseID: d.id,
+ DatabaseName: d.name,
+ Name: strings.ToLower(relName),
},
)
if err != nil {
@@ -101,9 +104,11 @@ func (d *Database) Relation(ctx context.Context, relName string) (engine.Relatio
case RelationTable, RelationView:
table := &Table{
- engine: d.engine,
- txnOperator: d.txnOperator,
- id: resp.ID,
+ engine: d.engine,
+ txnOperator: d.txnOperator,
+ id: resp.ID,
+ databaseName: resp.DatabaseName,
+ tableName: resp.RelationName,
}
return table, nil
diff --git a/pkg/vm/engine/txn/engine.go b/pkg/vm/engine/txn/engine.go
index c3d956a8c496d2890e356304f9105e92c0ea1237..ddbb1653c7ff2ce1bd0b68fe5ada9c2b639a0f58 100644
--- a/pkg/vm/engine/txn/engine.go
+++ b/pkg/vm/engine/txn/engine.go
@@ -102,6 +102,7 @@ func (e *Engine) Database(ctx context.Context, dbName string, txnOperator client
engine: e,
txnOperator: txnOperator,
id: resp.ID,
+ name: resp.Name,
}
return db, nil
diff --git a/pkg/vm/engine/txn/id.go b/pkg/vm/engine/txn/id.go
new file mode 100644
index 0000000000000000000000000000000000000000..c4a6698df18cbcee4d7f8b6c104df757e1fb7f0f
--- /dev/null
+++ b/pkg/vm/engine/txn/id.go
@@ -0,0 +1,47 @@
+// Copyright 2022 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package txnengine
+
+import (
+ crand "crypto/rand"
+ "encoding/binary"
+ "math/rand"
+)
+
+type ID int64
+
+func init() {
+ var seed int64
+ binary.Read(crand.Reader, binary.LittleEndian, &seed)
+ rand.Seed(seed)
+}
+
+func NewID() (id ID) {
+ //TODO will use an id generate service
+ id = ID(rand.Int63())
+ return
+}
+
+func (i ID) Less(than ID) bool {
+ return i < than
+}
+
+func (i ID) IsEmpty() bool {
+ return i == emptyID
+}
+
+var (
+ emptyID ID
+)
diff --git a/pkg/vm/engine/txn/log_tail.go b/pkg/vm/engine/txn/log_tail.go
index 94df60425411ccfd08c73eed1d6f6ae54e0ce1a7..2969eec677e0d5e636120a53f304c5e7e0989c32 100644
--- a/pkg/vm/engine/txn/log_tail.go
+++ b/pkg/vm/engine/txn/log_tail.go
@@ -32,7 +32,7 @@ func (t *Table) GetLogTail(
err error,
) {
- resps, err := DoTxnRequest[GetLogTailResp](
+ resps, err := DoTxnRequest[apipb.SyncLogTailResp](
ctx,
t.engine,
t.txnOperator.Read,
@@ -40,12 +40,11 @@ func (t *Table) GetLogTail(
return t.engine.shardPolicy.Stores([]logservicepb.DNStore{targetStore})
},
OpGetLogTail,
- GetLogTailReq{
- TableID: t.id,
- Request: apipb.SyncLogTailReq{
- CnHave: from,
- CnWant: to,
- Table: &apipb.TableID{}, // we dont use this
+ apipb.SyncLogTailReq{
+ CnHave: from,
+ CnWant: to,
+ Table: &apipb.TableID{
+ TbId: uint64(t.id),
},
},
)
@@ -53,5 +52,5 @@ func (t *Table) GetLogTail(
return nil, err
}
- return &resps[0].Response, nil
+ return &resps[0], nil
}
diff --git a/pkg/vm/engine/txn/operations.go b/pkg/vm/engine/txn/operations.go
index b3fd9404de13e6ab5e2a04e68d9067e0c43f08e5..0e83b8010f88a1285c720328daba5a0b72a888d9 100644
--- a/pkg/vm/engine/txn/operations.go
+++ b/pkg/vm/engine/txn/operations.go
@@ -17,7 +17,6 @@ package txnengine
import (
"encoding/gob"
- "github.com/matrixorigin/matrixone/pkg/common/moerr"
"github.com/matrixorigin/matrixone/pkg/container/batch"
"github.com/matrixorigin/matrixone/pkg/container/types"
"github.com/matrixorigin/matrixone/pkg/container/vector"
@@ -109,11 +108,54 @@ func init() {
}
-type ErrorResp struct {
- ErrExisted bool
- ID string
- Name string
- Why string
+type Request interface {
+ CreateDatabaseReq |
+ OpenDatabaseReq |
+ GetDatabasesReq |
+ DeleteDatabaseReq |
+ CreateRelationReq |
+ DeleteRelationReq |
+ OpenRelationReq |
+ GetRelationsReq |
+ AddTableDefReq |
+ DelTableDefReq |
+ DeleteReq |
+ GetPrimaryKeysReq |
+ GetTableDefsReq |
+ GetHiddenKeysReq |
+ TruncateReq |
+ UpdateReq |
+ WriteReq |
+ NewTableIterReq |
+ ReadReq |
+ CloseTableIterReq |
+ TableStatsReq |
+ apipb.SyncLogTailReq
+}
+
+type Response interface {
+ CreateDatabaseResp |
+ OpenDatabaseResp |
+ GetDatabasesResp |
+ DeleteDatabaseResp |
+ CreateRelationResp |
+ DeleteRelationResp |
+ OpenRelationResp |
+ GetRelationsResp |
+ AddTableDefResp |
+ DelTableDefResp |
+ DeleteResp |
+ GetPrimaryKeysResp |
+ GetTableDefsResp |
+ GetHiddenKeysResp |
+ TruncateResp |
+ UpdateResp |
+ WriteResp |
+ NewTableIterResp |
+ ReadResp |
+ CloseTableIterResp |
+ TableStatsResp |
+ apipb.SyncLogTailResp
}
type CreateDatabaseReq struct {
@@ -122,8 +164,7 @@ type CreateDatabaseReq struct {
}
type CreateDatabaseResp struct {
- ID string
- ErrResp ErrorResp
+ ID ID
}
type OpenDatabaseReq struct {
@@ -132,8 +173,8 @@ type OpenDatabaseReq struct {
}
type OpenDatabaseResp struct {
- ID string
- ErrResp ErrorResp
+ ID ID
+ Name string
}
type GetDatabasesReq struct {
@@ -150,45 +191,46 @@ type DeleteDatabaseReq struct {
}
type DeleteDatabaseResp struct {
- ID string
- ErrResp ErrorResp
+ ID ID
}
type CreateRelationReq struct {
- DatabaseID string
- Name string
- Type RelationType
- Defs []engine.TableDef
+ DatabaseID ID
+ DatabaseName string
+ Name string
+ Type RelationType
+ Defs []engine.TableDef
}
type CreateRelationResp struct {
- ID string
- ErrResp ErrorResp
+ ID ID
}
type DeleteRelationReq struct {
- DatabaseID string
- Name string
+ DatabaseID ID
+ DatabaseName string
+ Name string
}
type DeleteRelationResp struct {
- ID string
- ErrResp ErrorResp
+ ID ID
}
type OpenRelationReq struct {
- DatabaseID string
- Name string
+ DatabaseID ID
+ DatabaseName string
+ Name string
}
type OpenRelationResp struct {
- ID string
- Type RelationType
- ErrResp ErrorResp
+ ID ID
+ Type RelationType
+ DatabaseName string
+ RelationName string
}
type GetRelationsReq struct {
- DatabaseID string
+ DatabaseID ID
}
type GetRelationsResp struct {
@@ -196,107 +238,108 @@ type GetRelationsResp struct {
}
type AddTableDefReq struct {
- TableID string
+ TableID ID
Def engine.TableDef
+
+ DatabaseName string
+ TableName string
}
type AddTableDefResp struct {
- ErrResp ErrorResp
}
type DelTableDefReq struct {
- TableID string
- Def engine.TableDef
+ TableID ID
+ DatabaseName string
+ TableName string
+ Def engine.TableDef
}
type DelTableDefResp struct {
- ErrResp ErrorResp
}
type DeleteReq struct {
- TableID string
- ColumnName string
- Vector *vector.Vector
+ TableID ID
+ DatabaseName string
+ TableName string
+ ColumnName string
+ Vector *vector.Vector
}
type DeleteResp struct {
- ErrResp ErrorResp
}
type GetPrimaryKeysReq struct {
- TableID string
+ TableID ID
}
type GetPrimaryKeysResp struct {
- Attrs []*engine.Attribute
- ErrResp ErrorResp
+ Attrs []*engine.Attribute
}
type GetTableDefsReq struct {
- TableID string
+ TableID ID
}
type GetTableDefsResp struct {
- Defs []engine.TableDef
- ErrResp ErrorResp
+ Defs []engine.TableDef
}
type GetHiddenKeysReq struct {
- TableID string
+ TableID ID
}
type GetHiddenKeysResp struct {
- Attrs []*engine.Attribute
- ErrResp ErrorResp
+ Attrs []*engine.Attribute
}
type TruncateReq struct {
- TableID string
+ TableID ID
+ DatabaseName string
+ TableName string
}
type TruncateResp struct {
AffectedRows int64
- ErrResp ErrorResp
}
type UpdateReq struct {
- TableID string
- Batch *batch.Batch
+ TableID ID
+ DatabaseName string
+ TableName string
+ Batch *batch.Batch
}
type UpdateResp struct {
- ErrReadOnly moerr.Error
- ErrResp ErrorResp
}
type WriteReq struct {
- TableID string
- Batch *batch.Batch
+ TableID ID
+ DatabaseName string
+ TableName string
+ Batch *batch.Batch
}
type WriteResp struct {
- ErrResp ErrorResp
}
type NewTableIterReq struct {
- TableID string
+ TableID ID
Expr *plan.Expr
Shards [][]byte
}
type NewTableIterResp struct {
- IterID string
- ErrResp ErrorResp
+ IterID ID
}
type ReadReq struct {
- IterID string
+ IterID ID
ColNames []string
}
type ReadResp struct {
- Batch *batch.Batch
- ErrResp ErrorResp
+ Batch *batch.Batch
heap *mheap.Mheap
}
@@ -313,28 +356,16 @@ func (r *ReadResp) SetHeap(heap *mheap.Mheap) {
}
type CloseTableIterReq struct {
- IterID string
+ IterID ID
}
type CloseTableIterResp struct {
- ErrResp ErrorResp
}
type TableStatsReq struct {
- TableID string
+ TableID ID
}
type TableStatsResp struct {
- Rows int
- ErrResp ErrorResp
-}
-
-type GetLogTailReq struct {
- TableID string
- Request apipb.SyncLogTailReq
-}
-
-type GetLogTailResp struct {
- ErrRelationNotFound ErrorResp
- Response apipb.SyncLogTailResp
+ Rows int
}
diff --git a/pkg/vm/engine/txn/request.go b/pkg/vm/engine/txn/request.go
index 4a6e3a925cce1f785d941589a4888ffd3a7ccb1f..9aaab02784f414e086009a884a1a23fa3f9f920b 100644
--- a/pkg/vm/engine/txn/request.go
+++ b/pkg/vm/engine/txn/request.go
@@ -18,6 +18,7 @@ import (
"bytes"
"context"
"encoding/gob"
+ "fmt"
"time"
"github.com/matrixorigin/matrixone/pkg/common/moerr"
@@ -27,8 +28,8 @@ import (
)
func DoTxnRequest[
- Resp any,
- Req any,
+ Resp Response,
+ Req Request,
](
ctx context.Context,
e engine.Engine,
@@ -79,10 +80,10 @@ func DoTxnRequest[
if err != nil {
return
}
-
for _, resp := range result.Responses {
if resp.TxnError != nil {
- err = moerr.NewTxnError("resp txnError %s", resp.TxnError.Message)
+ //TODO no way to construct moerr.Error by code and message now
+ err = fmt.Errorf("code %v, message %v", resp.TxnError.Code, resp.TxnError.Message)
return
}
}
@@ -92,17 +93,6 @@ func DoTxnRequest[
if err = gob.NewDecoder(bytes.NewReader(res.CNOpResponse.Payload)).Decode(&resp); err != nil {
return
}
-
- // XXX This code is beyond me. Why do you need to use reflects and
- // type implements for RPC code.
- // respValue := reflect.ValueOf(resp)
- // for i := 0; i < respValue.NumField(); i++ {
- // field := respValue.Field(i)
- // if field.Type().Implements(errorType) && !field.IsZero() {
- // err = moerr.NewInternalError("txn request error %d req. This error handling code is messed up.")
- // return
- // }
- // }
resps = append(resps, resp)
}
diff --git a/pkg/vm/engine/txn/shard.go b/pkg/vm/engine/txn/shard.go
index 9a833972a01c4a99bc3456ece0cf2ae877912e05..ed95d463484118aa733322c07eec9135d3e502ae 100644
--- a/pkg/vm/engine/txn/shard.go
+++ b/pkg/vm/engine/txn/shard.go
@@ -40,7 +40,7 @@ func NewDefaultShardPolicy(heap *mheap.Mheap) ShardPolicy {
type ShardPolicy interface {
Vector(
ctx context.Context,
- tableID string,
+ tableID ID,
getDefs getDefsFunc,
colName string,
vec *vector.Vector,
@@ -52,7 +52,7 @@ type ShardPolicy interface {
Batch(
ctx context.Context,
- tableID string,
+ tableID ID,
getDefs getDefsFunc,
batch *batch.Batch,
nodes []logservicepb.DNStore,
@@ -132,7 +132,7 @@ var _ ShardPolicy = new(NoShard)
func (s *NoShard) Vector(
ctx context.Context,
- tableID string,
+ tableID ID,
getDefs getDefsFunc,
colName string,
vec *vector.Vector,
@@ -151,7 +151,7 @@ func (s *NoShard) Vector(
func (s *NoShard) Batch(
ctx context.Context,
- tableID string,
+ tableID ID,
getDefs getDefsFunc,
bat *batch.Batch,
nodes []logservicepb.DNStore,
diff --git a/pkg/vm/engine/txn/shard_fallback.go b/pkg/vm/engine/txn/shard_fallback.go
index fd8f4f14a41fc714936d7d8600fbb5aa0c8d300f..9e59f7150b3ba19f06eb9ca3689e03bb677184cc 100644
--- a/pkg/vm/engine/txn/shard_fallback.go
+++ b/pkg/vm/engine/txn/shard_fallback.go
@@ -27,7 +27,7 @@ type FallbackShard []ShardPolicy
var _ ShardPolicy = FallbackShard{}
-func (f FallbackShard) Batch(ctx context.Context, tableID string, getDefs func(context.Context) ([]engine.TableDef, error), batch *batch.Batch, nodes []logservicepb.DNStore) (sharded []*ShardedBatch, err error) {
+func (f FallbackShard) Batch(ctx context.Context, tableID ID, getDefs func(context.Context) ([]engine.TableDef, error), batch *batch.Batch, nodes []logservicepb.DNStore) (sharded []*ShardedBatch, err error) {
for _, policy := range f {
sharded, err := policy.Batch(ctx, tableID, getDefs, batch, nodes)
if err != nil {
@@ -41,7 +41,7 @@ func (f FallbackShard) Batch(ctx context.Context, tableID string, getDefs func(c
panic("all shard policy failed")
}
-func (f FallbackShard) Vector(ctx context.Context, tableID string, getDefs func(context.Context) ([]engine.TableDef, error), colName string, vec *vector.Vector, nodes []logservicepb.DNStore) (sharded []*ShardedVector, err error) {
+func (f FallbackShard) Vector(ctx context.Context, tableID ID, getDefs func(context.Context) ([]engine.TableDef, error), colName string, vec *vector.Vector, nodes []logservicepb.DNStore) (sharded []*ShardedVector, err error) {
for _, policy := range f {
sharded, err := policy.Vector(ctx, tableID, getDefs, colName, vec, nodes)
if err != nil {
diff --git a/pkg/vm/engine/txn/shard_hash.go b/pkg/vm/engine/txn/shard_hash.go
index 522f09fab58d3cad1ce9dcb9a2334f9e07784254..4558cce9cbc12d008e09da15d80ea2112ad58529 100644
--- a/pkg/vm/engine/txn/shard_hash.go
+++ b/pkg/vm/engine/txn/shard_hash.go
@@ -43,7 +43,7 @@ func NewHashShard(heap *mheap.Mheap) *HashShard {
func (*HashShard) Batch(
ctx context.Context,
- tableID string,
+ tableID ID,
getDefs getDefsFunc,
bat *batch.Batch,
nodes []logservicepb.DNStore,
@@ -158,7 +158,7 @@ func (*HashShard) Batch(
func (h *HashShard) Vector(
ctx context.Context,
- tableID string,
+ tableID ID,
getDefs getDefsFunc,
colName string,
vec *vector.Vector,
@@ -583,7 +583,7 @@ func getNullableValueFromVector(vec *vector.Vector, i int) (value Nullable) {
}
- panic(fmt.Errorf("unknown column type: %v", vec.Typ))
+ panic(fmt.Sprintf("unknown column type: %v", vec.Typ))
}
func appendNullableValueToVector(vec *vector.Vector, value Nullable, heap *mheap.Mheap) {
diff --git a/pkg/vm/engine/txn/table.go b/pkg/vm/engine/txn/table.go
index b513f6623d725e3221ca77f70685a44c8d953837..3aa28aace73c6ffd99e778d1aa0f09a9884ea174 100644
--- a/pkg/vm/engine/txn/table.go
+++ b/pkg/vm/engine/txn/table.go
@@ -16,6 +16,7 @@ package txnengine
import (
"context"
+ "fmt"
"github.com/matrixorigin/matrixone/pkg/container/batch"
"github.com/matrixorigin/matrixone/pkg/container/vector"
@@ -24,9 +25,11 @@ import (
)
type Table struct {
- engine *Engine
- txnOperator client.TxnOperator
- id string
+ id ID
+ engine *Engine
+ txnOperator client.TxnOperator
+ databaseName string
+ tableName string
}
var _ engine.Relation = new(Table)
@@ -65,8 +68,10 @@ func (t *Table) AddTableDef(ctx context.Context, def engine.TableDef) error {
t.engine.allNodesShards,
OpAddTableDef,
AddTableDefReq{
- TableID: t.id,
- Def: def,
+ TableID: t.id,
+ Def: def,
+ DatabaseName: t.databaseName,
+ TableName: t.tableName,
},
)
if err != nil {
@@ -85,8 +90,10 @@ func (t *Table) DelTableDef(ctx context.Context, def engine.TableDef) error {
t.engine.allNodesShards,
OpDelTableDef,
DelTableDefReq{
- TableID: t.id,
- Def: def,
+ TableID: t.id,
+ DatabaseName: t.databaseName,
+ TableName: t.tableName,
+ Def: def,
},
)
if err != nil {
@@ -122,9 +129,11 @@ func (t *Table) Delete(ctx context.Context, vec *vector.Vector, colName string)
thisShard(shard.Shard),
OpDelete,
DeleteReq{
- TableID: t.id,
- ColumnName: colName,
- Vector: shard.Vector,
+ TableID: t.id,
+ DatabaseName: t.databaseName,
+ TableName: t.tableName,
+ ColumnName: colName,
+ Vector: shard.Vector,
},
)
if err != nil {
@@ -207,7 +216,9 @@ func (t *Table) Truncate(ctx context.Context) (uint64, error) {
t.engine.allNodesShards,
OpTruncate,
TruncateReq{
- TableID: t.id,
+ TableID: t.id,
+ DatabaseName: t.databaseName,
+ TableName: t.tableName,
},
)
if err != nil {
@@ -250,8 +261,10 @@ func (t *Table) Update(ctx context.Context, data *batch.Batch) error {
thisShard(shard.Shard),
OpUpdate,
UpdateReq{
- TableID: t.id,
- Batch: shard.Batch,
+ TableID: t.id,
+ DatabaseName: t.databaseName,
+ TableName: t.tableName,
+ Batch: shard.Batch,
},
)
if err != nil {
@@ -290,8 +303,10 @@ func (t *Table) Write(ctx context.Context, data *batch.Batch) error {
thisShard(shard.Shard),
OpWrite,
WriteReq{
- TableID: t.id,
- Batch: shard.Batch,
+ TableID: t.id,
+ DatabaseName: t.databaseName,
+ TableName: t.tableName,
+ Batch: shard.Batch,
},
)
if err != nil {
@@ -323,5 +338,5 @@ func (t *Table) GetHideKeys(ctx context.Context) (attrs []*engine.Attribute, err
}
func (t *Table) GetTableID(ctx context.Context) string {
- return t.id
+ return fmt.Sprintf("%x", t.id)
}
diff --git a/pkg/vm/engine/txn/table_reader.go b/pkg/vm/engine/txn/table_reader.go
index 77bb3993e575a02b29f1026afee4397a695eedd6..9099be7fd558f15b5c6ab4fc9008b4454e568649 100644
--- a/pkg/vm/engine/txn/table_reader.go
+++ b/pkg/vm/engine/txn/table_reader.go
@@ -33,7 +33,7 @@ type TableReader struct {
type IterInfo struct {
Shard Shard
- IterID string
+ IterID ID
}
func (t *Table) NewReader(
@@ -88,10 +88,10 @@ func (t *Table) NewReader(
return nil, err
}
- iterIDSets := make([][]string, parallel)
+ iterIDSets := make([][]ID, parallel)
i := 0
for _, resp := range resps {
- if resp.IterID != "" {
+ if resp.IterID != emptyID {
iterIDSets[i] = append(iterIDSets[i], resp.IterID)
i++
if i >= parallel {