Skip to content

Commit

Permalink
Remove generateDocids and generateStoreID
Browse files Browse the repository at this point in the history
  • Loading branch information
asdine committed Sep 1, 2020
1 parent 31c774f commit 676945b
Show file tree
Hide file tree
Showing 5 changed files with 34 additions and 180 deletions.
72 changes: 25 additions & 47 deletions database/config.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
package database

import (
"bytes"
"crypto/rand"
"encoding/binary"
"fmt"
"sync"
"time"

"github.com/genjidb/genji/document"
"github.com/genjidb/genji/document/encoding/msgpack"
"github.com/genjidb/genji/engine"
"github.com/genjidb/genji/index"
"github.com/genjidb/genji/key"
)

const storePrefix = 't'
Expand Down Expand Up @@ -67,11 +63,13 @@ func (f *FieldConstraint) ScanDocument(d document.Document) error {
return nil
}

// TableInfo contains information about a table.
type TableInfo struct {
// name of the table.
tableName string
// storeID is used as a key to reference a table.
storeID []byte
readOnly bool
// name of the store associated with the table.
storeName []byte
readOnly bool
// if non-zero, this tableInfo has been created during the current transaction.
// it will be removed if the transaction is rolled back or set to false if its commited.
transactionID int64
Expand All @@ -91,11 +89,12 @@ func (ti *TableInfo) GetPrimaryKey() *FieldConstraint {
return nil
}

// ToDocument turns ti into a document.
func (ti *TableInfo) ToDocument() document.Document {
buf := document.NewFieldBuffer()

buf.Add("table_name", document.NewTextValue(ti.tableName))
buf.Add("store_id", document.NewBlobValue(ti.storeID))
buf.Add("store_name", document.NewBlobValue(ti.storeName))

vbuf := document.NewValueBuffer()
for _, fc := range ti.FieldConstraints {
Expand All @@ -108,19 +107,20 @@ func (ti *TableInfo) ToDocument() document.Document {
return buf
}

// ScanDocument decodes d into ti.
func (ti *TableInfo) ScanDocument(d document.Document) error {
v, err := d.GetByField("table_name")
if err != nil {
return err
}
ti.tableName = v.V.(string)

v, err = d.GetByField("store_id")
v, err = d.GetByField("store_name")
if err != nil {
return err
}
ti.storeID = make([]byte, len(v.V.([]byte)))
copy(ti.storeID, v.V.([]byte))
ti.storeName = make([]byte, len(v.V.([]byte)))
copy(ti.storeName, v.V.([]byte))

v, err = d.GetByField("field_constraints")
if err != nil {
Expand Down Expand Up @@ -173,6 +173,7 @@ func newTableInfoStore(tx engine.Transaction) (*tableInfoStore, error) {
}

// Insert a new tableInfo for the given table name.
// If info.storeName is nil, it generates one and stores it in info.
func (t *tableInfoStore) Insert(tx *Transaction, tableName string, info *TableInfo) error {
t.mu.Lock()
defer t.mu.Unlock()
Expand All @@ -188,12 +189,20 @@ func (t *tableInfoStore) Insert(tx *Transaction, tableName string, info *TableIn
return ErrTableAlreadyExists
}

v, err := msgpack.EncodeDocument(info.ToDocument())
st, err := tx.Tx.GetStore([]byte(tableInfoStoreName))
if err != nil {
return err
}

st, err := tx.Tx.GetStore([]byte(tableInfoStoreName))
if info.storeName == nil {
seq, err := st.NextSequence()
if err != nil {
return err
}
info.storeName = key.AppendInt64([]byte{storePrefix}, int64(seq))
}

v, err := msgpack.EncodeDocument(info.ToDocument())
if err != nil {
return err
}
Expand Down Expand Up @@ -287,8 +296,8 @@ func (t *tableInfoStore) loadAllTableInfo(tx engine.Transaction) error {
}

t.tableInfos[tableInfoStoreName] = TableInfo{
storeID: []byte(tableInfoStoreName),
readOnly: true,
storeName: []byte(tableInfoStoreName),
readOnly: true,
}
return nil
}
Expand Down Expand Up @@ -334,37 +343,6 @@ func (t *tableInfoStore) GetTableInfo() map[string]TableInfo {
return ti
}

// generateStoreID generates an ID used as a key to reference a table.
// The first byte contains the prefix used for table stores.
// The next 4 bytes represent the timestamp in second and the last 2 bytes are
// randomly generated.
func (t *tableInfoStore) generateStoreID() []byte {
t.mu.RLock()
defer t.mu.RUnlock()

var found bool = true
var id [7]byte
for found {
id[0] = storePrefix
binary.BigEndian.PutUint32(id[1:], uint32(time.Now().Unix()))
if _, err := rand.Reader.Read(id[5:]); err != nil {
panic(fmt.Errorf("cannot generate random number: %v;", err))
}

found = false
for _, ti := range t.tableInfos {
if bytes.Equal(ti.storeID, id[:]) {
// A store with this id already exists.
// Let's generate a new one.
found = true
break
}
}
}

return id[:]
}

// IndexConfig holds the configuration of an index.
type IndexConfig struct {
TableName string
Expand Down
12 changes: 1 addition & 11 deletions database/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
package database

import (
"sync"
"sync/atomic"

"github.com/genjidb/genji/engine"
Expand All @@ -12,17 +11,9 @@ import (
type Database struct {
ng engine.Engine

mu sync.Mutex

// tableInfoStore manages information about all the tables
tableInfoStore *tableInfoStore

// tableDocids holds the latest docid for a table.
// it is cached in this map the first time a table is accessed
// and is used by every call to table#Insert to generate the
// docid if the table doesn't have a primary key.
tableDocids map[string]int64

// This stores the last transaction id created.
// It starts at 0 at database startup and is
// incremented atomically every time Begin is called.
Expand All @@ -32,8 +23,7 @@ type Database struct {
// New initializes the DB using the given engine.
func New(ng engine.Engine) (*Database, error) {
db := Database{
ng: ng,
tableDocids: make(map[string]int64),
ng: ng,
}

ntx, err := db.ng.Begin(true)
Expand Down
75 changes: 2 additions & 73 deletions database/table.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package database

import (
"bytes"
"errors"
"fmt"
"math"

"github.com/genjidb/genji/document"
"github.com/genjidb/genji/document/encoding/msgpack"
Expand Down Expand Up @@ -386,81 +384,12 @@ func (t *Table) generateKey(d document.Document) ([]byte, error) {
return key.AppendValue(nil, v), nil
}

docid, err := t.generateDocid()
docid, err := t.Store.NextSequence()
if err != nil {
return nil, err
}

return key.AppendInt64(nil, docid), nil
}

// this function looks up for the highest key in the table,
// increments it, caches it in the database tableDocids map
// and returns it.
// if the docid is greater than max.Int64, it looks up for the lowest
// available docid in the table.
// Generating a docid is safe for concurrent access across
// multiple transactions.
func (t *Table) generateDocid() (int64, error) {
t.tx.db.mu.Lock()
defer t.tx.db.mu.Unlock()

var err error

// get the cached latest docid
lastDocid, ok := t.tx.db.tableDocids[t.name]
// if no key was found in the cache, get the largest key in the table
if !ok {
it := t.Store.NewIterator(engine.IteratorConfig{Reverse: true})
it.Seek(nil)
if it.Valid() {
t.tx.db.tableDocids[t.name], err = key.DecodeInt64(it.Item().Key())
if err != nil {
it.Close()
return 0, err
}
} else {
t.tx.db.tableDocids[t.name] = 0
}

err = it.Close()
if err != nil {
return 0, err
}

lastDocid = t.tx.db.tableDocids[t.name]
}

// if the id is bigger than an int64
// look for the smallest available docid
if lastDocid > math.MaxInt64-1 {
return t.getSmallestAvailableDocid()
}

lastDocid++

// cache it
t.tx.db.tableDocids[t.name] = lastDocid
return lastDocid, nil
}

// getSmallestAvailableDocid iterates through the table store
// and returns the first available docid.
func (t *Table) getSmallestAvailableDocid() (int64, error) {
it := t.Store.NewIterator(engine.IteratorConfig{})
defer it.Close()

var i int64 = 1

var buf [8]byte
for it.Seek(nil); it.Valid(); it.Next() {
if !bytes.Equal(it.Item().Key(), key.AppendInt64(buf[:0], i)) {
return i, nil
}
i++
}

return 0, errors.New("reached maximum number of documents in a table")
return key.AppendInt64(nil, int64(docid)), nil
}

// ValidateConstraints check the table configuration for constraints and validates the document
Expand Down
48 changes: 3 additions & 45 deletions database/table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,10 @@ package database_test
import (
"errors"
"fmt"
"math"
"testing"

"github.com/genjidb/genji/database"
"github.com/genjidb/genji/document"
"github.com/genjidb/genji/document/encoding/msgpack"
"github.com/genjidb/genji/engine/memoryengine"
"github.com/genjidb/genji/key"
"github.com/genjidb/genji/sql/parser"
Expand Down Expand Up @@ -178,61 +176,21 @@ func TestTableInsert(t *testing.T) {
key1 := insertDoc(db)
require.NoError(t, err)

// create new database
// create new database object
db, err = database.New(ng)
require.NoError(t, err)

key2 := insertDoc(db)

a, err := key.DecodeInt64(key1)
a, err := key.DecodeUint64(key1)
require.NoError(t, err)

b, err := key.DecodeInt64(key2)
b, err := key.DecodeUint64(key2)
require.NoError(t, err)

require.Equal(t, a+1, b)
})

t.Run("Should lookup for the smallest available docid if int64 overflowed", func(t *testing.T) {
tx, cleanup := newTestDB(t)
defer cleanup()

err := tx.CreateTable("test", nil)
require.NoError(t, err)

tb, err := tx.GetTable("test")
require.NoError(t, err)

manualInsert := func(id int64) {
docid := key.AppendInt64(nil, id)
v, err := msgpack.EncodeDocument(newDocument())
require.NoError(t, err)
err = tb.Store.Put(docid, v)
require.NoError(t, err)
}

// insert manually a document with the maximum docid
manualInsert(math.MaxInt64)
// insert manually a document with docid = 2
manualInsert(2)

expectDocid := func(want int64, got []byte) {
newDocid, err := key.DecodeInt64(got)
require.NoError(t, err)
require.Equal(t, want, newDocid)
}

// now insert a document and expect its docid to be 1
key, err := tb.Insert(newDocument())
require.NoError(t, err)
expectDocid(1, key)

// insert another one and expect its docid to be 3
key, err = tb.Insert(newDocument())
require.NoError(t, err)
expectDocid(3, key)
})

t.Run("Should use the right field if primary key is specified", func(t *testing.T) {
tx, cleanup := newTestDB(t)
defer cleanup()
Expand Down
7 changes: 3 additions & 4 deletions database/transaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,12 @@ func (tx *Transaction) CreateTable(name string, info *TableInfo) error {
}

info.tableName = name
info.storeID = tx.tableInfoStore.generateStoreID()
err := tx.tableInfoStore.Insert(tx, name, info)
if err != nil {
return err
}

err = tx.Tx.CreateStore(info.storeID)
err = tx.Tx.CreateStore(info.storeName)
if err != nil {
return fmt.Errorf("failed to create table %q: %w", name, err)
}
Expand All @@ -85,7 +84,7 @@ func (tx *Transaction) GetTable(name string) (*Table, error) {
return nil, err
}

s, err := tx.Tx.GetStore(ti.storeID)
s, err := tx.Tx.GetStore(ti.storeName)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -186,7 +185,7 @@ func (tx *Transaction) DropTable(name string) error {
return err
}

return tx.Tx.DropStore(ti.storeID)
return tx.Tx.DropStore(ti.storeName)
}

// CreateIndex creates an index with the given name.
Expand Down

0 comments on commit 676945b

Please sign in to comment.