-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: initial ledger state database (#77)
This creates the initial ledger state database with support for storing and retrieving blocks
- Loading branch information
Showing
8 changed files
with
539 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,171 @@ | ||
| // Copyright 2024 Blink Labs Software | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package database | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "io/fs" | ||
| "log/slog" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| badger "github.com/dgraph-io/badger/v4" | ||
| "github.com/glebarez/sqlite" | ||
| "gorm.io/gorm" | ||
| ) | ||
|
|
||
| type Database interface { | ||
| Metadata() *gorm.DB | ||
| Blob() *badger.DB | ||
| } | ||
|
|
||
| type BaseDatabase struct { | ||
| logger *slog.Logger | ||
| metadata *gorm.DB | ||
| blob *badger.DB | ||
| blobGcTimer *time.Ticker | ||
| } | ||
|
|
||
| // Metadata returns the underlying metadata DB instance | ||
| func (b *BaseDatabase) Metadata() *gorm.DB { | ||
| return b.metadata | ||
| } | ||
|
|
||
| // Blob returns the underling blob DB instance | ||
| func (b *BaseDatabase) Blob() *badger.DB { | ||
| return b.blob | ||
| } | ||
|
|
||
| func (b *BaseDatabase) init() error { | ||
| if b.logger == nil { | ||
| // Create logger to throw away logs | ||
| // We do this so we don't have to add guards around every log operation | ||
| b.logger = slog.New(slog.NewJSONHandler(io.Discard, nil)) | ||
| } | ||
| // Run GC periodically for Badger DB | ||
| b.blobGcTimer = time.NewTicker(5 * time.Minute) | ||
| go b.blobGc() | ||
| return nil | ||
| } | ||
|
|
||
| func (b *BaseDatabase) blobGc() { | ||
| for range b.blobGcTimer.C { | ||
| again: | ||
| err := b.blob.RunValueLogGC(0.5) | ||
| if err != nil { | ||
| // Log any actual errors | ||
| if !errors.Is(err, badger.ErrNoRewrite) { | ||
| b.logger.Warn( | ||
| fmt.Sprintf("blob DB: GC failure: %s", err), | ||
| ) | ||
| } | ||
| } else { | ||
| // Run it again if it just ran successfully | ||
| goto again | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // InMemoryDatabase stores all data in memory. Data will not be persisted | ||
| type InMemoryDatabase struct { | ||
| *BaseDatabase | ||
| } | ||
|
|
||
| // NewInMemory creates a new in-memory database | ||
| func NewInMemory(logger *slog.Logger) (*InMemoryDatabase, error) { | ||
| // Open sqlite DB | ||
| metadataDb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // Open Badger DB | ||
| badgerOpts := badger.DefaultOptions(""). | ||
| WithLogger(NewBadgerLogger(logger)). | ||
| // The default INFO logging is a bit verbose | ||
| WithLoggingLevel(badger.WARNING). | ||
| WithInMemory(true) | ||
| blobDb, err := badger.Open(badgerOpts) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| db := &InMemoryDatabase{ | ||
| BaseDatabase: &BaseDatabase{ | ||
| logger: logger, | ||
| metadata: metadataDb, | ||
| blob: blobDb, | ||
| }, | ||
| } | ||
| if err := db.init(); err != nil { | ||
| return nil, err | ||
| } | ||
| return db, nil | ||
| } | ||
|
|
||
| // PersistentDatabase stores its data on disk, providing persistence across restarts | ||
| type PersistentDatabase struct { | ||
| *BaseDatabase | ||
| dataDir string | ||
| } | ||
|
|
||
| // NewPersistent creates a new persistent database instance using the provided data directory | ||
| func NewPersistent(dataDir string, logger *slog.Logger) (*PersistentDatabase, error) { | ||
| // Make sure that we can read data dir, and create if it doesn't exist | ||
| if _, err := os.Stat(dataDir); err != nil { | ||
| if !errors.Is(err, fs.ErrNotExist) { | ||
| return nil, fmt.Errorf("failed to read data dir: %w", err) | ||
| } | ||
| // Create data directory | ||
| if err := os.MkdirAll(dataDir, fs.ModePerm); err != nil { | ||
| return nil, fmt.Errorf("failed to create data dir: %w", err) | ||
| } | ||
| } | ||
| // Open sqlite DB | ||
| metadataDbPath := filepath.Join( | ||
| dataDir, | ||
| "metadata.sqlite", | ||
| ) | ||
| metadataDb, err := gorm.Open(sqlite.Open(metadataDbPath), &gorm.Config{}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // Open Badger DB | ||
| blobDir := filepath.Join( | ||
| dataDir, | ||
| "blob", | ||
| ) | ||
| badgerOpts := badger.DefaultOptions(blobDir). | ||
| WithLogger(NewBadgerLogger(logger)). | ||
| // The default INFO logging is a bit verbose | ||
| WithLoggingLevel(badger.WARNING) | ||
| blobDb, err := badger.Open(badgerOpts) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| db := &PersistentDatabase{ | ||
| BaseDatabase: &BaseDatabase{ | ||
| logger: logger, | ||
| metadata: metadataDb, | ||
| blob: blobDb, | ||
| }, | ||
| dataDir: dataDir, | ||
| } | ||
| if err := db.init(); err != nil { | ||
| return nil, err | ||
| } | ||
| return db, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| // Copyright 2024 Blink Labs Software | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package database | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "log/slog" | ||
| ) | ||
|
|
||
| // BadgerLogger is a wrapper type to give our logger the expected interface | ||
| type BadgerLogger struct { | ||
| logger *slog.Logger | ||
| } | ||
|
|
||
| func NewBadgerLogger(logger *slog.Logger) *BadgerLogger { | ||
| if logger == nil { | ||
| // Create logger to throw away logs | ||
| // We do this so we don't have to add guards around every log operation | ||
| logger = slog.New(slog.NewJSONHandler(io.Discard, nil)) | ||
| } | ||
| return &BadgerLogger{logger: logger} | ||
| } | ||
|
|
||
| func (b *BadgerLogger) Infof(msg string, args ...any) { | ||
| b.logger.Info( | ||
| fmt.Sprintf(msg, args...), | ||
| ) | ||
| } | ||
|
|
||
| func (b *BadgerLogger) Warningf(msg string, args ...any) { | ||
| b.logger.Warn( | ||
| fmt.Sprintf(msg, args...), | ||
| ) | ||
| } | ||
|
|
||
| func (b *BadgerLogger) Debugf(msg string, args ...any) { | ||
| b.logger.Debug( | ||
| fmt.Sprintf(msg, args...), | ||
| ) | ||
| } | ||
|
|
||
| func (b *BadgerLogger) Errorf(msg string, args ...any) { | ||
| b.logger.Error( | ||
| fmt.Sprintf(msg, args...), | ||
| ) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.