chore: cosmetic fixes

Author: mr-karan

.gitignore

@@ -18,3 +18,6 @@ barrel*.lock
 barrel*.db
 barrel.hints
 config.toml
+bin/
+data/
+coverage.txt

Makefile

@@ -1,30 +1,30 @@
+APP-BIN := ./bin/barreldb.bin
+
 LAST_COMMIT := $(shell git rev-parse --short HEAD)
 LAST_COMMIT_DATE := $(shell git show -s --format=%ci ${LAST_COMMIT})
-VERSION := $(shell git describe --abbrev=1)
-BUILDSTR := ${VERSION} (build "\\\#"${LAST_COMMIT} $(shell date '+%Y-%m-%d %H:%M:%S'))
-
-BIN := ./bin/barreldb.bin
+VERSION := $(shell git describe --tags)
+BUILDSTR := ${VERSION} (Commit: ${LAST_COMMIT_DATE} (${LAST_COMMIT}), Build: $(shell date +"%Y-%m-%d% %H:%M:%S %z"))
 
 .PHONY: build
-build: $(BIN)
-
-$(BIN): $(shell find . -type f -name "*.go")
-	CGO_ENABLED=0 go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'" ./cmd/server/*.go
+build: ## Build binary.
+	go build -o ${APP-BIN} -ldflags="-X 'main.buildString=${BUILDSTR}'" ./cmd/server/
 
 .PHONY: run
-run:
-	CGO_ENABLED=0 go run -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'" ./cmd/server --config=cmd/server/config.toml
+run: ## Run binary.
+	./${APP-BIN} --config=./cmd/server/config.sample.toml
+
+.PHONY: clean
+clean: ## Remove temporary files and the `bin` folder.
+	rm -rf bin
+	rm -rf data/barrel_*
+
+.PHONY: fresh
+fresh: build run
 
 .PHONY: test
 test:
-	go test ./...
-
-# Use goreleaser to do a dry run producing local builds.
-.PHONY: release-dry
-release-dry:
-	goreleaser --parallelism 1 --rm-dist --snapshot --skip-validate --skip-publish
+	go test -v -failfast -race -coverpkg=./... -covermode=atomic -coverprofile=coverage.txt ./...
 
-# Use goreleaser to build production releases and publish them.
-.PHONY: release
-release:
-	goreleaser --parallelism 1 --rm-dist --skip-validate
+.PHONY: bench
+bench:
+	go test -bench=. -benchmem ./pkg/barrel/...

README.md

@@ -1,8 +1,122 @@
+<p align="center">
+<img src="./_docs/logo.png" alt="logo" width="15%" />
+</p>
+
 # barreldb
-A disk based KV store (based on Bitcask implementation)
 
-Bitcask is a log-structured hash table (LSHT) database that is designed for use as a storage engine for key-value data. It uses a log-structured approach to data management, which means that data is written to a log file and is periodically merged with other log files in the database to create new, larger log files.
+_A disk based key-value store based on [Bitcask](https://en.wikipedia.org/wiki/Bitcask)_.
+
+---
+
+BarrelDB is a Golang implementation of [Bitcask by Riak](https://riak.com/assets/bitcask-intro.pdf) paper and aims to closely follow the spec.
+
+Bitcask is based on a log-structured hash table to store key-value data on disk. It opens a "datafile" (term used for a Bitcask DB file) in an _append-only_ mode and all the writes are sequentially written to this file. Additionally, it also updates an in-memory hash table which maps the key with the offset of the record in the file. This clever yet simple design decision makes it possible to retrieve records from the disk using a _single_ disk seek.
+
+### Benefits of this approach
+
+- Low Latency: Write queries are handled with a single O(1) disk seek. Keys lookup happen in memory using a hash table lookup. This makes it possible to achieve low latency even with a lot of keys/values in the database. Bitcask also relies on the filesystem read-ahead cache for a faster reads.
+- High Throughput: Since the file is opened in "append only" mode, it can handle large volumes of write operations with ease. 
+- Predictable performance. The DB has a consistent performance even with growing number of records. This can be seen in benchmarks as well.
+- Crash friendly. Bitcask commits each record to the disk and also generates a "hints" file which makes it easy to recover in case of a crash.
+- Elegant design. Bitcask achieves a lot just by keeping the architecture simple and relying on filesystem primitives for complex scenarios (backup/recovery, cache etc).
+- Ability to handle datasets larger than RAM.
+
+### Limitations
+
+- The main limitation is that all the keys must fit in RAM since they're held inside as an in-memory hash table. A potential workaround for this could be to shard the keys in multiple buckets. Incoming records can be hashed into different buckets based on the key. A shard based approach allows each bucket to have limited RAM usage.
+
+## Internals
+
+You can refer to https://mrkaran.dev/ for a post which goes over the internals of Bitcask and also explains how BarrelDB works.
+
+## Usage
+
+### Library
+
+
+```go
+import (
+	"github.com/mr-karan/barreldb/pkg/barrel"
+)
+
+barrel, _ := barrel.Init(barrel.WithDir("data/"))
+
+// Set a key.
+barrel.Put("hello", []byte("world"))
+
+// Fetch the key.
+v, _ := barrel.Get("hello")
+
+// Delete a key.
+barrel.Delete("hello")
+
+// Set with expiry.
+barrel.PutEx("hello", []byte("world"), time.Second * 3)
+```
+
+For a complete example, visit [examples](./examples/main.go).
+
+### Redis Client
+
+`barreldb` implements the API over a simple Redis-compatible server (`barreldb`):
+
+```
+127.0.0.1:6379> set hello world
+OK
+127.0.0.1:6379> get hello
+"world"
+127.0.0.1:6379> set goodbye world 10s
+OK
+127.0.0.1:6379> get goodbye
+"world"
+127.0.0.1:6379> get goodbye
+ERR: invalid key: key is already expired
+```
+
+## Benchmarks
+
+Using `make bench`:
+
+```
+go test -bench=. -benchmem ./pkg/barrel/...
+HELLO
+goos: linux
+goarch: amd64
+pkg: github.com/mr-karan/barreldb/pkg/barrel
+cpu: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz
+BenchmarkPut/DisableSync-8                385432              3712 ns/op        1103.48 MB/s          88 B/op          4 allocs/op
+BenchmarkPut/AlwaysSync-8                    222           5510563 ns/op           0.74 MB/s         115 B/op          4 allocs/op
+BenchmarkGet-8                            840627              1304 ns/op        3142.20 MB/s        4976 B/op          5 allocs/op
+PASS
+ok      github.com/mr-karan/barreldb/pkg/barrel 10.751s
+```
+
+Using `redis-benchmark`:
+
+```
+$ redis-benchmark -p 6379 -t set -n 10000 -r 100000000
+Summary:
+  throughput summary: 140845.06 requests per second
+  latency summary (msec):
+          avg       min       p50       p95       p99       max
+        0.196     0.016     0.175     0.255     1.031     2.455
+
+$ redis-benchmark -p 6379 -t set -n 200000 -r 100000000
+Summary:
+  throughput summary: 143678.17 requests per second
+  latency summary (msec):
+          avg       min       p50       p95       p99       max
+        0.184     0.016     0.183     0.223     0.455     2.183
+
+$ redis-benchmark -p 6379 -t get -n 100000 -r 100000000
+Summary:
+  throughput summary: 170068.03 requests per second
+  latency summary (msec):
+          avg       min       p50       p95       p99       max
+        0.153     0.040     0.143     0.199     0.367     1.447
+```
 
-The process of merging log files in Bitcask is called compaction. When compaction occurs, the Bitcask database will select two or more log files to merge, and it will create a new log file that contains the combined data from the selected log files. This new log file will be used in place of the old log files, and the old log files will be deleted to free up space.
+## References
 
-The advantage of this approach is that it allows Bitcask to store data efficiently and to perform well even when dealing with large amounts of data. By periodically merging log files, Bitcask can avoid the need to split or resize its data files, which can be slow and expensive operations. It also allows Bitcask to perform efficient lookups of data by key, since the keys are stored in a hash table that is stored in memory.
+- [Bitcask paper](https://riak.com/assets/bitcask-intro.pdf)
+- [Highscalability article on Bitcask](http://highscalability.com/blog/2011/1/10/riaks-bitcask-a-log-structured-hash-table-for-fast-keyvalue.html)

TODO.md

@@ -50,4 +50,3 @@ Availability with N+1 node with raft
 - [ ] Merge
 - [ ] Hints file
 - [ ] Rotate size
-

_docs/logo.png

@@ -3,10 +3,5 @@ address = ":6379"
 
 [app]
 debug = false # Enable debug logging
-dir = "/data/barreldb" # Directory to store .db files
+dir = "./data" # Directory to store .db files
 read_only = false # Whether to run barreldb in a read only mode. Write operations are not allowed in this mode.
-always_fsync = false # Whether to call `fsync(2)` on every write call. This significantly affects the performance but if data durability is a big concern, consider turning it on. 
-fsync_interval = "5s" # If always_fsync is turned off, barreldb can flush filesystem buffers periodically at this given interval.
-max_file_size = 1000000 # Maximum size of one datafile (.db file) in bytes. After this size has reached, it'll get rotated and a new .db file will be used for storing newer data.
-eval_file_size_interval = "1m" # Periodic interval to check if the size of the active .db file has reached `max_file_size`.
-compaction_interval = "6h" # Periodic interval to perform compaction for optimising disk usage. It cleans up deleted/expired keys and merges old datafiles into a single file.

cmd/server/config.sample.toml

@@ -1,46 +1,72 @@
 package main
 
 import (
-	"fmt"
+	"log"
+	"os"
 	"time"
 
 	"github.com/mr-karan/barreldb/pkg/barrel"
 )
 
+var (
+	lo = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
+)
+
 func main() {
-	barrel, err := barrel.Init()
+	// Initialise.
+	barrel, err := barrel.Init(barrel.WithDir("data/"), barrel.WithAutoSync())
 	if err != nil {
-		panic(err)
+		lo.Fatalf("error initialising barrel: %v", err)
 	}
 
-	if err := barrel.PutEx("hello", []byte("world"), time.Second*5); err != nil {
-		panic(err)
-	}
-	if err := barrel.Put("good", []byte("bye")); err != nil {
-		panic(err)
+	var (
+		key = "hello"
+		val = []byte("world")
+	)
+
+	// Set a key.
+	if err := barrel.Put(key, val); err != nil {
+		lo.Fatalf("error setting a key: %v", err)
 	}
 
-	val, err := barrel.Get("hello")
+	// Fetch the key.
+	v, err := barrel.Get(key)
 	if err != nil {
-		panic(err)
+		lo.Fatalf("error fetching key %s: %v", key, err)
+	}
+	lo.Printf("fetched val: %s\n", string(v))
+
+	// Set a new key with an expiry.
+	key = "fruit"
+	val = []byte("apple")
+	ex := time.Second * 2
+	if err := barrel.PutEx(key, val, ex); err != nil {
+		lo.Fatalf("error setting a key with ex: %v", err)
 	}
-	fmt.Println(string(val))
 
-	val, err = barrel.Get("good")
+	// Wait for 3 seconds for expiry.
+	wait := time.Second * 3
+	lo.Printf("waiting for %s for the key to get expired", wait.String())
+	time.Sleep(wait)
+
+	// Try fetching the expired key.
+	_, err = barrel.Get(key)
 	if err != nil {
-		panic(err)
+		lo.Printf("error fetching key %s: %v\n", key, err)
 	}
 
-	fmt.Println(string(val))
+	// Delete the key.
+	if err := barrel.Delete(key); err != nil {
+		lo.Fatalf("error deleting key %s: %v", key, err)
+	}
 
+	// Fetch list of keys.
 	keys := barrel.List()
-	fmt.Println(keys)
-
-	val, err = barrel.Get("hello")
-	if err != nil {
-		panic(err)
+	for i, k := range keys {
+		lo.Printf("key %d is %s\n", i, k)
 	}
-	fmt.Println(string(val))
 
-	barrel.Shutdown()
+	if err := barrel.Shutdown(); err != nil {
+		lo.Fatalf("error closing barrel: %v", err)
+	}
 }

examples/main.go

@@ -1,7 +1,9 @@
 package barrel
 
 import (
+	"fmt"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -114,6 +116,15 @@ func TestAPI(t *testing.T) {
 		assert.Equal(len, 1)
 	})
 
+	t.Run("Fold", func(t *testing.T) {
+		upper := func(s string) error {
+			fmt.Println(strings.ToUpper(s))
+			return nil
+		}
+		err = brl.Fold(upper)
+		assert.NoError(err)
+	})
+
 	t.Run("Expiry", func(t *testing.T) {
 		err = brl.PutEx("keywithexpiry", []byte("valwithex"), time.Second*2)
 		assert.NoError(err)