Merge pull request #80 from qubic/dev

Merge dev into main

.github/workflows/push-empty-ticks-endpoint.yaml → .github/workflows/push-db-compactions.yaml

@@ -3,7 +3,7 @@ name: Deploy dev images to GHCR
 on:
   push:
     branches:
-      - 'empty-ticks-endpoint'
+      - 'db-compactions'
 
 jobs:
   push-store-image:

.github/workflows/push-interval-quorum-data.yaml

@@ -0,0 +1,25 @@
+name: Deploy dev images to GHCR
+
+on:
+  push:
+    branches:
+      - 'interval-quorum-data'
+
+jobs:
+  push-store-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout GitHub Action'
+        uses: actions/checkout@main
+
+      - name: 'Login to GitHub Container Registry'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: 'Build Inventory Image'
+        run: |
+          docker build . --tag ghcr.io/qubic/qubic-archiver:${{github.ref_name}}
+          docker push ghcr.io/qubic/qubic-archiver:${{github.ref_name}}

main.go

@@ -13,6 +13,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"runtime"
 	"syscall"
 	"time"
 )
@@ -80,19 +81,52 @@ func run() error {
 	}
 	log.Printf("main: Config :\n%v\n", out)
 
-	levelOptions := pebble.LevelOptions{
+	l1Options := pebble.LevelOptions{
+		BlockRestartInterval: 16,
+		BlockSize:            4096,
+		BlockSizeThreshold:   90,
+		Compression:          pebble.NoCompression,
+		FilterPolicy:         nil,
+		FilterType:           pebble.TableFilter,
+		IndexBlockSize:       4096,
+		TargetFileSize:       268435456, // 256 MB
+	}
+	l2Options := pebble.LevelOptions{
+		BlockRestartInterval: 16,
+		BlockSize:            4096,
+		BlockSizeThreshold:   90,
+		Compression:          pebble.ZstdCompression,
+		FilterPolicy:         nil,
+		FilterType:           pebble.TableFilter,
+		IndexBlockSize:       4096,
+		TargetFileSize:       l1Options.TargetFileSize * 10, // 2.5 GB
+	}
+	l3Options := pebble.LevelOptions{
+		BlockRestartInterval: 16,
+		BlockSize:            4096,
+		BlockSizeThreshold:   90,
+		Compression:          pebble.ZstdCompression,
+		FilterPolicy:         nil,
+		FilterType:           pebble.TableFilter,
+		IndexBlockSize:       4096,
+		TargetFileSize:       l2Options.TargetFileSize * 10, // 25 GB
+	}
+	l4Options := pebble.LevelOptions{
 		BlockRestartInterval: 16,
 		BlockSize:            4096,
 		BlockSizeThreshold:   90,
 		Compression:          pebble.ZstdCompression,
 		FilterPolicy:         nil,
 		FilterType:           pebble.TableFilter,
 		IndexBlockSize:       4096,
-		TargetFileSize:       2097152,
+		TargetFileSize:       l3Options.TargetFileSize * 10, // 250 GB
 	}
 
 	pebbleOptions := pebble.Options{
-		Levels: []pebble.LevelOptions{levelOptions},
+		Levels:                   []pebble.LevelOptions{l1Options, l2Options, l3Options, l4Options},
+		MaxConcurrentCompactions: func() int { return runtime.NumCPU() },
+		MemTableSize:             268435456, // 256 MB
+		EventListener:            store.NewPebbleEventListener(),
 	}
 
 	db, err := pebble.Open(cfg.Qubic.StorageFolder, &pebbleOptions)