Merge pull request #559 from fairDataSociety/staging

Staging

.github/workflows/coverge.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20.1'
+          go-version: '1.21.4'
       - name: Checkout
         uses: actions/checkout@v3
         with:
@@ -24,6 +24,6 @@ jobs:
         env:
           DEEPSOURCE_DSN: ${{ secrets.DEEPSOURCE_DSN }}
         run: |
-          go test -coverprofile=cover.out $(go list ./... | grep -v wasm)
+          go test -coverprofile=cover.out $(go list ./... | grep -v wasm) -timeout=30m
           curl https://deepsource.io/cli | sh
           ./bin/deepsource report --analyzer test-coverage --key go --value-file ./cover.out

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go: ['1.20.1']
+        go: ['1.21.4']
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
     - name: Setup Go

.github/workflows/release.yaml

@@ -18,7 +18,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20.1'
+          go-version: '1.21.4'
       - name: Checkout
         uses: actions/checkout@v3
         with:
@@ -45,7 +45,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20.1'
+          go-version: '1.21.4'
       - name: Checkout
         uses: actions/checkout@v3
         with:
@@ -85,7 +85,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20.1'
+          go-version: '1.21.4'
       - name: Checkout
         uses: actions/checkout@v3
         with:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20.2-alpine AS build
+FROM golang:1.21.4-alpine AS build
 
 WORKDIR /usr/fairos
 COPY go.mod go.sum /usr/fairos/

Makefile

@@ -1,6 +1,6 @@
 GO ?= go
 GOLANGCI_LINT ?= $$($(GO) env GOPATH)/bin/golangci-lint
-GOLANGCI_LINT_VERSION ?= v1.52.2
+GOLANGCI_LINT_VERSION ?= v1.55.2
 GOGOPROTOBUF ?= protoc-gen-gogofaster
 GOGOPROTOBUF_VERSION ?= v1.3.1
 
@@ -41,11 +41,11 @@ vet:
 
 .PHONY: test-race
 test-race:
-	$(GO) test -race -timeout 1000000ms -v "$(DEST)"
+	$(GO) test -race -timeout 30m -v "$(DEST)"
 
 .PHONY: test
 test:
-	$(GO) test -v "$(DEST)"
+	$(GO) test -v "$(DEST)" -timeout=30m
 
 .PHONY: githooks
 githooks:
@@ -74,7 +74,7 @@ release:
 		-v `pwd`:/go/src/github.com/fairDataSociety/fairOS-dfs \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-w /go/src/github.com/fairDataSociety/fairOS-dfs \
-		ghcr.io/goreleaser/goreleaser-cross:v1.20.2 release --rm-dist
+		ghcr.io/goreleaser/goreleaser-cross:v1.21.0 release --rm-dist
 
 .PHONY: release-dry-run
 release-dry-run:
@@ -84,7 +84,7 @@ release-dry-run:
 		-v `pwd`:/go/src/github.com/fairDataSociety/fairOS-dfs \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		-w /go/src/github.com/fairDataSociety/fairOS-dfs \
-		ghcr.io/goreleaser/goreleaser-cross:v1.20.2 release --rm-dist \
+		ghcr.io/goreleaser/goreleaser-cross:v1.21.0 release --rm-dist \
 		--skip-validate=true \
 		--skip-publish
 

cmd/dfs/cmd/config.go

@@ -12,6 +12,8 @@ var (
 	optionVerbosity          = "verbosity"
 	optionBeeApi             = "bee.bee-api-endpoint"
 	optionBeePostageBatchId  = "bee.postage-batch-id"
+	optionFeedCacheSize      = "feed.cache-size"
+	optionFeedCacheTTL       = "feed.cache-ttl"
 	optionCookieDomain       = "cookie-domain"
 	optionNetwork            = "ens-network"
 	optionRPC                = "rpc"

cmd/dfs/cmd/dev.go

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"syscall"
+	"testing"
+
+	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
+	mockstorer "github.com/ethersphere/bee/pkg/storer/mock"
+	"github.com/fairdatasociety/fairOS-dfs/pkg/api"
+	"github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee"
+	"github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock"
+	"github.com/fairdatasociety/fairOS-dfs/pkg/dfs"
+	mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock"
+	"github.com/fairdatasociety/fairOS-dfs/pkg/logging"
+	"github.com/fairdatasociety/fairOS-dfs/pkg/user"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var devCmd = &cobra.Command{
+	Use:   "dev",
+	Short: "run fairOS-dfs in development mode",
+	Run: func(cmd *cobra.Command, args []string) {
+		startDevServer()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(devCmd)
+}
+
+func startDevServer() {
+	storer := mockstorer.New()
+	t := &testing.T{}
+	fmt.Println(`▓█████▄ ▓█████ ██▒   █▓
+▒██▀ ██▌▓█   ▀▓██░   █▒
+░██   █▌▒███   ▓██  █▒░
+░▓█▄   ▌▒▓█  ▄  ▒██ █░░
+░▒████▓ ░▒████▒  ▒▀█░  
+ ▒▒▓  ▒ ░░ ▒░ ░  ░ ▐░  
+ ░ ▒  ▒  ░ ░  ░  ░ ░░  
+ ░ ░  ░    ░       ░░  
+   ░       ░  ░     ░  
+ ░                 ░   `)
+	beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{
+		Storer:          storer,
+		PreventRedirect: true,
+		Post:            mockpost.New(mockpost.WithAcceptAll()),
+	})
+	fmt.Println("Bee running at: ", beeUrl)
+	logger := logging.New(io.Discard, logrus.DebugLevel)
+	mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger)
+	ens := mock2.NewMockNamespaceManager()
+
+	users := user.NewUsers(mockClient, ens, -1, 0, logger)
+	dfsApi := dfs.NewMockDfsAPI(mockClient, users, logger)
+	handler = api.NewMockHandler(dfsApi, logger, []string{"http://localhost:3000"})
+	defer handler.Close()
+	httpPort = ":9090"
+	pprofPort = ":9091"
+	srv := startHttpService(logger)
+	fmt.Printf("Server running at:http://127.0.0.1%s\n", httpPort)
+	defer func() {
+		err := srv.Shutdown(context.TODO())
+		if err != nil {
+			logger.Error("failed to shutdown server", err.Error())
+		}
+	}()
+	done := make(chan os.Signal, 1)
+	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
+
+	<-done
+}

cmd/dfs/cmd/server.go

@@ -80,6 +80,15 @@ can consume it.`,
 		if err := config.BindPFlag(optionDFSPprofPort, cmd.Flags().Lookup("pprofPort")); err != nil {
 			return err
 		}
+		if err := config.BindPFlag(optionFeedCacheSize, cmd.Flags().Lookup("feedCacheSize")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionFeedCacheTTL, cmd.Flags().Lookup("feedCacheTTL")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionDFSPprofPort, cmd.Flags().Lookup("pprofPort")); err != nil {
+			return err
+		}
 		if err := config.BindPFlag(optionCookieDomain, cmd.Flags().Lookup("cookieDomain")); err != nil {
 			return err
 		}
@@ -175,9 +184,11 @@ can consume it.`,
 		logger.Info("verbosity      : ", verbosity)
 		logger.Info("httpPort       : ", httpPort)
 		logger.Info("pprofPort      : ", pprofPort)
+		logger.Info("pprofPort      : ", pprofPort)
+		logger.Info("pprofPort      : ", pprofPort)
 		logger.Info("cookieDomain   : ", cookieDomain)
-		logger.Info("postageBlockId : ", postageBlockId)
-		logger.Info("corsOrigins    : ", corsOrigins)
+		logger.Info("feedCacheSize  : ", config.GetInt(optionFeedCacheSize))
+		logger.Info("feedCacheTTL   : ", config.GetString(optionFeedCacheTTL))
 
 		ctx, cancel := context.WithCancel(cmd.Context())
 		defer cancel()
@@ -190,6 +201,8 @@ can consume it.`,
 			EnsConfig:          ensConfig,
 			SubscriptionConfig: subscriptionConfig,
 			Logger:             logger,
+			FeedCacheSize:      config.GetInt(optionFeedCacheSize),
+			FeedCacheTTL:       config.GetString(optionFeedCacheTTL),
 		}
 
 		hdlr, err := api.New(ctx, opts)
@@ -229,6 +242,8 @@ func init() {
 	serverCmd.Flags().BoolVar(&swag, "swag", false, "should run swagger-ui")
 	serverCmd.Flags().String("httpPort", defaultDFSHttpPort, "http port")
 	serverCmd.Flags().String("pprofPort", defaultDFSPprofPort, "pprof port")
+	serverCmd.Flags().Int("feedCacheSize", -1, "Keep feed updates in lru cache for faster access. -1 to disable")
+	serverCmd.Flags().String("feedCacheTTL", "0s", "How long to keep feed updates in lru cache. 0s to disable")
 	serverCmd.Flags().String("cookieDomain", defaultCookieDomain, "the domain to use in the cookie")
 	serverCmd.Flags().String("postageBlockId", "", "the postage block used to store the data in bee")
 	serverCmd.Flags().StringSlice("cors-origins", defaultCORSAllowedOrigins, "allow CORS headers for the given origins")

cmd/dfs/cmd/server_test.go

@@ -17,8 +17,11 @@ import (
 	"testing"
 	"time"
 
+	mockpost "github.com/ethersphere/bee/pkg/postage/mock"
+	mockstorer "github.com/ethersphere/bee/pkg/storer/mock"
 	"github.com/fairdatasociety/fairOS-dfs/cmd/common"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/api"
+	"github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/blockstore/bee/mock"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/dfs"
 	mock2 "github.com/fairdatasociety/fairOS-dfs/pkg/ensm/eth/mock"
@@ -43,11 +46,18 @@ func randStringRunes(n int) string {
 }
 
 func TestApis(t *testing.T) {
-	mockClient := mock.NewMockBeeClient()
+	storer := mockstorer.New()
+	beeUrl := mock.NewTestBeeServer(t, mock.TestServerOptions{
+		Storer:          storer,
+		PreventRedirect: true,
+		Post:            mockpost.New(mockpost.WithAcceptAll()),
+	})
+
+	logger := logging.New(io.Discard, logrus.DebugLevel)
+	mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger)
 	ens := mock2.NewMockNamespaceManager()
-	logger := logging.New(io.Discard, logrus.ErrorLevel)
 
-	users := user.NewUsers(mockClient, ens, logger)
+	users := user.NewUsers(mockClient, ens, 500, 0, logger)
 	dfsApi := dfs.NewMockDfsAPI(mockClient, users, logger)
 	handler = api.NewMockHandler(dfsApi, logger, []string{"http://localhost:3000"})
 	defer handler.Close()
@@ -64,7 +74,7 @@ func TestApis(t *testing.T) {
 		}
 	}()
 
-	// wait 10 seconds for the server to start
+	// wait for the server to start
 	<-time.After(time.Second * 3)
 	t.Run("login-fail-test", func(t *testing.T) {
 		c := http.Client{Timeout: time.Duration(1) * time.Minute}