Merge pull request #1131 from iotaledger/develop

Merge v0.5.3 into master

.dockerignore

@@ -1,5 +1,8 @@
 .git
 .gitignore
+.github/
+docs/
+.idea/
 
 LICENSE
 README.md

.github/workflows/docker-develop.yml

@@ -11,15 +11,16 @@ jobs:
 
   docker:
     runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
     steps:
+      -   name: Checkout repository
+          uses: actions/checkout@v2
 
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Publish to Docker Hub
-        uses: docker/build-push-action@92e71463491f2d026a477188b8ad3a0fdd9d672c
-        with:
-          repository: iotaledger/goshimmer
-          username: '${{ secrets.IOTALEDGER_HUB_DOCKER_LOGIN }}'
-          password: '${{ secrets.IOTALEDGER_HUB_DOCKER_PASSWORD }}'
-          tags: develop
+      -   name: Publish to Docker Hub
+          uses: docker/build-push-action@92e71463491f2d026a477188b8ad3a0fdd9d672c
+          with:
+            repository: iotaledger/goshimmer
+            username: '${{ secrets.IOTALEDGER_HUB_DOCKER_LOGIN }}'
+            password: '${{ secrets.IOTALEDGER_HUB_DOCKER_PASSWORD }}'
+            tags: develop

.github/workflows/integration-tests.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
     paths-ignore:
       - 'docs/**'
+env:
+  DOCKER_BUILDKIT: 1
+
 
 jobs:
 

.github/workflows/release.yml

@@ -28,6 +28,8 @@ jobs:
     name: Docker
     runs-on:
       - ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.github/workflows/reviewdog.yml

@@ -12,8 +12,8 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Run golangci-lint
-        uses: docker://reviewdog/action-golangci-lint:v1.9
+      - name: Run golangci-lint # reviewdog v1.19.0, golangci-lint v1.38.0
+        uses: reviewdog/action-golangci-lint@93be4324306dcbba508544d891a7b0576bb28ddd
         with:
           github_token: ${{ secrets.github_token }}
           golangci_lint_flags: "--timeout=10m"

.golangci.yml

@@ -1,27 +1,149 @@
-run:
-    tests: true
-
 issues:
     exclude-use-default: false
     max-issues-per-linter: 0
+    max-same-issues: 0
+    new-from-rev: e79d2a0c90ae4c80aa56afebcd06d5540bbdc75f # Enhance golangci-lint (#1113)
+    exclude:
+        - 'Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked' # errcheck
+        - 'err113: do not define dynamic errors, use wrapped static errors instead:' # goerr113
+        - 'type name will be used as [0-9A-Za-z_.]+ by other packages, and that stutters; consider calling this' # golint
+        - 'Potential file inclusion via variable' # gosec
+        - "G404: Use of weak random number generator" # gosec
+        - 'Subprocess launch(ed with variable|ing should be audited)' # gosec
+        - 'Use of unsafe calls should be audited' # gosec
+        - 'G108: Profiling endpoint is automatically exposed on /debug/pprof' # gosec
+        - '(Expect directory permissions to be 0750 or less|Expect file permissions to be 0600 or less)' # gosec
+        - 'G101: Potential hardcoded credentials' # gosec
+        - '(G104|G307)' # gosec Duplicated errcheck checks.
+        - 'ST1000: at least one file in a package should have a package comment' # stylecheck
+        - 'ST1005: error strings should not be capitalized' # stylecheck
+        - '`[0-9A-Za-z_.]+` - `[0-9A-Za-z_.]+` always receives `[0-9A-Za-z_.]+`' # unparam
+    exclude-rules:
+        -   path: (_test\.go|example_test\.go|example_[0-9A-Za-z_-]+_test\.go)
+            linters:
+                - errcheck
+                - bodyclose
+                - noctx
+                - gosec
+                - funlen
+                - gomnd
+        -   path: (_test\.go|example_test\.go|example_[0-9A-Za-z_-]+_test\.go)
+            linters:
+                - gocritic
+            text: "unnecessaryBlock: block doesn't have definitions, can be simply deleted"
 
 linters-settings:
-    gofmt:
-        simplify: true
-    golint:
-        min-confidence: 0.8
+    funlen:
+        lines: 100
+        statements: 50
+    gocritic:
+        enabled-tags:
+            - diagnostic
+            - style
+            - opinionated
+        disabled-checks:
+            - commentedOutCode
+            - ifElseChain
+            - commentFormatting
+            - dupImport
+            - unnamedResult
+            - octalLiteral
+            - whyNoLint
+            - wrapperFunc
+    goimports:
+        local-prefixes: github.com/iotaledger/goshimmer
+    gomnd:
+        settings:
+            mnd:
+                # don't include the "operation" and "assign"
+                checks: argument,case,condition,return
     govet:
         check-shadowing: true
     misspell:
         locale: US
+    nolintlint:
+        allow-unused: false # report any unused nolint directives
 
 linters:
+    disable-all: true
     enable:
-        - misspell
-        - gofmt
-        - goimports
-        - govet
-        - golint
-    disable:
+        - bodyclose # Checks whether HTTP response body is closed successfully.
+        - deadcode # Finds unused code.
+        - dogsled # Checks assignments with too many blank identifiers (e.g. x, , , _, := f()).
+        - dupl # Tool for code clone detection.
+        # Errcheck is a program for checking for unchecked errors in go programs.
+        # These unchecked errors can be critical bugs in some cases
         - errcheck
-        - gochecknoglobals
+        - exportloopref # Checks for pointers to enclosing loop variables.
+        - funlen # Tool for detection of long functions.
+        - goconst # Finds repeated strings that could be replaced by a constant.
+        - gocritic # Provides many diagnostics that check for bugs, performance and style issues.
+        - gocyclo # Computes and checks the cyclomatic complexity of functions.
+        - goerr113 # Golang linter to check the errors handling expressions.
+        - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification.
+        - gofumpt # Gofumpt checks whether code was gofumpt-ed. A more strict version of gofmt.
+        - goimports # Sorts and format imports.
+        - golint # golint checks style mistakes.
+        - gomnd # An analyzer to detect magic numbers.
+        - goprintffuncname # Checks that printf-like functions are named with f at the end.
+        - gosec # Inspects source code for security problems.
+        - gosimple # Linter for Go source code that specializes in simplifying a code.
+        - govet # Vet examines Go source code and reports suspicious constructs.
+        - ineffassign # Detects when assignments to existing variables are not used.
+        - misspell # Finds commonly misspelled English words in comments.
+        - nakedret # Finds naked returns in functions greater than a specified function length.
+        - noctx # noctx finds sending http request without context.Context.
+        - nolintlint # Reports ill-formed or insufficient nolint directives.
+        - staticcheck # Staticcheck is a go vet on steroids, applying a ton of static analysis checks.
+        - structcheck # Finds unused struct fields.
+        - stylecheck # Stylecheck is a replacement for golint.
+        - unconvert # Remove unnecessary type conversions.
+        - unparam # Reports unused function parameters.
+        - unused # Checks Go code for unused constants, variables, functions and types.
+        - varcheck # Finds unused global variables and constants.
+        - whitespace # Tool for detection of leading and trailing whitespace.
+
+
+    # don't enable:
+    # asciicheck: Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false]
+    # cyclop: checks function and package cyclomatic complexity [fast: true, auto-fix: false]
+    # depguard: Go linter that checks if package imports are in a list of acceptable packages [fast: true, auto-fix: false]
+    # durationcheck: check for two durations multiplied together [fast: true, auto-fix: false]
+    # errorlint: go-errorlint is a source code linter for Go software that can be used to find code that will cause problemswith the error wrapping scheme introduced in Go 1.13. [fast: true, auto-fix: false]
+    # exhaustive: check exhaustiveness of enum switch statements [fast: true, auto-fix: false]
+    # exhaustivestruct: Checks if all struct's fields are initialized [fast: true, auto-fix: false]
+    # forbidigo: Forbids identifiers [fast: true, auto-fix: false]
+    # forcetypeassert: finds forced type assertions [fast: true, auto-fix: false]
+    # gci: Gci control golang package import order and make it always deterministic. [fast: true, auto-fix: true]
+    # gochecknoglobals: check that no global variables exist [fast: true, auto-fix: false]
+    # gochecknoinits: Checks that no init functions are present in Go code [fast: true, auto-fix: false]
+    # gocognit: Computes and checks the cognitive complexity of functions [fast: true, auto-fix: false]
+    # godot: Check if comments end in a period [fast: true, auto-fix: true]
+    # godox: Tool for detection of FIXME, TODO and other comment keywords [fast: true, auto-fix: false]
+    # gofmt: Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification [fast: true, auto-fix: true]
+    # goheader: Checks is file header matches to pattern [fast: true, auto-fix: false]
+    # gomodguard: Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. [fast: true, auto-fix: false]
+    # ifshort: Checks that your code uses short syntax for if-statements whenever possible [fast: true, auto-fix: false]
+    # importas: Enforces consistent import aliases [fast: true, auto-fix: false]
+    # interfacer: Linter that suggests narrower interface types [fast: true, auto-fix: false]
+    # lll: Reports long lines [fast: true, auto-fix: false]
+    # makezero: Finds slice declarations with non-zero initial length [fast: true, auto-fix: false]
+    # maligned: Tool to detect Go structs that would take less memory if their fields were sorted [fast: true, auto-fix: false]
+    # nestif: Reports deeply nested if statements [fast: true, auto-fix: false]
+    # nilerr: Finds the code that returns nil even if it checks that the error is not nil. [fast: true, auto-fix: false]
+    # nlreturn: nlreturn checks for a new line before return and branch statements to increase code clarity [fast: true, auto-fix: false]
+    # paralleltest: paralleltest detects missing usage of t.Parallel() method in your Go test [fast: true, auto-fix: false]
+    # prealloc: Finds slice declarations that could potentially be preallocated [fast: true, auto-fix: false]
+    # predeclared: find code that shadows one of Go's predeclared identifiers [fast: true, auto-fix: false]
+    # revive: Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. [fast: true, auto-fix: false]
+    # rowserrcheck: checks whether Err of rows is checked successfully [fast: true, auto-fix: false]
+    # scopelint: Scopelint checks for unpinned variables in go programs [fast: true, auto-fix: false]
+    # sqlclosecheck: Checks that sql.Rows and sql.Stmt are closed. [fast: true, auto-fix: false]
+    # testpackage: linter that makes you use a separate _test package [fast: true, auto-fix: false]
+    # thelper: thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers [fast: true, auto-fix: false]
+    # tparallel: tparallel detects inappropriate usage of t.Parallel() method in your Go test codes [fast: true, auto-fix: false]
+    # typecheck: Like the front-end of a Go compiler, parses and type-checks Go code [fast: true, auto-fix: false]
+    # wastedassign: wastedassign finds wasted assignment statements. [fast: true, auto-fix: false]
+    # wrapcheck: Checks that errors returned from external packages are wrapped [fast: true, auto-fix: false]
+    # wsl: Whitespace Linter - Forces you to use empty lines! [fast: true, auto-fix: false]
+

CHANGELOG.md

@@ -1,3 +1,16 @@
+# v0.5.3 - 2021-03-25
+* Added new API endpoints
+* Added models navigation through the Dashboard Explorer
+* Added new diagnostic APIs
+* Added new docs sections
+* Fix dashboard mana event feed
+* Fix markers issue
+* Fix UnlockBlocks check
+* Fix loading of config parameters
+* Fix bug in the Get method of ColoredBalances
+* Enhance golangci-lint
+* **Breaking**: bumps network and database versions
+
 # v0.5.2 - 2021-03-17
 * Fix markers past cone check
 * Add more information to explorer and message API

Dockerfile

@@ -1,3 +1,5 @@
+# syntax = docker/dockerfile:1.2.1
+
 ############################
 # Build
 ############################
@@ -19,13 +21,16 @@ ENV GO111MODULE=on
 RUN go mod download
 RUN go mod verify
 
-# Copy everything from the current directory to the PWD(Present Working Directory) inside the container
-COPY . .
-
-# Build the binary
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-      -ldflags='-w -s -extldflags "-static"' -a \
-       -o /go/bin/goshimmer
+# 1. Mount everything from the current directory to the PWD(Present Working Directory) inside the container
+# 2. Mount the build cache volume
+# 3. Build the binary
+# 4. Verify that goshimmer binary is statically linked
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
+    -ldflags='-w -s -extldflags "-static"' \
+    -o /go/bin/goshimmer; \
+    ./check_static.sh
 
 ############################
 # Image

check_static.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# This script uses ldd utility to verify that result goshimmer binary is statically linked
+
+ldd /go/bin/goshimmer &> /dev/null
+if [ $? -ne 0 ]; then
+  exit 0
+else
+  echo "/go/bin/goshimmer must be a statically linked binary"
+  exit 1
+fi

client/data.go

@@ -12,7 +12,6 @@ const (
 
 // Data sends the given data (payload) by creating a message in the backend.
 func (api *GoShimmerAPI) Data(data []byte) (string, error) {
-
 	res := &webapi_data.Response{}
 	if err := api.do(http.MethodPost, routeData,
 		&webapi_data.Request{Data: data}, res); err != nil {

client/drng.go

@@ -14,7 +14,6 @@ const (
 
 // BroadcastCollectiveBeacon sends the given collective beacon (payload) by creating a message in the backend.
 func (api *GoShimmerAPI) BroadcastCollectiveBeacon(payload []byte) (string, error) {
-
 	res := &webapi_drng.CollectiveBeaconResponse{}
 	if err := api.do(http.MethodPost, routeCollectiveBeacon,
 		&webapi_drng.CollectiveBeaconRequest{Payload: payload}, res); err != nil {

client/lib.go

@@ -3,6 +3,7 @@ package client
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -102,7 +103,7 @@ func interpretBody(res *http.Response, decodeTo interface{}) error {
 
 	errRes := &errorresponse{}
 	if err := json.Unmarshal(resBody, errRes); err != nil {
-		return fmt.Errorf("unable to read error from response body: %w", err)
+		return fmt.Errorf("unable to read error from response body: %w repsonseBody: %s", err, resBody)
 	}
 
 	switch res.StatusCode {
@@ -131,9 +132,9 @@ func (api *GoShimmerAPI) do(method string, route string, reqObj interface{}, res
 			return err
 		}
 	}
-
+	ctx := context.TODO()
 	// construct request
-	req, err := http.NewRequest(method, fmt.Sprintf("%s/%s", api.baseURL, route), func() io.Reader {
+	req, err := http.NewRequestWithContext(ctx, method, fmt.Sprintf("%s/%s", api.baseURL, route), func() io.Reader {
 		if data == nil {
 			return nil
 		}

client/message.go

@@ -3,23 +3,39 @@ package client
 import (
 	"net/http"
 
-	webapi_message "github.com/iotaledger/goshimmer/plugins/webapi/message"
+	"github.com/iotaledger/goshimmer/plugins/webapi/jsonmodels"
 )
 
 const (
-	routeFindByID    = "message/findById"
-	routeSendPayload = "message/sendPayload"
+	routeMessage         = "messages/"
+	routeMessageMetadata = "/metadata"
+	routeSendPayload     = "messages/payload"
 )
 
-// FindMessageByID finds messages by the given base58 encoded IDs. The messages are returned in the same order as
-// the given IDs. Non available messages are empty at their corresponding index.
-func (api *GoShimmerAPI) FindMessageByID(base58EncodedIDs []string) (*webapi_message.FindByIDResponse, error) {
-	res := &webapi_message.FindByIDResponse{}
+// GetMessage is the handler for the /messages/:messageID endpoint.
+func (api *GoShimmerAPI) GetMessage(base58EncodedID string) (*jsonmodels.Message, error) {
+	res := &jsonmodels.Message{}
 
 	if err := api.do(
-		http.MethodPost,
-		routeFindByID,
-		&webapi_message.FindByIDRequest{IDs: base58EncodedIDs},
+		http.MethodGet,
+		routeMessage+base58EncodedID,
+		nil,
+		res,
+	); err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+// GetMessageMetadata is the handler for the /messages/:messageID/metadata endpoint.
+func (api *GoShimmerAPI) GetMessageMetadata(base58EncodedID string) (*jsonmodels.MessageMetadata, error) {
+	res := &jsonmodels.MessageMetadata{}
+
+	if err := api.do(
+		http.MethodGet,
+		routeMessage+base58EncodedID+routeMessageMetadata,
+		nil,
 		res,
 	); err != nil {
 		return nil, err
@@ -30,9 +46,9 @@ func (api *GoShimmerAPI) FindMessageByID(base58EncodedIDs []string) (*webapi_mes
 
 // SendPayload send a message with the given payload.
 func (api *GoShimmerAPI) SendPayload(payload []byte) (string, error) {
-	res := &webapi_message.SendPayloadResponse{}
+	res := &jsonmodels.PostPayloadResponse{}
 	if err := api.do(http.MethodPost, routeSendPayload,
-		&webapi_message.SendPayloadRequest{Payload: payload}, res); err != nil {
+		&jsonmodels.PostPayloadRequest{Payload: payload}, res); err != nil {
 		return "", err
 	}