init ci refactor (#69)

Co-authored-by: Maksim Konovalov <maksim.konovalov@vk.team>

.github/workflows/check.yml

@@ -0,0 +1,51 @@
+name: Run checks
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  luacheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Setup Tarantool
+        uses: tarantool/setup-tarantool@v2
+        with:
+          tarantool-version: '2.8'
+
+      - name: Setup tt
+        run: |
+          curl -L https://tarantool.io/release/2/installer.sh | sudo bash
+          sudo apt install -y tt
+          tt version
+
+      - name: Setup luacheck
+        run: tt rocks install luacheck 0.25.0
+
+      - name: Run luacheck
+        run: ./.rocks/bin/luacheck .
+
+  golangci-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v2
+
+      - uses: actions/checkout@v2
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        continue-on-error: true
+        with:
+          # The first run is for GitHub Actions error format.
+          args: --config=.golangci.yaml
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # The second run is for human-readable error format with a file name
+          # and a line number.
+          args: --out-${NO_FUTURE}format colored-line-number --config=.golangci.yaml

.github/workflows/main.yml

@@ -10,43 +10,84 @@ on:
     branches: [ "master" ]
 
 jobs:
+  all-tests:
+    # We could replace it with ubuntu-latest after fixing the bug:
+    # https://github.com/tarantool/setup-tarantool/issues/37
+    runs-on: ubuntu-20.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        golang:
+          - '1.20'
+          - 'stable'
+        tarantool:
+          - '2.8'
+          - '2.10'
+          - 'master'
+        include:
+          - tarantool: 'master'
+            golang: '1.20'
+            coverage: true
 
-  test:
-    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+      - name: Clone the connector
+        uses: actions/checkout@v3
 
-    - name: Set up Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.21'
+      - name: Setup tt
+        run: |
+          curl -L https://tarantool.io/release/2/installer.sh | sudo bash
+          sudo apt install -y tt
 
-    - name: Test
-      run: make test
+      - name: Setup tt environment
+        run: tt init
 
-    - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v4.0.1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        slug: KaymeKaydex/go-vshard-router
+      - name: Setup Tarantool ${{ matrix.tarantool }}
+        if: matrix.tarantool != 'master'
+        uses: tarantool/setup-tarantool@v2
+        with:
+          tarantool-version: ${{ matrix.tarantool }}
 
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v2
+      - name: Get Tarantool master commit
+        if: matrix.tarantool == 'master'
+        run: |
+          commit_hash=$(git ls-remote https://github.com/tarantool/tarantool.git --branch master | head -c 8)
+          echo "LATEST_COMMIT=${commit_hash}" >> $GITHUB_ENV
+        shell: bash
 
-      - uses: actions/checkout@v2
+      - name: Cache Tarantool master
+        if: matrix.tarantool == 'master'
+        id: cache-latest
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ github.workspace }}/bin
+            ${{ github.workspace }}/include
+          key: cache-latest-${{ env.LATEST_COMMIT }}
 
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        continue-on-error: true
+      - name: Setup Tarantool master
+        if: matrix.tarantool == 'master' && steps.cache-latest.outputs.cache-hit != 'true'
+        run: |
+          sudo tt install tarantool master
+
+      - name: Add Tarantool master to PATH
+        if: matrix.tarantool == 'master'
+        run: echo "${GITHUB_WORKSPACE}/bin" >> $GITHUB_PATH
+
+      - name: Setup golang for the connector and tests
+        uses: actions/setup-go@v3
         with:
-          # The first run is for GitHub Actions error format.
-          args: --config=.golangci.yaml
+          go-version: ${{ matrix.golang }}
+
+      - name: Run tests
+        run: |
+          export START_PORT=33000
+          export NREPLICASETS=5
+          make test
 
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+      - name: Upload coverage reports to Codecov
+        if: ${{ matrix.coverage }}
+        uses: codecov/codecov-action@v4.0.1
         with:
-          # The second run is for human-readable error format with a file name
-          # and a line number.
-          args: --out-${NO_FUTURE}format colored-line-number --config=.golangci.yaml
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: KaymeKaydex/go-vshard-router

.luacheckrc

@@ -0,0 +1,56 @@
+std = "luajit"
+codes = true
+
+read_globals = {
+	-- Tarantool vars:
+	"box",
+	"tonumber64",
+    "package",
+	"spacer",
+	"F",
+	"T",
+
+	package = {
+		fields = {
+			reload = {
+				fields = {
+					"count",
+					"register",
+				}
+			}
+		}
+	},
+
+	-- Exported by package 'config'
+	"config",
+}
+
+max_line_length = 200
+
+ignore = {
+	"212",
+	"213",
+	"411", -- ignore was previously defined
+	"422", -- ignore shadowing
+	"111", -- ignore non standart functions; for tarantool there is global functions
+}
+
+local conf_rules = {
+	read_globals = {
+		"instance_name",
+	},
+	globals = {
+		"box", "etcd",
+	}
+}
+
+exclude_files = {
+	"tests/tnt/.rocks/*",
+    "examples/customer/tarantool/.rocks/*",
+	"tests/tnt/tmp/*", -- ignore tmp tarantool files
+	"examples/customer/tarantool/*",                    -- TODO: now we ignore examples from original vshard example
+	".rocks/*",                                         -- ignore rocks after tests prepare
+}
+
+
+files["etc/*.lua"] = conf_rules

CHANGELOG.md

@@ -27,6 +27,10 @@ TESTS:
 * New tnt tests for discovery logic
 * New tnt tests for RouterMapCallRWImpl
 * New tnt tests for topology logic
+* Big CI update
+  * 2 sections for CI: static checks and tests
+  * integration tests run on ci with Tarantool cluster on vshard
+  * implemented luacheck for static checks
 
 EXAMPLES:
 * customer go mod fixed 

Makefile

@@ -1,4 +1,4 @@
-TEST_TIMEOUT?=20s
+TEST_TIMEOUT?=100s
 EXTENDED_TEST_TIMEOUT=1m
 GO_CMD?=go
 LOCAL_BIN:=$(CURDIR)/bin
@@ -14,10 +14,16 @@ ifeq ($(wildcard $(GOLANGCI_BIN)),)
 	$(GO_CMD) install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_TAG)
 endif
 
-test:
-	$(GO_CMD) test ./... -parallel=10 -timeout=$(TEST_TIMEOUT) -coverprofile=coverage.out.tmp
+prepare-tnt:
+	@$(MAKE) -C ./tests/tnt prepare
+
+test: prepare-tnt
+	export START_PORT=33000
+	export NREPLICASETS=5
+	$(GO_CMD) test ./... -race -parallel=10 -timeout=$(TEST_TIMEOUT) -covermode=atomic -coverprofile=coverage.out.tmp -coverpkg="./..."
 	@cat coverage.out.tmp | grep -v "mock" > coverage.out
 	@rm coverage.out.tmp
+	@$(MAKE) -C ./tests/tnt cluster-down
 
 cover: test
 	 $(GO_CMD) tool cover -html=coverage.out

examples/customer/tarantool/router.lua

@@ -1,7 +1,8 @@
 #!/usr/bin/env tarantool
 
 require('strict').on()
-fiber = require('fiber')
+local fiber = require('fiber')
+rawset(_G, 'fiber', fiber) -- set fiber as global
 
 -- Check if we are running under test-run
 if os.getenv('ADMIN') then
@@ -29,6 +30,8 @@ end
 cfg.bucket_count = 10000
 
 -- Start the database with sharding
-vshard = require('vshard')
+local vshard = require 'vshard'
+rawset(_G, 'vshard', vshard) -- set vshard as global
+
 vshard.router.cfg(cfg)
 

examples/customer/tarantool/storage.lua

@@ -28,7 +28,9 @@ replicasets = {'cbf06940-0790-498b-948d-042b62cf3d29',
                'ac522f65-aa94-4134-9f64-51ee384f1a54'}
 
 -- Start the database with sharding
-vshard = require('vshard')
+local vshard = require 'vshard'
+rawset(_G, 'vshard', vshard) -- set as global variable
+
 vshard.storage.cfg(cfg, names[NAME])
 
 box.once('access:v1', function()

tests/integration/testdata/config.lua

@@ -2,25 +2,6 @@
 -- from outside the root directory
 if package.setsearchroot ~= nil then
     package.setsearchroot()
-else
-    -- Workaround for rocks loading in tarantool 1.10
-    -- It can be removed in tarantool > 2.2
-    -- By default, when you do require('mymodule'), tarantool looks into
-    -- the current working directory and whatever is specified in
-    -- package.path and package.cpath. If you run your app while in the
-    -- root directory of that app, everything goes fine, but if you try to
-    -- start your app with "tarantool myapp/init.lua", it will fail to load
-    -- its modules, and modules from myapp/.rocks.
-    local fio = require('fio')
-    local app_dir = fio.abspath(fio.dirname(arg[0]))
-    package.path = app_dir .. '/?.lua;' .. package.path
-    package.path = app_dir .. '/?/init.lua;' .. package.path
-    package.path = app_dir .. '/.rocks/share/tarantool/?.lua;' .. package.path
-    package.path = app_dir .. '/.rocks/share/tarantool/?/init.lua;' .. package.path
-    package.cpath = app_dir .. '/?.so;' .. package.cpath
-    package.cpath = app_dir .. '/?.dylib;' .. package.cpath
-    package.cpath = app_dir .. '/.rocks/lib/tarantool/?.so;' .. package.cpath
-    package.cpath = app_dir .. '/.rocks/lib/tarantool/?.dylib;' .. package.cpath
 end
 
 local vshard = require('vshard')

tests/tnt/Makefile

@@ -1,3 +1,8 @@
+# TTCTL is tararantool control util (tt or tarantoolctl)
+TTCTL := tt
+SHELL := /bin/bash
+
+
 export NREPLICASETS:=5
 export START_PORT:=33000
 export OBJC_DISABLE_INITIALIZE_FORK_SAFETY:=YES # See review comments for PR #56 why
@@ -11,8 +16,12 @@ default: run
 
 all: default
 
-run: | clean cluster-up bootstrap gotest cluster-down
+prepare: deps clean cluster-up bootstrap
+
+run: | prepare gotest cluster-down
 
+deps:
+	$(TTCTL) rocks install vshard 0.1.26
 # cleanup tmp working directory
 clean:
 	@echo "${GREEN}STAGE: CLEANUP${NC}"
@@ -54,7 +63,7 @@ cluster-down:
 # run go tests, minus "-" signs before command allows failures, otherwise cluster-down stage won't run.
 gotest:
 	@echo "${GREEN}STAGE: RUN GOTESTS${NC}"
-	-go test -race -parallel=20 -coverpkg="../../" -coverprofile cover.out -timeout=90s
+	go test -race -parallel=20 -coverpkg="../../" -coverprofile cover.out -timeout=90s
 
 open-coverage:
 	go tool cover -html=cover.out

tests/tnt/cfgmaker.go

@@ -1,4 +1,4 @@
-package tnt_test
+package tnt
 
 import (
 	"fmt"

tests/tnt/cfgmaker.lua

@@ -2,6 +2,7 @@
 
 require('strict').on()
 
+-- luacheck: ignore
 local config_example = {
     sharding = {
         ['cbf06940-0790-498b-948d-042b62cf3d29'] = { -- replicaset #1
@@ -31,7 +32,7 @@ local config_example = {
             },
         }, -- replicaset #2
     }, -- sharding
-    replication_connect_quorum = 0,
+    replication_connect_quorum = 2,
 }
 
 local function get_uid(rs_id, instance_id)