Merge branch 'main' into geoffw/goreleaser

.devcontainer/.env

@@ -2,4 +2,5 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_DB=postgres
 POSTGRES_HOSTNAME=localhost
-LC_COLLATE=POSIX
+LC_COLLATE=POSIX
+FML_LOG_LEVEL=DEBUG

.devcontainer/Dockerfile

@@ -13,9 +13,11 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 
 # [Optional] Uncomment the next lines to use go get to install anything else you need
 USER vscode
-RUN go install github.com/vektra/mockery/v2@v2.30.1
-RUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.3
-RUN go install github.com/goreleaser/goreleaser@v1.18.2
+RUN go install github.com/vektra/mockery/v2@v2.30.16 \
+ && go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.3 \
+ && go install golang.org/x/tools/cmd/goimports@v0.11.0 \
+ && go install mvdan.cc/gofumpt@v0.5.0
+ && go install github.com/goreleaser/goreleaser@v1.18.2
 USER root
 
 # [Optional] Uncomment this line to install global node packages.

.devcontainer/devcontainer.json

@@ -4,7 +4,7 @@
 	"name": "Fasttrack",
 	"dockerComposeFile": "docker-compose.yml",
 	"service": "app",
-	"workspaceFolder": "/workspaces/fasttrack",
+	"workspaceFolder": "/workspaces/fasttrackml",
 	// Configure tool-specific properties.
 	"customizations": {
 		// Configure properties specific to VS Code.
@@ -15,7 +15,7 @@
 				"go.useLanguageServer": true,
 				"go.gopath": "/go",
 				"go.goroot": "/usr/local/go",
-				"terminal.integrated.defaultProfile.linux": "zsh",
+				"terminal.integrated.defaultProfile.linux": "zsh"
 			},
 			// Add the IDs of extensions you want installed when the container is created.
 			"extensions": [
@@ -36,6 +36,9 @@
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",
 	"features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {
+			"dockerDashComposeVersion": "v2"
+		},
 		"ghcr.io/devcontainers/features/git:1": {},
 		"ghcr.io/devcontainers/features/github-cli:1": {},
 		"ghcr.io/dhoeric/features/k6:1": {}

.devcontainer/docker-compose.yml

@@ -31,7 +31,7 @@ services:
     volumes:
       - home:/home/vscode
       - workspaces:/workspaces
-      - ..:/workspaces/fasttrack:cached
+      - ..:/workspaces/fasttrackml:cached
 
     # Overrides default command so things don't shut down after the process ends.
     command: sleep infinity

.github/workflows/ci.yml

@@ -29,11 +29,26 @@ jobs:
         id: tags
         run: echo tags=$(jq -r '."go.buildTags"' .vscode/settings.json) >> $GITHUB_OUTPUT
 
+      - name: Download formatters
+        run: |
+          go install golang.org/x/tools/cmd/goimports@v0.11.0
+          go install mvdan.cc/gofumpt@v0.5.0
+
       - name: Check formatting
-        run: gofmt -l .
+        run: |
+          unformatted=$(
+            gofumpt -l .
+            goimports -l --local github.com/G-Research/fasttrackml .
+          )
+          if [ -n "$unformatted" ]; then
+            for file in $unformatted; do
+              echo "::error file=$file::$file is not formatted properly (hint: run \"make go-format\" to fix this)"
+            done
+            exit 1
+          fi
 
       - name: Download mockery
-        run: go install github.com/vektra/mockery/v2@v2.20.0
+        run: go install github.com/vektra/mockery/v2@v2.30.16
 
       - name: Generate mocks
         run: make mocks-generate
@@ -60,7 +75,7 @@ jobs:
           go-version: "1.20"
 
       - name: Download mockery
-        run: go install github.com/vektra/mockery/v2@v2.20.0
+        run: go install github.com/vektra/mockery/v2@v2.30.16
 
       - name: Generate mocks
         run: make mocks-generate
@@ -69,19 +84,25 @@ jobs:
         run: make test-go-unit
 
   golang-integration-tests:
+    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
     name: Golang Integration Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
           go-version: "1.20"
 
       - name: Run Integration Tests
         run: make service-test
+        env:
+          DOCKER_BUILDKIT: 1
 
   python-integration-tests:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id

.gitignore

@@ -9,7 +9,7 @@ fasttrackml
 fasttrackml.db*
 
 # Imported integration tests
-tests/**/*.src
+tests/integration/python/*/*.src
 
 # Quick start
 outputs

.vscode/launch.json

@@ -12,10 +12,8 @@
             "program": "${workspaceFolder}",
             "args": [
                 "server",
-                "--log-level",
-                "debug",
                 "--database-uri",
-                "postgres://postgres:postgres@localhost:5432/postgres"
+                "postgres://postgres:postgres@localhost/postgres"
             ]
         },
         {
@@ -26,8 +24,6 @@
             "program": "${workspaceFolder}",
             "args": [
                 "server",
-                "--log-level",
-                "debug",
                 "--database-uri",
                 "sqlite://fasttrackml.db?mode=memory&cache=shared"
             ]
@@ -40,8 +36,6 @@
             "program": "${workspaceFolder}",
             "args": [
                 "server",
-                "--log-level",
-                "debug",
                 "--database-uri",
                 "sqlite://fasttrackml.db"
             ]
@@ -54,8 +48,6 @@
             "program": "${workspaceFolder}",
             "args": [
                 "server",
-                "--log-level",
-                "debug",
                 "--database-uri",
                 "sqlite://encrypted.db?_key=passphrase"
             ]

.vscode/settings.json

@@ -1,4 +1,7 @@
 {
-    "go.testTags": "netgo osusergo sqlite_foreign_keys sqlite_math_functions sqlite_omit_load_extension sqlite_unlock_notify sqlite_vacuum_incr",
-    "go.buildTags": "netgo osusergo sqlite_foreign_keys sqlite_math_functions sqlite_omit_load_extension sqlite_unlock_notify sqlite_vacuum_incr"
+    "go.buildTags": "netgo osusergo sqlite_foreign_keys sqlite_math_functions sqlite_omit_load_extension sqlite_unlock_notify sqlite_vacuum_incr",
+    "gopls": {
+        "formatting.local": "github.com/G-Research/fasttrackml",
+        "formatting.gofumpt": true
+    }
 }

Makefile

@@ -4,21 +4,12 @@
 #
 # Project-specific variables
 #
-# Service name. Used for binary name, docker-compose service name, etc...
-SERVICE=fasttrack-service
+# App name.
+APP=fml
 # Enable Go Modules.
 GO111MODULE=on
 
 #
-# General variables
-#
-# Path to Docker file
-PATH_DOCKER_FILE=$(realpath ./docker/Dockerfile)
-# Path to docker-compose file
-PATH_DOCKER_COMPOSE_FILE=$(realpath ./docker/docker-compose.yml)
-# Docker compose starting options.
-DOCKER_COMPOSE_OPTIONS= -f $(PATH_DOCKER_COMPOSE_FILE)
-
 # Default target (help)
 #
 .PHONY: help
@@ -43,9 +34,15 @@ go-get: ## get go modules.
 	@go mod download
 
 .PHONY: go-build
-go-build: ## build service binary.
+go-build: ## build app binary.
 	@echo '>>> Building go binary.'
-	@go build -ldflags="-s -w" -o $(SERVICE) ./main.go
+	@go build -ldflags="-linkmode external -extldflags -static -s -w" -tags "$$(jq -r '."go.buildTags"' .vscode/settings.json)" -o $(APP) ./main.go
+
+.PHONY: go-format
+go-format: ## format go code.
+	@echo '>>> Formatting go code.'
+	@gofumpt -w .
+	@goimports -w -local github.com/G-Research/fasttrackml .
 
 #
 # Tests targets.
@@ -61,45 +58,58 @@ test-go-integration: ## run go integration tests.
 	go test -v -p 1 -tags="integration" ./tests/integration/golang/...
 
 PHONY: test-python-integration
-test-python-integration: build ## run the MLFlow python integration tests.
+test-python-integration: test-python-integration-mlflow test-python-integration-aim  ## run all the python integration tests.
+
+PHONY: test-python-integration-mlflow
+test-python-integration-mlflow: build ## run the MLFlow python integration tests.
 	@echo ">>> Running MLFlow python integration tests."
-	tests/mlflow/test.sh
+	tests/integration/python/mlflow/test.sh
+
+PHONY: test-python-integration-aim
+test-python-integration-aim: build ## run the Aim python integration tests.
+	@echo ">>> Running Aim python integration tests."
+	tests/integration/python/aim/test.sh
 
 #
 # Service test targets
 #
 .PHONY: service-build
-service-build: ## build service and all it's dependencies
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) build --no-cache
+service-build: ## build service and all its dependencies
+	@docker-compose build
 
 .PHONY: start-service-dependencies
 service-start-dependencies: ## start service dependencies in docker.
 	@echo ">>> Start all Service dependencies."
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) up \
+	@docker-compose up \
 	-d \
-	fasttrack-postgres
+	postgres
 
 .PHONY: service-start
 service-start: service-build service-start-dependencies ## start service in docker.
 	@echo ">>> Sleeping 5 seconds until dependencies start."
 	@sleep 5
 	@echo ">>> Starting service."
 	@echo ">>> Starting up service container."
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) up -d $(SERVICE)
+	@docker-compose up -d service
 
 .PHONY: service-stop
 service-stop: ## stop service in docker.
 	@echo ">>> Stopping service."
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) down -v --remove-orphans
+	@docker-compose stop
 
 .PHONY: service-restart
 service-restart: service-stop service-start ## restart service in docker
 
 .PHONY: service-test
 service-test: service-stop service-start ## run tests over the service in docker.
 	@echo ">>> Running tests over service."
-	@docker-compose $(DOCKER_COMPOSE_OPTIONS) \
-		run fasttrack-integration-tests
+	@docker-compose \
+		run integration-tests
+
+.PHONY: service-clean
+service-clean: ## clean service in docker.
+	@echo ">>> Cleaning service."
+	@docker-compose down -v --remove-orphans
 
 #
 # Mockery targets.
@@ -130,4 +140,4 @@ build: node-build go-build
 PHONY: run
 run: build ## run the FastTrackML server
 	@echo ">>> Running the FasttrackML server."
-	./$(SERVICE) server
+	./$(APP) server

docker-compose.yml

@@ -0,0 +1,33 @@
+services:
+  service:
+    build:
+      context: .
+      args:
+        tags: netgo osusergo sqlite_foreign_keys sqlite_math_functions sqlite_omit_load_extension sqlite_unlock_notify sqlite_vacuum_incr
+    depends_on:
+      - postgres
+    environment:
+      FML_DATABASE_URI: postgres://postgres:postgres@postgres/postgres
+      FML_LOG_LEVEL: debug
+
+  postgres:
+    image: postgres:latest
+    environment:
+      - POSTGRES_PASSWORD=postgres
+
+  integration-tests:
+    image: golang:1.20
+    command: make test-go-integration
+    volumes:
+      - .:/go/src
+      - go-cache:/go/pkg
+    working_dir: /go/src
+    depends_on:
+      - service
+      - postgres
+    environment:
+      FML_DATABASE_URI: postgres://postgres:postgres@postgres/postgres
+      FML_SERVICE_URI: http://service:5000
+
+volumes:
+  go-cache: