feat: add a database for the gateway

.devcontainer/devcontainer.json

@@ -0,0 +1,11 @@
+{
+	"name": "Data services devcontainer",
+	"dockerComposeFile": "docker-compose.yml",
+	"service": "gateway",
+	"containerUser": "root",
+	"workspaceFolder": "/workspace",
+	"shutdownAction": "stopCompose",
+	"features": {
+	},
+	"postCreateCommand": "go mod download"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,35 @@
+services:
+  gateway:
+    image: "mcr.microsoft.com/devcontainers/go:2-1.26"
+    volumes:
+      - ../:/workspace:cached
+    # Overrides default command so things don't shut down after the process ends.
+    entrypoint: sleep infinity
+    environment:
+      DB_USER: renku_gateway
+      DB_NAME: renku_gateway
+      DB_PASSWORD: renku_gateway
+      DB_HOST: 127.0.0.1
+    network_mode: service:db
+    depends_on:
+      - db
+    group_add:
+      - "8983" # Allows access to the solr data in the shared volume
+
+  db:
+    image: postgres:16
+    restart: unless-stopped
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+      - type: bind
+        source: ./generate_ulid_func.sql
+        target: /docker-entrypoint-initdb.d/generate_ulid_func.sql
+    environment:
+      POSTGRES_USER: renku_gateway
+      POSTGRES_DB: postgres
+      POSTGRES_PASSWORD: renku_gateway
+    ports:
+      - "5432:5432" # postgresql
+
+volumes:
+  postgres-data:

.devcontainer/generate_ulid_func.sql

@@ -0,0 +1,89 @@
+CREATE DATABASE renku_template;
+ALTER DATABASE renku_template WITH is_template TRUE;
+\c renku_template;
+
+-- From https://github.com/geckoboard/pgulid/blob/master/pgulid.sql
+-- Taken at commit sha b265253
+-- pgulid is based on OK Log's Go implementation of the ULID spec
+--
+-- https://github.com/oklog/ulid
+-- https://github.com/ulid/spec
+--
+-- Copyright 2016 The Oklog Authors
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+-- NOTE: REPLACE will error if you change the name, args or return type of the function
+-- There is no CREATE IF EXISTS, this is the closest thing that gives similar functionality
+CREATE OR REPLACE FUNCTION generate_ulid()
+RETURNS TEXT
+AS $$
+DECLARE
+  -- Crockford's Base32
+  encoding   BYTEA = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
+  timestamp  BYTEA = E'\\000\\000\\000\\000\\000\\000';
+  output     TEXT = '';
+
+  unix_time  BIGINT;
+  ulid       BYTEA;
+BEGIN
+  -- 6 timestamp bytes
+  unix_time = (EXTRACT(EPOCH FROM CLOCK_TIMESTAMP()) * 1000)::BIGINT;
+  timestamp = SET_BYTE(timestamp, 0, (unix_time >> 40)::BIT(8)::INTEGER);
+  timestamp = SET_BYTE(timestamp, 1, (unix_time >> 32)::BIT(8)::INTEGER);
+  timestamp = SET_BYTE(timestamp, 2, (unix_time >> 24)::BIT(8)::INTEGER);
+  timestamp = SET_BYTE(timestamp, 3, (unix_time >> 16)::BIT(8)::INTEGER);
+  timestamp = SET_BYTE(timestamp, 4, (unix_time >> 8)::BIT(8)::INTEGER);
+  timestamp = SET_BYTE(timestamp, 5, unix_time::BIT(8)::INTEGER);
+
+  -- 10 entropy bytes
+  ulid = timestamp || gen_random_bytes(10);
+
+  -- Encode the timestamp
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 224) >> 5));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 0) & 31)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 1) & 248) >> 3));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 1) & 7) << 2) | ((GET_BYTE(ulid, 2) & 192) >> 6)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 2) & 62) >> 1));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 2) & 1) << 4) | ((GET_BYTE(ulid, 3) & 240) >> 4)));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 3) & 15) << 1) | ((GET_BYTE(ulid, 4) & 128) >> 7)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 4) & 124) >> 2));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 4) & 3) << 3) | ((GET_BYTE(ulid, 5) & 224) >> 5)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 5) & 31)));
+
+  -- Encode the entropy
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 6) & 248) >> 3));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 6) & 7) << 2) | ((GET_BYTE(ulid, 7) & 192) >> 6)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 7) & 62) >> 1));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 7) & 1) << 4) | ((GET_BYTE(ulid, 8) & 240) >> 4)));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 8) & 15) << 1) | ((GET_BYTE(ulid, 9) & 128) >> 7)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 9) & 124) >> 2));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 9) & 3) << 3) | ((GET_BYTE(ulid, 10) & 224) >> 5)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 10) & 31)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 11) & 248) >> 3));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 11) & 7) << 2) | ((GET_BYTE(ulid, 12) & 192) >> 6)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 12) & 62) >> 1));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 12) & 1) << 4) | ((GET_BYTE(ulid, 13) & 240) >> 4)));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 13) & 15) << 1) | ((GET_BYTE(ulid, 14) & 128) >> 7)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 14) & 124) >> 2));
+  output = output || CHR(GET_BYTE(encoding, ((GET_BYTE(ulid, 14) & 3) << 3) | ((GET_BYTE(ulid, 15) & 224) >> 5)));
+  output = output || CHR(GET_BYTE(encoding, (GET_BYTE(ulid, 15) & 31)));
+
+  RETURN output;
+END
+$$
+LANGUAGE plpgsql
+VOLATILE;
+
+CREATE DATABASE renku_gateway TEMPLATE renku_template;

cmd/gateway/main.go

@@ -30,6 +30,7 @@ import (
 )
 
 func main() {
+	// FAKE CHANGE
 	// Logging setup
 	slog.SetDefault(jsonLogger)
 	// Load configuration

cmd/migrations/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+
+	_ "github.com/SwissDataScienceCenter/renku-gateway/internal/pg/migrations"
+	_ "github.com/jackc/pgx/v5/stdlib"
+	"github.com/pressly/goose/v3"
+)
+
+var (
+	flags = flag.NewFlagSet("goose", flag.ExitOnError)
+	dir   = flags.String("dir", "internal/pg/migrations", "directory with migration files")
+)
+
+func main() {
+	if err := flags.Parse(os.Args[1:]); err != nil {
+		log.Fatalf("goose: failed to parse flags: %v", err)
+	}
+	args := flags.Args()
+
+	if len(args) < 1 {
+		flags.Usage()
+		os.Exit(1)
+	}
+
+	dbURL, err := getPostgresURL()
+	if err != nil {
+		log.Fatalf("goose: could not form postgres database URL: %v\n", err)
+	}
+
+	db, err := goose.OpenDBWithDriver("pgx", dbURL)
+	if err != nil {
+		log.Fatalf("goose: unable to connect to database: %v", err)
+	}
+
+	defer func() {
+		if err := db.Close(); err != nil {
+			log.Fatalf("goose: failed to close DB: %v", err)
+		}
+	}()
+
+	command := args[0]
+	arguments := []string{}
+	if len(args) > 1 {
+		arguments = append(arguments, args[1:]...)
+	}
+
+	ctx := context.Background()
+	if err := goose.RunContext(ctx, command, db, *dir, arguments...); err != nil {
+		log.Fatalf("goose %v: %v", command, err)
+	}
+}
+
+func getPostgresURL() (postgresURL string, err error) {
+	// TODO: use gateway config
+	dbName := os.Getenv(("DB_NAME"))
+	user := os.Getenv("DB_USER")
+	password := os.Getenv("DB_PASSWORD")
+	host := os.Getenv("DB_HOST")
+	url, err := url.Parse(fmt.Sprintf("postgres://%s:%s@%s:5432/%s", user, password, host, dbName))
+	if err != nil {
+		return postgresURL, err
+	}
+	return url.String(), nil
+}

cmd/try_migrate/main.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"log/slog"
+	"net/url"
+	"os"
+
+	_ "github.com/SwissDataScienceCenter/renku-gateway/internal/pg/migrations"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/stdlib"
+	"github.com/jackc/pgx/v5/tracelog"
+	"github.com/pressly/goose/v3"
+)
+
+var (
+	flags = flag.NewFlagSet("goose", flag.ExitOnError)
+	dir   = flags.String("dir", "internal/pg/migrations", "directory with migration files")
+)
+
+// Try: we can do something like this at the start of the gateway process to run migrations
+func main() {
+	dbURL, err := getPostgresURL()
+	if err != nil {
+		log.Fatalf("goose: could not form postgres database URL: %v\n", err)
+	}
+
+	connConfig, err := pgx.ParseConfig(dbURL)
+	if err != nil {
+		log.Fatalf("goose: could not create database connection: %v\n", err)
+	}
+	connConfig.Tracer = &tracelog.TraceLog{
+		Logger:   &traceLogger{},
+		LogLevel: tracelog.LogLevelTrace,
+	}
+
+	db, err := goose.OpenDBWithDriver("pgx", stdlib.RegisterConnConfig(connConfig))
+	if err != nil {
+		log.Fatalf("goose: unable to connect to database: %v", err)
+	}
+
+	defer func() {
+		if err := db.Close(); err != nil {
+			log.Fatalf("goose: failed to close DB: %v", err)
+		}
+	}()
+
+	err = goose.UpContext(context.Background(), db, *dir)
+	if err != nil {
+		log.Fatalf("goose: failed to run migrations: %v", err)
+	}
+}
+
+func getPostgresURL() (postgresURL string, err error) {
+	// TODO: use gateway config
+	dbName := os.Getenv(("DB_NAME"))
+	user := os.Getenv("DB_USER")
+	password := os.Getenv("DB_PASSWORD")
+	host := os.Getenv("DB_HOST")
+	url, err := url.Parse(fmt.Sprintf("postgres://%s:%s@%s:5432/%s", user, password, host, dbName))
+	if err != nil {
+		return postgresURL, err
+	}
+	return url.String(), nil
+}
+
+type traceLogger struct{}
+
+func (tl *traceLogger) Log(ctx context.Context, level tracelog.LogLevel, msg string, data map[string]any) {
+	attrs := []slog.Attr{{Key: "Lvl", Value: slog.StringValue(level.String())}}
+	for key, value := range data {
+		attrs = append(attrs, slog.Attr{Key: key, Value: slog.AnyValue(value)})
+	}
+
+	slog.Default().LogAttrs(ctx, slog.LevelError, msg, attrs...)
+}

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/oapi-codegen/runtime v1.1.2
 	github.com/oklog/ulid/v2 v2.1.1
 	github.com/posthog/posthog-go v1.9.1
+	github.com/pressly/goose/v3 v3.27.0
 	github.com/redis/go-redis/v9 v9.17.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
@@ -27,6 +28,15 @@ require (
 	golang.org/x/time v0.14.0
 )
 
+require (
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/mfridman/interpolate v0.0.2 // indirect
+	github.com/sethvargo/go-retry v0.3.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+)
+
 require (
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -44,6 +54,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/jackc/pgx/v5 v5.8.0
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/mailru/easyjson v0.9.1 // indirect
@@ -79,18 +90,18 @@ require (
 	github.com/zitadel/logging v0.6.2 // indirect
 	github.com/zitadel/schema v1.3.2 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/otel v1.39.0 // indirect
-	go.opentelemetry.io/otel/metric v1.39.0 // indirect
-	go.opentelemetry.io/otel/trace v1.39.0 // indirect
+	go.opentelemetry.io/otel v1.40.0 // indirect
+	go.opentelemetry.io/otel/metric v1.40.0 // indirect
+	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.46.0 // indirect
-	golang.org/x/mod v0.31.0 // indirect
-	golang.org/x/net v0.48.0 // indirect
+	golang.org/x/crypto v0.48.0 // indirect
+	golang.org/x/mod v0.32.0 // indirect
+	golang.org/x/net v0.50.0 // indirect
 	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.40.0 // indirect
-	golang.org/x/text v0.33.0 // indirect
-	golang.org/x/tools v0.40.0 // indirect
+	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/tools v0.41.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect