Skip to content

Commit

Permalink
fix: handle reset when services are excluded from starting
Browse files Browse the repository at this point in the history
  • Loading branch information
@sweatybridge
sweatybridge committed Jul 6, 2023
1 parent 2bcc9c7 commit faea6d2
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 71 deletions.
70 changes: 53 additions & 17 deletions internal/db/reset/reset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/pkg/stdcopy"
"github.com/jackc/pgconn"
"github.com/jackc/pgerrcode"
Expand Down Expand Up @@ -71,6 +72,11 @@ func resetDatabase(ctx context.Context, fsys afero.Fs, options ...func(*pgx.Conn
if err := initDatabase(ctx, options...); err != nil {
return err
}
if utils.Config.Db.MajorVersion > 14 {
if err := InitSchema15(ctx, utils.DbId); err != nil {
return err
}
}
if err := RestartDatabase(ctx, os.Stderr); err != nil {
return err
}
Expand Down Expand Up @@ -154,24 +160,29 @@ func RestartDatabase(ctx context.Context, w io.Writer) error {
if !WaitForHealthyService(ctx, utils.DbId, healthTimeout) {
return ErrDatabase
}
// TODO: update storage-api to handle postgres restarts
if err := utils.Docker.ContainerRestart(ctx, utils.StorageId, container.StopOptions{}); err != nil {
return fmt.Errorf("failed to restart storage-api: %w", err)
}
// Reload PostgREST schema cache.
if err := utils.Docker.ContainerKill(ctx, utils.RestId, "SIGUSR1"); err != nil {
return fmt.Errorf("failed to reload PostgREST schema cache: %w", err)
}
// TODO: update gotrue to handle postgres restarts
if err := utils.Docker.ContainerRestart(ctx, utils.GotrueId, container.StopOptions{}); err != nil {
return fmt.Errorf("failed to restart gotrue: %w", err)
}
// TODO: update realtime to handle postgres restarts
if err := utils.Docker.ContainerRestart(ctx, utils.RealtimeId, container.StopOptions{}); err != nil {
return fmt.Errorf("failed to restart realtime: %w", err)
// No need to restart PostgREST because it automatically reconnects and listens for schema changes
services := []string{utils.StorageId, utils.GotrueId, utils.RealtimeId}
errCh := make(chan error, len(services))
utils.WaitAll(services, func(id string) {
if err := utils.Docker.ContainerRestart(ctx, id, container.StopOptions{}); err != nil && !errdefs.IsNotFound(err) {
errCh <- fmt.Errorf("Failed to restart %s: %w", id, err)
} else {
errCh <- nil
}
})
// Combine errors
var err error
for range services {
if err == nil {
err = <-errCh
continue
}
if next := <-errCh; next != nil {
err = fmt.Errorf("%w\n%w", err, next)
}
}
// Wait for services with internal schema migrations
return WaitForServiceReady(ctx, []string{utils.StorageId, utils.GotrueId})
// Do not wait for service healthy as those services may be excluded from starting
return err
}

func RetryEverySecond(ctx context.Context, callback func() bool, timeout time.Duration) bool {
Expand Down Expand Up @@ -278,3 +289,28 @@ func likeEscapeSchema(schemas []string) (result []string) {
}
return result
}

func InitSchema15(ctx context.Context, host string) error {
// Apply service migrations
if err := utils.DockerRunOnceWithStream(ctx, utils.StorageImage, []string{
"ANON_KEY=" + utils.Config.Auth.AnonKey,
"SERVICE_KEY=" + utils.Config.Auth.ServiceRoleKey,
"PGRST_JWT_SECRET=" + utils.Config.Auth.JwtSecret,
fmt.Sprintf("DATABASE_URL=postgresql://supabase_storage_admin:%s@%s:5432/postgres", utils.Config.Db.Password, host),
fmt.Sprintf("FILE_SIZE_LIMIT=%v", utils.Config.Storage.FileSizeLimit),
"STORAGE_BACKEND=file",
"TENANT_ID=stub",
// TODO: https://github.com/supabase/storage-api/issues/55
"REGION=stub",
"GLOBAL_S3_BUCKET=stub",
}, []string{"node", "dist/scripts/migrate-call.js"}, io.Discard, os.Stderr); err != nil {
return err
}
return utils.DockerRunOnceWithStream(ctx, utils.GotrueImage, []string{
"GOTRUE_LOG_LEVEL=error",
"GOTRUE_DB_DRIVER=postgres",
fmt.Sprintf("GOTRUE_DB_DATABASE_URL=postgresql://supabase_auth_admin:%s@%s:5432/postgres", utils.Config.Db.Password, host),
"GOTRUE_SITE_URL=" + utils.Config.Auth.SiteUrl,
"GOTRUE_JWT_SECRET=" + utils.Config.Auth.JwtSecret,
}, []string{"gotrue", "migrate"}, io.Discard, os.Stderr)
}
78 changes: 50 additions & 28 deletions internal/db/reset/reset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,15 @@ import (
)

func TestResetCommand(t *testing.T) {
t.Run("throws error on connect failure", func(t *testing.T) {
// Setup in-memory fs
fsys := afero.NewMemMapFs()
// Run test
err := Run(context.Background(), pgconn.Config{Password: "postgres"}, fsys)
// Check error
assert.ErrorContains(t, err, "invalid port (outside range)")
})

t.Run("throws error on missing config", func(t *testing.T) {
err := Run(context.Background(), pgconn.Config{}, afero.NewMemMapFs())
assert.ErrorIs(t, err, os.ErrNotExist)
Expand Down Expand Up @@ -253,46 +262,59 @@ func TestRestartDatabase(t *testing.T) {
Health: &types.Health{Status: "healthy"},
},
}})
// Restarts postgREST
utils.RestId = "test-rest"
gock.New(utils.Docker.DaemonHost()).
Post("/v" + utils.Docker.ClientVersion() + "/containers/" + utils.RestId + "/kill").
Reply(http.StatusOK)
// Restarts storage-api
// Restarts services
utils.StorageId = "test-storage"
gock.New(utils.Docker.DaemonHost()).
Post("/v" + utils.Docker.ClientVersion() + "/containers/" + utils.StorageId + "/restart").
Reply(http.StatusOK)
// Restarts gotrue
utils.GotrueId = "test-auth"
gock.New(utils.Docker.DaemonHost()).
Post("/v" + utils.Docker.ClientVersion() + "/containers/" + utils.GotrueId + "/restart").
Reply(http.StatusOK)
// Restarts realtime
utils.RealtimeId = "test-realtime"
for _, container := range []string{utils.StorageId, utils.GotrueId, utils.RealtimeId} {
gock.New(utils.Docker.DaemonHost()).
Post("/v" + utils.Docker.ClientVersion() + "/containers/" + container + "/restart").
Reply(http.StatusOK)
}
// Run test
err := RestartDatabase(context.Background(), io.Discard)
// Check error
assert.NoError(t, err)
assert.Empty(t, apitest.ListUnmatchedRequests())
})

t.Run("throws error on service restart failure", func(t *testing.T) {
utils.DbId = "test-reset"
// Setup mock docker
require.NoError(t, apitest.MockDocker(utils.Docker))
defer gock.OffAll()
// Restarts postgres
gock.New(utils.Docker.DaemonHost()).
Post("/v" + utils.Docker.ClientVersion() + "/containers/" + utils.RealtimeId + "/restart").
Post("/v" + utils.Docker.ClientVersion() + "/containers/" + utils.DbId + "/restart").
Reply(http.StatusOK)
// Wait for services ready
for _, container := range []string{utils.StorageId, utils.GotrueId} {
gock.New(utils.Docker.DaemonHost()).
Get("/v" + utils.Docker.ClientVersion() + "/containers/" + utils.DbId + "/json").
Reply(http.StatusOK).
JSON(types.ContainerJSON{ContainerJSONBase: &types.ContainerJSONBase{
State: &types.ContainerState{
Running: true,
Health: &types.Health{Status: "healthy"},
},
}})
// Restarts services
utils.StorageId = "test-storage"
utils.GotrueId = "test-auth"
utils.RealtimeId = "test-realtime"
for _, container := range []string{utils.StorageId, utils.GotrueId, utils.RealtimeId} {
gock.New(utils.Docker.DaemonHost()).
Get("/v" + utils.Docker.ClientVersion() + "/containers/" + container + "/json").
Reply(http.StatusOK).
JSON(types.ContainerJSON{ContainerJSONBase: &types.ContainerJSONBase{
State: &types.ContainerState{
Running: true,
Health: &types.Health{Status: "healthy"},
},
}})
Post("/v" + utils.Docker.ClientVersion() + "/containers/" + container + "/restart").
Reply(http.StatusServiceUnavailable)
}
// Run test
err := RestartDatabase(context.Background(), io.Discard)
// Check error
assert.NoError(t, err)
assert.ErrorContains(t, err, "Failed to restart "+utils.StorageId)
assert.ErrorContains(t, err, "Failed to restart "+utils.GotrueId)
assert.ErrorContains(t, err, "Failed to restart "+utils.RealtimeId)
assert.Empty(t, apitest.ListUnmatchedRequests())
})

t.Run("logs error on restart failure", func(t *testing.T) {
t.Run("throws error on db restart failure", func(t *testing.T) {
utils.DbId = "test-db"
// Setup mock docker
require.NoError(t, apitest.MockDocker(utils.Docker))
Expand All @@ -308,7 +330,7 @@ func TestRestartDatabase(t *testing.T) {
assert.Empty(t, apitest.ListUnmatchedRequests())
})

t.Run("timeout health check", func(t *testing.T) {
t.Run("throws error on health check timeout", func(t *testing.T) {
utils.DbId = "test-reset"
healthTimeout = 0 * time.Second
// Setup mock docker
Expand Down
27 changes: 1 addition & 26 deletions internal/db/start/start.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func initSchema(ctx context.Context, conn *pgx.Conn, host string, w io.Writer) e
if utils.Config.Db.MajorVersion <= 14 {
return initSchema14(ctx, conn)
}
return initSchema15(ctx, host)
return reset.InitSchema15(ctx, host)
}

func initSchema14(ctx context.Context, conn *pgx.Conn) error {
Expand All @@ -150,31 +150,6 @@ func initSchema14(ctx context.Context, conn *pgx.Conn) error {
return apply.BatchExecDDL(ctx, conn, strings.NewReader(utils.InitialSchemaSql))
}

func initSchema15(ctx context.Context, host string) error {
// Apply service migrations
if err := utils.DockerRunOnceWithStream(ctx, utils.StorageImage, []string{
"ANON_KEY=" + utils.Config.Auth.AnonKey,
"SERVICE_KEY=" + utils.Config.Auth.ServiceRoleKey,
"PGRST_JWT_SECRET=" + utils.Config.Auth.JwtSecret,
fmt.Sprintf("DATABASE_URL=postgresql://supabase_storage_admin:%s@%s:5432/postgres", utils.Config.Db.Password, host),
fmt.Sprintf("FILE_SIZE_LIMIT=%v", utils.Config.Storage.FileSizeLimit),
"STORAGE_BACKEND=file",
"TENANT_ID=stub",
// TODO: https://github.com/supabase/storage-api/issues/55
"REGION=stub",
"GLOBAL_S3_BUCKET=stub",
}, []string{"node", "dist/scripts/migrate-call.js"}, io.Discard, os.Stderr); err != nil {
return err
}
return utils.DockerRunOnceWithStream(ctx, utils.GotrueImage, []string{
"GOTRUE_LOG_LEVEL=error",
"GOTRUE_DB_DRIVER=postgres",
fmt.Sprintf("GOTRUE_DB_DATABASE_URL=postgresql://supabase_auth_admin:%s@%s:5432/postgres", utils.Config.Db.Password, host),
"GOTRUE_SITE_URL=" + utils.Config.Auth.SiteUrl,
"GOTRUE_JWT_SECRET=" + utils.Config.Auth.JwtSecret,
}, []string{"gotrue", "migrate"}, io.Discard, os.Stderr)
}

func setupDatabase(ctx context.Context, fsys afero.Fs, w io.Writer, options ...func(*pgx.ConnConfig)) error {
conn, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{}, options...)
if err != nil {
Expand Down

0 comments on commit faea6d2

Please sign in to comment.