From d20ea8cecc9932a9c1ea0772dfeb921b74002501 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 28 Aug 2024 11:51:36 +0200 Subject: [PATCH 01/33] bus: update settings --- api/events.go | 24 +-- api/setting.go | 9 + autopilot/autopilot.go | 1 - bus/bus.go | 72 ++++++- bus/client/settings.go | 59 +++--- bus/routes.go | 329 +++++++++++++++--------------- cmd/renterd/node.go | 2 +- internal/test/e2e/cluster.go | 12 +- internal/test/e2e/cluster_test.go | 2 +- internal/test/e2e/events_test.go | 10 +- internal/test/e2e/gouging_test.go | 4 +- internal/test/e2e/s3_test.go | 2 +- internal/worker/cache.go | 9 - internal/worker/cache_test.go | 1 - stores/settingsdb.go | 26 --- stores/settingsdb_test.go | 39 +--- stores/sql/database.go | 6 - stores/sql/main.go | 24 --- stores/sql/mysql/main.go | 8 - stores/sql/sqlite/main.go | 8 - worker/s3/s3.go | 1 - 21 files changed, 295 insertions(+), 353 deletions(-) diff --git a/api/events.go b/api/events.go index e9600e53b..38d490506 100644 --- a/api/events.go +++ b/api/events.go @@ -19,7 +19,6 @@ const ( EventAdd = "add" EventUpdate = "update" - EventDelete = "delete" EventArchive = "archive" EventRenew = "renew" ) @@ -68,11 +67,6 @@ type ( Update interface{} `json:"update"` Timestamp time.Time `json:"timestamp"` } - - EventSettingDelete struct { - Key string `json:"key"` - Timestamp time.Time `json:"timestamp"` - } ) var ( @@ -138,15 +132,6 @@ var ( URL: url, } } - - WebhookSettingDelete = func(url string, headers map[string]string) webhooks.Webhook { - return webhooks.Webhook{ - Event: EventDelete, - Headers: headers, - Module: ModuleSetting, - URL: url, - } - } ) func ParseEventWebhook(event webhooks.Event) (interface{}, error) { @@ -201,19 +186,12 @@ func ParseEventWebhook(event webhooks.Event) (interface{}, error) { return e, nil } case ModuleSetting: - switch event.Event { - case EventUpdate: + if event.Event == EventUpdate { var e EventSettingUpdate if err := json.Unmarshal(bytes, &e); err != nil { return nil, err } return e, nil - case EventDelete: - var e EventSettingDelete - if err := json.Unmarshal(bytes, &e); err != nil { - return nil, err - } - return e, nil } } return nil, fmt.Errorf("%w: module %s event %s", ErrUnknownEvent, event.Module, event.Event) diff --git a/api/setting.go b/api/setting.go index 5976b00b2..34f5dfd31 100644 --- a/api/setting.go +++ b/api/setting.go @@ -239,6 +239,15 @@ func (gs GougingSettings) Validate() error { return nil } +// Validate returns an error if the upload packing settings are not considered +// valid. +func (up UploadPackingSettings) Validate() error { + if up.Enabled && up.SlabBufferMaxSizeSoft <= 0 { + return errors.New("SlabBufferMaxSizeSoft must be greater than zero when upload packing is enabled") + } + return nil +} + // Redundancy returns the effective storage redundancy of the // RedundancySettings. func (rs RedundancySettings) Redundancy() float64 { diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 9ea235a11..fb554fb2c 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -73,7 +73,6 @@ type Bus interface { SlabsForMigration(ctx context.Context, healthCutoff float64, set string, limit int) ([]api.UnhealthySlab, error) // settings - UpdateSetting(ctx context.Context, key string, value interface{}) error GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) RedundancySettings(ctx context.Context) (rs api.RedundancySettings, err error) diff --git a/bus/bus.go b/bus/bus.go index c5ae1113e..f5117825d 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -294,9 +294,7 @@ type ( // A SettingStore stores settings. SettingStore interface { - DeleteSetting(ctx context.Context, key string) error Setting(ctx context.Context, key string) (string, error) - Settings(ctx context.Context) ([]string, error) UpdateSetting(ctx context.Context, key, value string) error } @@ -485,10 +483,16 @@ func (b *Bus) Handler() http.Handler { "DELETE /sectors/:hk/:root": b.sectorsHostRootHandlerDELETE, - "GET /settings": b.settingsHandlerGET, - "GET /setting/:key": b.settingKeyHandlerGET, - "PUT /setting/:key": b.settingKeyHandlerPUT, - "DELETE /setting/:key": b.settingKeyHandlerDELETE, + "GET /settings/gouging": b.settingsGougingHandlerGET, + "PUT /settings/gouging": b.settingsGougingHandlerPUT, + "GET /settings/pinned": b.settingsPinnedHandlerGET, + "PUT /settings/pinned": b.settingsPinnedHandlerPUT, + "GET /settings/redundancy": b.settingsRedundancyHandlerGET, + "PUT /settings/redundancy": b.settingsRedundancyHandlerPUT, + "GET /settings/s3authentication": b.settingsS3AuthenticationHandlerGET, + "PUT /settings/s3authentication": b.settingsS3AuthenticationHandlerPUT, + "GET /settings/uploadpacking": b.settingsUploadPackingHandlerGET, + "PUT /settings/uploadpacking": b.settingsUploadPackingHandlerPUT, "POST /slabs/migration": b.slabsMigrationHandlerPOST, "GET /slabs/partial/:key": b.slabsPartialHandlerGET, @@ -738,3 +742,59 @@ func (b *Bus) deriveSubKey(purpose string) types.PrivateKey { } return pk } + +func (b *Bus) fetchSetting(ctx context.Context, key string, value interface{}) error { + // testnets have different redundancy settings + defaultRedundancySettings := api.DefaultRedundancySettings + if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { + defaultRedundancySettings = api.DefaultRedundancySettingsTestnet + } + + defaults := map[string]interface{}{ + api.SettingGouging: api.DefaultGougingSettings, + api.SettingPricePinning: api.DefaultPricePinSettings, + api.SettingRedundancy: defaultRedundancySettings, + api.SettingUploadPacking: api.DefaultUploadPackingSettings, + } + + setting, err := b.ss.Setting(ctx, key) + if errors.Is(err, api.ErrSettingNotFound) { + val, ok := defaults[key] + if !ok { + return fmt.Errorf("%w: unknown setting '%s'", api.ErrSettingNotFound, key) + } + + bytes, _ := json.Marshal(val) + if err := b.ss.UpdateSetting(ctx, key, string(bytes)); err != nil { + b.logger.Warn(fmt.Sprintf("failed to update default setting '%s': %v", key, err)) + } + return json.Unmarshal(bytes, &val) + } else if err != nil { + return err + } + + return json.Unmarshal([]byte(setting), &value) +} + +func (b *Bus) updateSetting(ctx context.Context, key string, value string, updatePinMgr bool) error { + err := b.ss.UpdateSetting(ctx, key, value) + if err != nil { + return err + } + + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + Key: key, + Update: value, + Timestamp: time.Now().UTC(), + }, + }) + + if updatePinMgr { + b.pinMgr.TriggerUpdate() + } + + return nil +} diff --git a/bus/client/settings.go b/bus/client/settings.go index 22714cf8b..e813a7417 100644 --- a/bus/client/settings.go +++ b/bus/client/settings.go @@ -2,65 +2,72 @@ package client import ( "context" - "fmt" "go.sia.tech/renterd/api" ) // ContractSetSettings returns the contract set settings. -func (c *Client) ContractSetSettings(ctx context.Context) (gs api.ContractSetSetting, err error) { - err = c.Setting(ctx, api.SettingContractSet, &gs) +func (c *Client) ContractSetSettings(ctx context.Context) (css api.ContractSetSetting, err error) { + err = c.c.WithContext(ctx).GET("/setting/contractset", &css) return } -// DeleteSetting will delete the setting with given key. -func (c *Client) DeleteSetting(ctx context.Context, key string) error { - return c.c.WithContext(ctx).DELETE(fmt.Sprintf("/setting/%s", key)) +// UpdateContractSetSetting updates the given setting. +func (c *Client) UpdateContractSetSetting(ctx context.Context, css api.ContractSetSetting) error { + return c.c.WithContext(ctx).PUT("/setting/contractset", css) } // GougingSettings returns the gouging settings. func (c *Client) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { - err = c.Setting(ctx, api.SettingGouging, &gs) + err = c.c.WithContext(ctx).GET("/setting/gouging", &gs) return } +// UpdateGougingSettings updates the given setting. +func (c *Client) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { + return c.c.WithContext(ctx).PUT("/setting/gouging", gs) +} + // PricePinningSettings returns the contract set settings. func (c *Client) PricePinningSettings(ctx context.Context) (pps api.PricePinSettings, err error) { - err = c.Setting(ctx, api.SettingPricePinning, &pps) + err = c.c.WithContext(ctx).GET("/setting/pinned", &pps) return } -// RedundancySettings returns the redundancy settings. -func (c *Client) RedundancySettings(ctx context.Context) (rs api.RedundancySettings, err error) { - err = c.Setting(ctx, api.SettingRedundancy, &rs) - return +// UpdatePinnedSettings updates the given setting. +func (c *Client) UpdatePinnedSettings(ctx context.Context, pps api.PricePinSettings) error { + return c.c.WithContext(ctx).PUT("/setting/pinned", pps) } -// S3AuthenticationSettings returns the S3 authentication settings. -func (c *Client) S3AuthenticationSettings(ctx context.Context) (as api.S3AuthenticationSettings, err error) { - err = c.Setting(ctx, api.SettingS3Authentication, &as) +// RedundancySettings returns the redundancy settings. +func (c *Client) RedundancySettings(ctx context.Context) (rs api.RedundancySettings, err error) { + err = c.c.WithContext(ctx).GET("/setting/redundancy", &rs) return } -// Setting returns the value for the setting with given key. -func (c *Client) Setting(ctx context.Context, key string, value interface{}) (err error) { - err = c.c.WithContext(ctx).GET(fmt.Sprintf("/setting/%s", key), &value) - return +// UpdateRedundancySettings updates the given setting. +func (c *Client) UpdateRedundancySettings(ctx context.Context, rs api.RedundancySettings) error { + return c.c.WithContext(ctx).PUT("/setting/redundancy", rs) } -// Settings returns the keys of all settings. -func (c *Client) Settings(ctx context.Context) (settings []string, err error) { - err = c.c.WithContext(ctx).GET("/settings", &settings) +// S3AuthenticationSettings returns the S3 authentication settings. +func (c *Client) S3AuthenticationSettings(ctx context.Context) (as api.S3AuthenticationSettings, err error) { + err = c.c.WithContext(ctx).GET("/setting/s3authentication", &as) return } -// UpdateSetting will update the given setting under the given key. -func (c *Client) UpdateSetting(ctx context.Context, key string, value interface{}) error { - return c.c.WithContext(ctx).PUT(fmt.Sprintf("/setting/%s", key), value) +// UpdateS3AuthenticationSettings updates the given setting. +func (c *Client) UpdateS3AuthenticationSettings(ctx context.Context, as api.S3AuthenticationSettings) error { + return c.c.WithContext(ctx).PUT("/setting/s3authentication", as) } // UploadPackingSettings returns the upload packing settings. func (c *Client) UploadPackingSettings(ctx context.Context) (ups api.UploadPackingSettings, err error) { - err = c.Setting(ctx, api.SettingUploadPacking, &ups) + err = c.c.WithContext(ctx).GET("/setting/uploadpacking", &ups) return } + +// UpdateUploadPackingSettings updates the given setting. +func (c *Client) UpdateUploadPackingSettings(ctx context.Context, ups api.UploadPackingSettings) error { + return c.c.WithContext(ctx).PUT("/setting/uploadpacking", ups) +} diff --git a/bus/routes.go b/bus/routes.go index f020c5944..76c7df4ec 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -33,15 +33,6 @@ import ( "go.uber.org/zap" ) -func (b *Bus) fetchSetting(ctx context.Context, key string, value interface{}) error { - if val, err := b.ss.Setting(ctx, key); err != nil { - return fmt.Errorf("could not get contract set settings: %w", err) - } else if err := json.Unmarshal([]byte(val), &value); err != nil { - b.logger.Panicf("failed to unmarshal %v settings '%s': %v", key, val, err) - } - return nil -} - func (b *Bus) consensusAcceptBlock(jc jape.Context) { var block types.Block if jc.Decode(&block) != nil { @@ -1261,6 +1252,176 @@ func (b *Bus) packedSlabsHandlerDonePOST(jc jape.Context) { jc.Check("failed to mark packed slab(s) as uploaded", b.ms.MarkPackedSlabsUploaded(jc.Request.Context(), psrp.Slabs)) } +func (b *Bus) settingsGougingHandlerGET(jc jape.Context) { + var gs api.GougingSettings + if err := b.fetchSetting(jc.Request.Context(), api.SettingGouging, &gs); errors.Is(err, api.ErrSettingNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("failed to get gouging settings", err) == nil { + jc.Encode(gs) + } +} + +func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { + var gs api.GougingSettings + if jc.Decode(&gs) != nil { + return + } else if err := gs.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update gouging settings, error: %v", err), http.StatusBadRequest) + return + } + + // marshal the setting + data, err := json.Marshal(gs) + if err != nil { + jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) + return + } + + // update the setting + if jc.Check("could not update gouging settings", b.updateSetting(jc.Request.Context(), api.SettingGouging, string(data), true)) != nil { + return + } +} + +func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { + var pps api.PricePinSettings + if err := b.fetchSetting(jc.Request.Context(), api.SettingPricePinning, &pps); errors.Is(err, api.ErrSettingNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("failed to get price pinning settings", err) == nil { + // populate the Autopilots map with the current autopilots + aps, err := b.as.Autopilots(jc.Request.Context()) + if jc.Check("failed to fetch autopilots", err) != nil { + return + } + for _, ap := range aps { + if _, exists := pps.Autopilots[ap.ID]; !exists { + pps.Autopilots[ap.ID] = api.AutopilotPins{} + } + } + jc.Encode(pps) + } +} + +func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { + var pps api.PricePinSettings + if jc.Decode(&pps) != nil { + return + } else if err := pps.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update price pinning settings, error: %v", err), http.StatusBadRequest) + return + } else if pps.Enabled { + if _, err := ibus.NewForexClient(pps.ForexEndpointURL).SiacoinExchangeRate(jc.Request.Context(), pps.Currency); err != nil { + jc.Error(fmt.Errorf("couldn't update price pinning settings, forex API unreachable,error: %v", err), http.StatusBadRequest) + return + } + } + + // marshal the setting + data, err := json.Marshal(pps) + if err != nil { + jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) + return + } + + // update the setting + if jc.Check("could not update price pinning settings", b.updateSetting(jc.Request.Context(), api.SettingPricePinning, string(data), true)) != nil { + return + } +} + +func (b *Bus) settingsRedundancyHandlerGET(jc jape.Context) { + var rs api.RedundancySettings + if err := b.fetchSetting(jc.Request.Context(), api.SettingRedundancy, &rs); errors.Is(err, api.ErrSettingNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("failed to get redundancy settings", err) == nil { + jc.Encode(rs) + } +} + +func (b *Bus) settingsRedundancyHandlerPUT(jc jape.Context) { + var rs api.RedundancySettings + if jc.Decode(&rs) != nil { + return + } else if err := rs.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update redundancy settings, error: %v", err), http.StatusBadRequest) + return + } + + // marshal the setting + data, err := json.Marshal(rs) + if err != nil { + jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) + return + } + + // update the setting + if jc.Check("could not update redundancy settings", b.updateSetting(jc.Request.Context(), api.SettingRedundancy, string(data), false)) != nil { + return + } +} + +func (b *Bus) settingsS3AuthenticationHandlerGET(jc jape.Context) { + var s3as api.S3AuthenticationSettings + if err := b.fetchSetting(jc.Request.Context(), api.SettingS3Authentication, &s3as); errors.Is(err, api.ErrSettingNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("failed to get s3 authentication settings", err) == nil { + jc.Encode(s3as) + } +} + +func (b *Bus) settingsS3AuthenticationHandlerPUT(jc jape.Context) { + var s3as api.S3AuthenticationSettings + if jc.Decode(&s3as) != nil { + return + } else if err := s3as.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update s3 authentication settings, error: %v", err), http.StatusBadRequest) + return + } + + // marshal the setting + data, err := json.Marshal(s3as) + if err != nil { + jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) + return + } + + // update the setting + if jc.Check("could not update s3 authentication settings", b.updateSetting(jc.Request.Context(), api.SettingS3Authentication, string(data), false)) != nil { + return + } +} + +func (b *Bus) settingsUploadPackingHandlerGET(jc jape.Context) { + var ups api.UploadPackingSettings + if err := b.fetchSetting(jc.Request.Context(), api.SettingUploadPacking, &ups); errors.Is(err, api.ErrSettingNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("failed to get upload packing settings", err) == nil { + jc.Encode(ups) + } +} + +func (b *Bus) settingsUploadPackingHandlerPUT(jc jape.Context) { + var ups api.UploadPackingSettings + if jc.Decode(&ups) != nil { + return + } else if err := ups.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update upload packing settings, error: %v", err), http.StatusBadRequest) + return + } + + // marshal the setting + data, err := json.Marshal(ups) + if err != nil { + jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) + return + } + + // update the setting + if jc.Check("could not update upload packing settings", b.updateSetting(jc.Request.Context(), api.SettingUploadPacking, string(data), false)) != nil { + return + } +} + func (b *Bus) sectorsHostRootHandlerDELETE(jc jape.Context) { var hk types.PublicKey var root types.Hash256 @@ -1405,156 +1566,6 @@ func (b *Bus) slabsPartialHandlerPOST(jc jape.Context) { }) } -func (b *Bus) settingsHandlerGET(jc jape.Context) { - if settings, err := b.ss.Settings(jc.Request.Context()); jc.Check("couldn't load settings", err) == nil { - jc.Encode(settings) - } -} - -func (b *Bus) settingKeyHandlerGET(jc jape.Context) { - jc.Custom(nil, (any)(nil)) - - key := jc.PathParam("key") - if key == "" { - jc.Error(errors.New("path parameter 'key' can not be empty"), http.StatusBadRequest) - return - } - - setting, err := b.ss.Setting(jc.Request.Context(), jc.PathParam("key")) - if errors.Is(err, api.ErrSettingNotFound) { - jc.Error(err, http.StatusNotFound) - return - } else if err != nil { - jc.Error(err, http.StatusInternalServerError) - return - } - resp := []byte(setting) - - // populate autopilots of price pinning settings with defaults for better DX - if key == api.SettingPricePinning { - var pps api.PricePinSettings - err = json.Unmarshal([]byte(setting), &pps) - if jc.Check("failed to unmarshal price pinning settings", err) != nil { - return - } else if pps.Autopilots == nil { - pps.Autopilots = make(map[string]api.AutopilotPins) - } - // populate the Autopilots map with the current autopilots - aps, err := b.as.Autopilots(jc.Request.Context()) - if jc.Check("failed to fetch autopilots", err) != nil { - return - } - for _, ap := range aps { - if _, exists := pps.Autopilots[ap.ID]; !exists { - pps.Autopilots[ap.ID] = api.AutopilotPins{} - } - } - // encode the settings back - resp, err = json.Marshal(pps) - if jc.Check("failed to marshal price pinning settings", err) != nil { - return - } - } - jc.ResponseWriter.Header().Set("Content-Type", "application/json") - jc.ResponseWriter.Write(resp) -} - -func (b *Bus) settingKeyHandlerPUT(jc jape.Context) { - key := jc.PathParam("key") - if key == "" { - jc.Error(errors.New("path parameter 'key' can not be empty"), http.StatusBadRequest) - return - } - - var value interface{} - if jc.Decode(&value) != nil { - return - } - - data, err := json.Marshal(value) - if err != nil { - jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) - return - } - - switch key { - case api.SettingGouging: - var gs api.GougingSettings - if err := json.Unmarshal(data, &gs); err != nil { - jc.Error(fmt.Errorf("couldn't update gouging settings, invalid request body, %t", value), http.StatusBadRequest) - return - } else if err := gs.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update gouging settings, error: %v", err), http.StatusBadRequest) - return - } - b.pinMgr.TriggerUpdate() - case api.SettingRedundancy: - var rs api.RedundancySettings - if err := json.Unmarshal(data, &rs); err != nil { - jc.Error(fmt.Errorf("couldn't update redundancy settings, invalid request body"), http.StatusBadRequest) - return - } else if err := rs.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update redundancy settings, error: %v", err), http.StatusBadRequest) - return - } - case api.SettingS3Authentication: - var s3as api.S3AuthenticationSettings - if err := json.Unmarshal(data, &s3as); err != nil { - jc.Error(fmt.Errorf("couldn't update s3 authentication settings, invalid request body"), http.StatusBadRequest) - return - } else if err := s3as.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update s3 authentication settings, error: %v", err), http.StatusBadRequest) - return - } - case api.SettingPricePinning: - var pps api.PricePinSettings - if err := json.Unmarshal(data, &pps); err != nil { - jc.Error(fmt.Errorf("couldn't update price pinning settings, invalid request body"), http.StatusBadRequest) - return - } else if err := pps.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update price pinning settings, invalid settings, error: %v", err), http.StatusBadRequest) - return - } else if pps.Enabled { - if _, err := ibus.NewForexClient(pps.ForexEndpointURL).SiacoinExchangeRate(jc.Request.Context(), pps.Currency); err != nil { - jc.Error(fmt.Errorf("couldn't update price pinning settings, forex API unreachable,error: %v", err), http.StatusBadRequest) - return - } - } - b.pinMgr.TriggerUpdate() - } - - if jc.Check("could not update setting", b.ss.UpdateSetting(jc.Request.Context(), key, string(data))) == nil { - b.broadcastAction(webhooks.Event{ - Module: api.ModuleSetting, - Event: api.EventUpdate, - Payload: api.EventSettingUpdate{ - Key: key, - Update: value, - Timestamp: time.Now().UTC(), - }, - }) - } -} - -func (b *Bus) settingKeyHandlerDELETE(jc jape.Context) { - key := jc.PathParam("key") - if key == "" { - jc.Error(errors.New("path parameter 'key' can not be empty"), http.StatusBadRequest) - return - } - - if jc.Check("could not delete setting", b.ss.DeleteSetting(jc.Request.Context(), key)) == nil { - b.broadcastAction(webhooks.Event{ - Module: api.ModuleSetting, - Event: api.EventDelete, - Payload: api.EventSettingDelete{ - Key: key, - Timestamp: time.Now().UTC(), - }, - }) - } -} - func (b *Bus) contractIDAncestorsHandler(jc jape.Context) { var fcid types.FileContractID if jc.DecodeParam("id", &fcid) != nil { diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 9defbc127..5ab3bf13d 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -435,7 +435,7 @@ func (n *node) Run() error { as.V4Keypairs[k] = v } // update settings - if err := n.bus.UpdateSetting(context.Background(), api.SettingS3Authentication, as); err != nil { + if err := n.bus.UpdateS3AuthenticationSettings(context.Background(), as); err != nil { return fmt.Errorf("failed to update S3 authentication settings: %w", err) } } diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index b500643d3..189a92e43 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -436,14 +436,14 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { } // Update the bus settings. - tt.OK(busClient.UpdateSetting(ctx, api.SettingGouging, test.GougingSettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingContractSet, test.ContractSetSettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingPricePinning, test.PricePinSettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingRedundancy, test.RedundancySettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingS3Authentication, api.S3AuthenticationSettings{ + tt.OK(busClient.UpdateGougingSettings(ctx, test.GougingSettings)) + tt.OK(busClient.UpdateContractSetSetting(ctx, test.ContractSetSettings)) + tt.OK(busClient.UpdatePinnedSettings(ctx, test.PricePinSettings)) + tt.OK(busClient.UpdateRedundancySettings(ctx, test.RedundancySettings)) + tt.OK(busClient.UpdateS3AuthenticationSettings(ctx, api.S3AuthenticationSettings{ V4Keypairs: map[string]string{test.S3AccessKeyID: test.S3SecretAccessKey}, })) - tt.OK(busClient.UpdateSetting(ctx, api.SettingUploadPacking, api.UploadPackingSettings{ + tt.OK(busClient.UpdateUploadPackingSettings(ctx, api.UploadPackingSettings{ Enabled: enableUploadPacking, SlabBufferMaxSizeSoft: api.DefaultUploadPackingSettings.SlabBufferMaxSizeSoft, })) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 255999911..2c1c1e16e 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -1142,7 +1142,7 @@ func TestEphemeralAccounts(t *testing.T) { w := cluster.Worker tt := cluster.tt - tt.OK(b.UpdateSetting(context.Background(), api.SettingRedundancy, api.RedundancySettings{ + tt.OK(b.UpdateRedundancySettings(context.Background(), api.RedundancySettings{ MinShards: 1, TotalShards: 1, })) diff --git a/internal/test/e2e/events_test.go b/internal/test/e2e/events_test.go index 4972adf1b..e1bc29df7 100644 --- a/internal/test/e2e/events_test.go +++ b/internal/test/e2e/events_test.go @@ -26,7 +26,6 @@ func TestEvents(t *testing.T) { api.WebhookContractRenew, api.WebhookContractSetUpdate, api.WebhookHostUpdate, - api.WebhookSettingDelete, api.WebhookSettingUpdate, } @@ -125,10 +124,7 @@ func TestEvents(t *testing.T) { // update settings gs := gp.GougingSettings gs.HostBlockHeightLeeway = 100 - tt.OK(b.UpdateSetting(context.Background(), api.SettingGouging, gs)) - - // delete setting - tt.OK(b.DeleteSetting(context.Background(), api.SettingRedundancy)) + tt.OK(b.UpdateGougingSettings(context.Background(), gs)) // update host setting h := cluster.hosts[0] @@ -182,10 +178,6 @@ func TestEvents(t *testing.T) { if update.HostBlockHeightLeeway != 100 { t.Fatalf("unexpected update %+v", update) } - case api.EventSettingDelete: - if e.Key != api.SettingRedundancy || e.Timestamp.IsZero() { - t.Fatalf("unexpected event %+v", e) - } } } } diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go index 5be1784cb..6126cd36e 100644 --- a/internal/test/e2e/gouging_test.go +++ b/internal/test/e2e/gouging_test.go @@ -70,7 +70,7 @@ func TestGouging(t *testing.T) { // update the gouging settings to limit the max storage price to 100H gs := test.GougingSettings gs.MaxStoragePrice = types.NewCurrency64(100) - if err := b.UpdateSetting(context.Background(), api.SettingGouging, gs); err != nil { + if err := b.UpdateGougingSettings(context.Background(), gs); err != nil { t.Fatal(err) } @@ -118,7 +118,7 @@ func TestGouging(t *testing.T) { } // set optimised settings - tt.OK(b.UpdateSetting(context.Background(), api.SettingGouging, resp.Recommendation.GougingSettings)) + tt.OK(b.UpdateGougingSettings(context.Background(), resp.Recommendation.GougingSettings)) // evaluate optimised settings resp, err = cluster.Autopilot.EvaluateConfig(context.Background(), test.AutopilotConfig, resp.Recommendation.GougingSettings, test.RedundancySettings) diff --git a/internal/test/e2e/s3_test.go b/internal/test/e2e/s3_test.go index 3f20e22ad..5775d2292 100644 --- a/internal/test/e2e/s3_test.go +++ b/internal/test/e2e/s3_test.go @@ -825,7 +825,7 @@ func TestS3SettingsValidate(t *testing.T) { }, } for i, test := range tests { - err := cluster.Bus.UpdateSetting(context.Background(), api.SettingS3Authentication, api.S3AuthenticationSettings{ + err := cluster.Bus.UpdateS3AuthenticationSettings(context.Background(), api.S3AuthenticationSettings{ V4Keypairs: map[string]string{ test.id: test.key, }, diff --git a/internal/worker/cache.go b/internal/worker/cache.go index dfc749d2a..2ec207dc9 100644 --- a/internal/worker/cache.go +++ b/internal/worker/cache.go @@ -183,9 +183,6 @@ func (c *cache) HandleEvent(event webhooks.Event) (err error) { case api.EventSettingUpdate: log = log.With("key", e.Key, "ts", e.Timestamp) err = c.handleSettingUpdate(e) - case api.EventSettingDelete: - log = log.With("key", e.Key, "ts", e.Timestamp) - c.handleSettingDelete(e) default: log.Info("unhandled event", e) return @@ -310,12 +307,6 @@ func (c *cache) handleHostUpdate(e api.EventHostUpdate) { c.cache.Set(cacheKeyDownloadContracts, contracts) } -func (c *cache) handleSettingDelete(e api.EventSettingDelete) { - if e.Key == api.SettingGouging || e.Key == api.SettingRedundancy { - c.cache.Invalidate(cacheKeyGougingParams) - } -} - func (c *cache) handleSettingUpdate(e api.EventSettingUpdate) (err error) { // return early if the cache doesn't have gouging params to update value, found, _ := c.cache.Get(cacheKeyGougingParams) diff --git a/internal/worker/cache_test.go b/internal/worker/cache_test.go index 9bc8d682d..0fa3c10d8 100644 --- a/internal/worker/cache_test.go +++ b/internal/worker/cache_test.go @@ -170,7 +170,6 @@ func TestWorkerCache(t *testing.T) { {Module: api.ModuleContract, Event: api.EventRenew, Payload: nil}, {Module: api.ModuleHost, Event: api.EventUpdate, Payload: nil}, {Module: api.ModuleSetting, Event: api.EventUpdate, Payload: nil}, - {Module: api.ModuleSetting, Event: api.EventDelete, Payload: nil}, } { if err := c.HandleEvent(event); err != nil { t.Fatal(err) diff --git a/stores/settingsdb.go b/stores/settingsdb.go index 7a895108c..ea31b25bd 100644 --- a/stores/settingsdb.go +++ b/stores/settingsdb.go @@ -7,23 +7,6 @@ import ( sql "go.sia.tech/renterd/stores/sql" ) -// DeleteSetting implements the bus.SettingStore interface. -func (s *SQLStore) DeleteSetting(ctx context.Context, key string) error { - s.settingsMu.Lock() - defer s.settingsMu.Unlock() - - // delete from database first - if err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - return tx.DeleteSettings(ctx, key) - }); err != nil { - return err - } - - // delete from cache - delete(s.settings, key) - return nil -} - // Setting implements the bus.SettingStore interface. func (s *SQLStore) Setting(ctx context.Context, key string) (string, error) { // Check cache first. @@ -47,15 +30,6 @@ func (s *SQLStore) Setting(ctx context.Context, key string) (string, error) { return value, nil } -// Settings implements the bus.SettingStore interface. -func (s *SQLStore) Settings(ctx context.Context) (settings []string, err error) { - err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - settings, err = tx.Settings(ctx) - return err - }) - return -} - // UpdateSetting implements the bus.SettingStore interface. func (s *SQLStore) UpdateSetting(ctx context.Context, key, value string) error { // update db first diff --git a/stores/settingsdb_test.go b/stores/settingsdb_test.go index cf2582579..9eda8f546 100644 --- a/stores/settingsdb_test.go +++ b/stores/settingsdb_test.go @@ -2,10 +2,7 @@ package stores import ( "context" - "errors" "testing" - - "go.sia.tech/renterd/api" ) // TestSQLSettingStore tests the bus.SettingStore methods on the SQLSettingStore. @@ -13,52 +10,24 @@ func TestSQLSettingStore(t *testing.T) { ss := newTestSQLStore(t, defaultTestSQLStoreConfig) defer ss.Close() - // assert there are no settings - ctx := context.Background() - if keys, err := ss.Settings(ctx); err != nil { - t.Fatal(err) - } else if len(keys) != 0 { - t.Fatalf("unexpected number of settings, %v != 0", len(keys)) - } - // add a setting - if err := ss.UpdateSetting(ctx, "foo", "bar"); err != nil { + if err := ss.UpdateSetting(context.Background(), "foo", "bar"); err != nil { t.Fatal(err) } - // assert it's returned - if keys, err := ss.Settings(ctx); err != nil { - t.Fatal(err) - } else if len(keys) != 1 { - t.Fatalf("unexpected number of settings, %v != 1", len(keys)) - } else if keys[0] != "foo" { - t.Fatalf("unexpected key, %s != 'foo'", keys[0]) - } - // assert we can query the setting by key - if value, err := ss.Setting(ctx, "foo"); err != nil { + if value, err := ss.Setting(context.Background(), "foo"); err != nil { t.Fatal(err) } else if value != "bar" { t.Fatalf("unexpected value, %s != 'bar'", value) } // assert we can update the setting - if err := ss.UpdateSetting(ctx, "foo", "barbaz"); err != nil { + if err := ss.UpdateSetting(context.Background(), "foo", "barbaz"); err != nil { t.Fatal(err) - } else if value, err := ss.Setting(ctx, "foo"); err != nil { + } else if value, err := ss.Setting(context.Background(), "foo"); err != nil { t.Fatal(err) } else if value != "barbaz" { t.Fatalf("unexpected value, %s != 'barbaz'", value) } - - // delete the setting - if err := ss.DeleteSetting(ctx, "foo"); err != nil { - t.Fatal(err) - } else if _, err := ss.Setting(ctx, "foo"); !errors.Is(err, api.ErrSettingNotFound) { - t.Fatal("should fail with api.ErrSettingNotFound", err) - } else if keys, err := ss.Settings(ctx); err != nil { - t.Fatal(err) - } else if len(keys) != 0 { - t.Fatalf("unexpected number of settings, %v != 0", len(keys)) - } } diff --git a/stores/sql/database.go b/stores/sql/database.go index cc2aab0df..de0a45987 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -150,9 +150,6 @@ type ( // prefix and returns 'true' if any object was deleted. DeleteObjects(ctx context.Context, bucket, prefix string, limit int64) (bool, error) - // DeleteSettings deletes the settings with the given key. - DeleteSettings(ctx context.Context, key string) error - // DeleteWebhook deletes the webhook with the matching module, event and // URL of the provided webhook. If the webhook doesn't exist, // webhooks.ErrWebhookNotFound is returned. @@ -328,9 +325,6 @@ type ( // Setting returns the setting with the given key from the database. Setting(ctx context.Context, key string) (string, error) - // Settings returns all available settings from the database. - Settings(ctx context.Context) ([]string, error) - // Slab returns the slab with the given ID or api.ErrSlabNotFound. Slab(ctx context.Context, key object.EncryptionKey) (object.Slab, error) diff --git a/stores/sql/main.go b/stores/sql/main.go index bb03bd86d..4dd8ee95c 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -541,13 +541,6 @@ func DeleteMetadata(ctx context.Context, tx sql.Tx, objID int64) error { return err } -func DeleteSettings(ctx context.Context, tx sql.Tx, key string) error { - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", key); err != nil { - return fmt.Errorf("failed to delete setting '%s': %w", key, err) - } - return nil -} - func DeleteWebhook(ctx context.Context, tx sql.Tx, wh webhooks.Webhook) error { res, err := tx.Exec(ctx, "DELETE FROM webhooks WHERE module = ? AND event = ? AND url = ?", wh.Module, wh.Event, wh.URL) if err != nil { @@ -2212,23 +2205,6 @@ func Setting(ctx context.Context, tx sql.Tx, key string) (string, error) { return value, nil } -func Settings(ctx context.Context, tx sql.Tx) ([]string, error) { - rows, err := tx.Query(ctx, "SELECT `key` FROM settings") - if err != nil { - return nil, fmt.Errorf("failed to query settings: %w", err) - } - defer rows.Close() - var settings []string - for rows.Next() { - var setting string - if err := rows.Scan(&setting); err != nil { - return nil, fmt.Errorf("failed to scan setting key") - } - settings = append(settings, setting) - } - return settings, nil -} - func SetUncleanShutdown(ctx context.Context, tx sql.Tx) error { _, err := tx.Exec(ctx, "UPDATE ephemeral_accounts SET clean_shutdown = 0, requires_sync = 1") if err != nil { diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index 08ff0010e..b45a24a0f 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -340,10 +340,6 @@ func (tx *MainDatabaseTx) InsertMultipartUpload(ctx context.Context, bucket, key return ssql.InsertMultipartUpload(ctx, tx, bucket, key, ec, mimeType, metadata) } -func (tx *MainDatabaseTx) DeleteSettings(ctx context.Context, key string) error { - return ssql.DeleteSettings(ctx, tx, key) -} - func (tx *MainDatabaseTx) DeleteWebhook(ctx context.Context, wh webhooks.Webhook) error { return ssql.DeleteWebhook(ctx, tx, wh) } @@ -823,10 +819,6 @@ func (tx *MainDatabaseTx) Setting(ctx context.Context, key string) (string, erro return ssql.Setting(ctx, tx, key) } -func (tx *MainDatabaseTx) Settings(ctx context.Context) ([]string, error) { - return ssql.Settings(ctx, tx) -} - func (tx *MainDatabaseTx) SetUncleanShutdown(ctx context.Context) error { return ssql.SetUncleanShutdown(ctx, tx) } diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index b72ec5e8c..4a5e9481a 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -332,10 +332,6 @@ func (tx *MainDatabaseTx) DeleteHostSector(ctx context.Context, hk types.PublicK return ssql.DeleteHostSector(ctx, tx, hk, root) } -func (tx *MainDatabaseTx) DeleteSettings(ctx context.Context, key string) error { - return ssql.DeleteSettings(ctx, tx, key) -} - func (tx *MainDatabaseTx) DeleteWebhook(ctx context.Context, wh webhooks.Webhook) error { return ssql.DeleteWebhook(ctx, tx, wh) } @@ -822,10 +818,6 @@ func (tx *MainDatabaseTx) Setting(ctx context.Context, key string) (string, erro return ssql.Setting(ctx, tx, key) } -func (tx *MainDatabaseTx) Settings(ctx context.Context) ([]string, error) { - return ssql.Settings(ctx, tx) -} - func (tx *MainDatabaseTx) SetUncleanShutdown(ctx context.Context) error { return ssql.SetUncleanShutdown(ctx, tx) } diff --git a/worker/s3/s3.go b/worker/s3/s3.go index d5cbb71a3..efa921030 100644 --- a/worker/s3/s3.go +++ b/worker/s3/s3.go @@ -43,7 +43,6 @@ type Bus interface { MultipartUploadParts(ctx context.Context, bucket, object string, uploadID string, marker int, limit int64) (resp api.MultipartListPartsResponse, _ error) S3AuthenticationSettings(ctx context.Context) (as api.S3AuthenticationSettings, err error) - UpdateSetting(ctx context.Context, key string, value interface{}) error UploadParams(ctx context.Context) (api.UploadParams, error) } From 7542b550a17607d657a0518cd2ec50dd941c4928 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 28 Aug 2024 13:49:51 +0200 Subject: [PATCH 02/33] bus: combine settings --- api/setting.go | 69 +++++++------ autopilot/autopilot.go | 14 +-- bus/bus.go | 154 +++--------------------------- bus/client/settings.go | 50 +++------- bus/routes.go | 82 +++++----------- cmd/renterd/node.go | 19 ++-- internal/bus/pinmanager.go | 8 +- internal/bus/pinmanager_test.go | 6 +- internal/test/config.go | 10 +- internal/test/e2e/cluster.go | 25 +++-- internal/test/e2e/cluster_test.go | 13 ++- internal/test/e2e/s3_test.go | 8 +- internal/worker/cache.go | 2 +- worker/mocks_test.go | 4 +- worker/s3/authentication.go | 4 +- worker/s3/s3.go | 2 +- 16 files changed, 156 insertions(+), 314 deletions(-) diff --git a/api/setting.go b/api/setting.go index 34f5dfd31..3b81eb87e 100644 --- a/api/setting.go +++ b/api/setting.go @@ -10,12 +10,10 @@ import ( ) const ( - SettingContractSet = "contractset" - SettingGouging = "gouging" - SettingPricePinning = "pricepinning" - SettingRedundancy = "redundancy" - SettingS3Authentication = "s3authentication" - SettingUploadPacking = "uploadpacking" + SettingGouging = "gouging" + SettingPinned = "pinned" + SettingS3 = "s3" + SettingUploads = "uploads" ) const ( @@ -53,13 +51,18 @@ var ( // DefaultPricePinSettings define the default price pin settings the bus is // configured with on startup. These values can be adjusted using the // settings API. - DefaultPricePinSettings = PricePinSettings{ + DefaultPricePinSettings = PinnedSettings{ Enabled: false, Currency: "usd", ForexEndpointURL: "https://api.siascan.com/exchange-rate/siacoin", Threshold: 0.05, } + DefaultUploadSettings = UploadSettings{ + Packing: DefaultUploadPackingSettings, + Redundancy: DefaultRedundancySettings, + } + // DefaultUploadPackingSettings define the default upload packing settings // the bus is configured with on startup. DefaultUploadPackingSettings = UploadPackingSettings{ @@ -86,12 +89,6 @@ var ( ) type ( - // ContractSetSetting contains the default contract set used by the worker for - // uploads and migrations. - ContractSetSetting struct { - Default string `json:"default"` - } - // GougingSettings contain some price settings used in price gouging. GougingSettings struct { // MaxRPCPrice is the maximum allowed base price for RPCs @@ -132,10 +129,10 @@ type ( MigrationSurchargeMultiplier uint64 `json:"migrationSurchargeMultiplier"` } - // PricePinSettings holds the configuration for pinning certain settings to - // a specific currency (e.g., USD). It uses a Forex API to fetch the current + // PinnedSettings holds the configuration for pinning certain settings to a + // specific currency (e.g., USD). It uses a Forex API to fetch the current // exchange rate, allowing users to set prices in USD instead of SC. - PricePinSettings struct { + PinnedSettings struct { // Enabled can be used to either enable or temporarily disable price // pinning. If enabled, both the currency and the Forex endpoint URL // must be valid. @@ -160,6 +157,23 @@ type ( GougingSettingsPins GougingSettingsPins `json:"gougingSettingsPins"` } + // UploadSettings contains various settings related to uploads. + UploadSettings struct { + DefaultContractSet string `json:"defaultContractSet"` + Packing UploadPackingSettings `json:"packing"` + Redundancy RedundancySettings `json:"redundancy"` + } + + UploadPackingSettings struct { + Enabled bool `json:"enabled"` + SlabBufferMaxSizeSoft int64 `json:"slabBufferMaxSizeSoft"` + } + + RedundancySettings struct { + MinShards int `json:"minShards"` + TotalShards int `json:"totalShards"` + } + // AutopilotPins contains the available autopilot settings that can be // pinned. AutopilotPins struct { @@ -180,22 +194,15 @@ type ( Value float64 `json:"value"` } - // RedundancySettings contain settings that dictate an object's redundancy. - RedundancySettings struct { - MinShards int `json:"minShards"` - TotalShards int `json:"totalShards"` + // S3Settings contains various settings related to the S3 API. + S3Settings struct { + Authentication S3AuthenticationSettings `json:"authentication"` } // S3AuthenticationSettings contains S3 auth settings. S3AuthenticationSettings struct { V4Keypairs map[string]string `json:"v4Keypairs"` } - - // UploadPackingSettings contains upload packing settings. - UploadPackingSettings struct { - Enabled bool `json:"enabled"` - SlabBufferMaxSizeSoft int64 `json:"slabBufferMaxSizeSoft"` - } ) // IsPinned returns true if the pin is enabled and the value is greater than 0. @@ -204,7 +211,7 @@ func (p Pin) IsPinned() bool { } // Validate returns an error if the price pin settings are not considered valid. -func (pps PricePinSettings) Validate() error { +func (pps PinnedSettings) Validate() error { if pps.ForexEndpointURL == "" { return fmt.Errorf("price pin settings must have a forex endpoint URL") } @@ -281,8 +288,12 @@ func (rs RedundancySettings) Validate() error { // Validate returns an error if the authentication settings are not considered // valid. -func (s3as S3AuthenticationSettings) Validate() error { - for accessKeyID, secretAccessKey := range s3as.V4Keypairs { +func (s3s S3Settings) Validate() error { + return s3s.Authentication.Validate() +} + +func (s3a S3AuthenticationSettings) Validate() error { + for accessKeyID, secretAccessKey := range s3a.V4Keypairs { if accessKeyID == "" { return fmt.Errorf("AccessKeyID cannot be empty") } else if len(accessKeyID) < S3MinAccessKeyLen || len(accessKeyID) > S3MaxAccessKeyLen { diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index fb554fb2c..00728d6ba 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -74,7 +74,7 @@ type Bus interface { // settings GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) - RedundancySettings(ctx context.Context) (rs api.RedundancySettings, err error) + UploadSettings(ctx context.Context) (us api.UploadSettings, err error) // syncer SyncerPeers(ctx context.Context) (resp []string, err error) @@ -847,10 +847,10 @@ func (ap *Autopilot) buildState(ctx context.Context) (*contractor.MaintenanceSta return nil, fmt.Errorf("could not fetch consensus state, err: %v", err) } - // fetch redundancy settings - rs, err := ap.bus.RedundancySettings(ctx) + // fetch upload settings + us, err := ap.bus.UploadSettings(ctx) if err != nil { - return nil, fmt.Errorf("could not fetch redundancy settings, err: %v", err) + return nil, fmt.Errorf("could not fetch upload settings, err: %v", err) } // fetch gouging settings @@ -900,7 +900,7 @@ func (ap *Autopilot) buildState(ctx context.Context) (*contractor.MaintenanceSta return &contractor.MaintenanceState{ GS: gs, - RS: rs, + RS: us.Redundancy, AP: autopilot, Address: address, @@ -935,9 +935,9 @@ func compatV105Host(ctx context.Context, cfg api.ContractsConfig, b Bus, hk type if err != nil { return fmt.Errorf("failed to fetch gouging settings from bus: %w", err) } - _, err = b.RedundancySettings(ctx) + _, err = b.UploadSettings(ctx) if err != nil { - return fmt.Errorf("failed to fetch redundancy settings from bus: %w", err) + return fmt.Errorf("failed to fetch upload settings from bus: %w", err) } _, err = b.ConsensusState(ctx) if err != nil { diff --git a/bus/bus.go b/bus/bus.go index f5117825d..af64411fb 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -11,7 +11,6 @@ import ( "math/big" "net" "net/http" - "strings" "time" "go.sia.tech/core/consensus" @@ -357,11 +356,6 @@ func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksMa rhp2: rhp2.New(rhp.NewFallbackDialer(store, net.Dialer{}, l), l), } - // init settings - if err := b.initSettings(ctx); err != nil { - return nil, err - } - // create account manager b.accountsMgr, err = ibus.NewAccountManager(ctx, store, l) if err != nil { @@ -483,16 +477,14 @@ func (b *Bus) Handler() http.Handler { "DELETE /sectors/:hk/:root": b.sectorsHostRootHandlerDELETE, - "GET /settings/gouging": b.settingsGougingHandlerGET, - "PUT /settings/gouging": b.settingsGougingHandlerPUT, - "GET /settings/pinned": b.settingsPinnedHandlerGET, - "PUT /settings/pinned": b.settingsPinnedHandlerPUT, - "GET /settings/redundancy": b.settingsRedundancyHandlerGET, - "PUT /settings/redundancy": b.settingsRedundancyHandlerPUT, - "GET /settings/s3authentication": b.settingsS3AuthenticationHandlerGET, - "PUT /settings/s3authentication": b.settingsS3AuthenticationHandlerPUT, - "GET /settings/uploadpacking": b.settingsUploadPackingHandlerGET, - "PUT /settings/uploadpacking": b.settingsUploadPackingHandlerPUT, + "GET /settings/gouging": b.settingsGougingHandlerGET, + "PUT /settings/gouging": b.settingsGougingHandlerPUT, + "GET /settings/pinned": b.settingsPinnedHandlerGET, + "PUT /settings/pinned": b.settingsPinnedHandlerPUT, + "GET /settings/s3": b.settingsS3HandlerGET, + "PUT /settings/s3": b.settingsS3HandlerPUT, + "GET /settings/uploads": b.settingsRedundancyHandlerGET, + "PUT /settings/uploads": b.settingsRedundancyHandlerPUT, "POST /slabs/migration": b.slabsMigrationHandlerPOST, "GET /slabs/partial/:key": b.slabsPartialHandlerGET, @@ -611,120 +603,6 @@ func (b *Bus) formContract(ctx context.Context, hostSettings rhpv2.HostSettings, return contract, nil } -// initSettings loads the default settings if the setting is not already set and -// ensures the settings are valid -func (b *Bus) initSettings(ctx context.Context) error { - // testnets have different redundancy settings - defaultRedundancySettings := api.DefaultRedundancySettings - if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { - defaultRedundancySettings = api.DefaultRedundancySettingsTestnet - } - - // load default settings if the setting is not already set - for key, value := range map[string]interface{}{ - api.SettingGouging: api.DefaultGougingSettings, - api.SettingPricePinning: api.DefaultPricePinSettings, - api.SettingRedundancy: defaultRedundancySettings, - api.SettingUploadPacking: api.DefaultUploadPackingSettings, - } { - if _, err := b.ss.Setting(ctx, key); errors.Is(err, api.ErrSettingNotFound) { - if bytes, err := json.Marshal(value); err != nil { - panic("failed to marshal default settings") // should never happen - } else if err := b.ss.UpdateSetting(ctx, key, string(bytes)); err != nil { - return err - } - } - } - - // check redundancy settings for validity - var rs api.RedundancySettings - if rss, err := b.ss.Setting(ctx, api.SettingRedundancy); err != nil { - return err - } else if err := json.Unmarshal([]byte(rss), &rs); err != nil { - return err - } else if err := rs.Validate(); err != nil { - b.logger.Warn(fmt.Sprintf("invalid redundancy setting found '%v', overwriting the redundancy settings with the default settings", rss)) - bytes, _ := json.Marshal(defaultRedundancySettings) - if err := b.ss.UpdateSetting(ctx, api.SettingRedundancy, string(bytes)); err != nil { - return err - } - } - - // check gouging settings for validity - var gs api.GougingSettings - if gss, err := b.ss.Setting(ctx, api.SettingGouging); err != nil { - return err - } else if err := json.Unmarshal([]byte(gss), &gs); err != nil { - return err - } else if err := gs.Validate(); err != nil { - // compat: apply default EA gouging settings - gs.MinMaxEphemeralAccountBalance = api.DefaultGougingSettings.MinMaxEphemeralAccountBalance - gs.MinPriceTableValidity = api.DefaultGougingSettings.MinPriceTableValidity - gs.MinAccountExpiry = api.DefaultGougingSettings.MinAccountExpiry - if err := gs.Validate(); err == nil { - b.logger.Info(fmt.Sprintf("updating gouging settings with default EA settings: %+v", gs)) - bytes, _ := json.Marshal(gs) - if err := b.ss.UpdateSetting(ctx, api.SettingGouging, string(bytes)); err != nil { - return err - } - } else { - // compat: apply default host block leeway settings - gs.HostBlockHeightLeeway = api.DefaultGougingSettings.HostBlockHeightLeeway - if err := gs.Validate(); err == nil { - b.logger.Info(fmt.Sprintf("updating gouging settings with default HostBlockHeightLeeway settings: %v", gs)) - bytes, _ := json.Marshal(gs) - if err := b.ss.UpdateSetting(ctx, api.SettingGouging, string(bytes)); err != nil { - return err - } - } else { - b.logger.Warn(fmt.Sprintf("invalid gouging setting found '%v', overwriting the gouging settings with the default settings", gss)) - bytes, _ := json.Marshal(api.DefaultGougingSettings) - if err := b.ss.UpdateSetting(ctx, api.SettingGouging, string(bytes)); err != nil { - return err - } - } - } - } - - // compat: default price pin settings - var pps api.PricePinSettings - if pss, err := b.ss.Setting(ctx, api.SettingPricePinning); err != nil { - return err - } else if err := json.Unmarshal([]byte(pss), &pps); err != nil { - return err - } else if err := pps.Validate(); err != nil { - // overwrite values with defaults - var updates []string - if pps.ForexEndpointURL == "" { - pps.ForexEndpointURL = api.DefaultPricePinSettings.ForexEndpointURL - updates = append(updates, fmt.Sprintf("set PricePinSettings.ForexEndpointURL to %v", pps.ForexEndpointURL)) - } - if pps.Currency == "" { - pps.Currency = api.DefaultPricePinSettings.Currency - updates = append(updates, fmt.Sprintf("set PricePinSettings.Currency to %v", pps.Currency)) - } - if pps.Threshold == 0 { - pps.Threshold = api.DefaultPricePinSettings.Threshold - updates = append(updates, fmt.Sprintf("set PricePinSettings.Threshold to %v", pps.Threshold)) - } - - var updated []byte - if err := pps.Validate(); err == nil { - b.logger.Info(fmt.Sprintf("updating price pinning settings with default values: %v", strings.Join(updates, ", "))) - updated, _ = json.Marshal(pps) - } else { - b.logger.Warn(fmt.Sprintf("updated price pinning settings are invalid (%v), they have been overwritten with the default settings", err)) - updated, _ = json.Marshal(api.DefaultPricePinSettings) - } - - if err := b.ss.UpdateSetting(ctx, api.SettingPricePinning, string(updated)); err != nil { - return err - } - } - - return nil -} - func (b *Bus) deriveRenterKey(hostKey types.PublicKey) types.PrivateKey { seed := blake2b.Sum256(append(b.deriveSubKey("renterkey"), hostKey[:]...)) pk := types.NewPrivateKeyFromSeed(seed[:]) @@ -744,17 +622,15 @@ func (b *Bus) deriveSubKey(purpose string) types.PrivateKey { } func (b *Bus) fetchSetting(ctx context.Context, key string, value interface{}) error { - // testnets have different redundancy settings - defaultRedundancySettings := api.DefaultRedundancySettings - if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { - defaultRedundancySettings = api.DefaultRedundancySettingsTestnet + defaults := map[string]interface{}{ + api.SettingGouging: api.DefaultGougingSettings, + api.SettingPinned: api.DefaultPricePinSettings, + api.SettingUploads: api.DefaultUploadSettings, } - defaults := map[string]interface{}{ - api.SettingGouging: api.DefaultGougingSettings, - api.SettingPricePinning: api.DefaultPricePinSettings, - api.SettingRedundancy: defaultRedundancySettings, - api.SettingUploadPacking: api.DefaultUploadPackingSettings, + // testnets have different redundancy settings + if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { + defaults[api.SettingUploads] = api.DefaultRedundancySettingsTestnet } setting, err := b.ss.Setting(ctx, key) diff --git a/bus/client/settings.go b/bus/client/settings.go index e813a7417..35c48eac2 100644 --- a/bus/client/settings.go +++ b/bus/client/settings.go @@ -6,17 +6,6 @@ import ( "go.sia.tech/renterd/api" ) -// ContractSetSettings returns the contract set settings. -func (c *Client) ContractSetSettings(ctx context.Context) (css api.ContractSetSetting, err error) { - err = c.c.WithContext(ctx).GET("/setting/contractset", &css) - return -} - -// UpdateContractSetSetting updates the given setting. -func (c *Client) UpdateContractSetSetting(ctx context.Context, css api.ContractSetSetting) error { - return c.c.WithContext(ctx).PUT("/setting/contractset", css) -} - // GougingSettings returns the gouging settings. func (c *Client) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { err = c.c.WithContext(ctx).GET("/setting/gouging", &gs) @@ -29,45 +18,34 @@ func (c *Client) UpdateGougingSettings(ctx context.Context, gs api.GougingSettin } // PricePinningSettings returns the contract set settings. -func (c *Client) PricePinningSettings(ctx context.Context) (pps api.PricePinSettings, err error) { +func (c *Client) PricePinningSettings(ctx context.Context) (pps api.PinnedSettings, err error) { err = c.c.WithContext(ctx).GET("/setting/pinned", &pps) return } // UpdatePinnedSettings updates the given setting. -func (c *Client) UpdatePinnedSettings(ctx context.Context, pps api.PricePinSettings) error { +func (c *Client) UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error { return c.c.WithContext(ctx).PUT("/setting/pinned", pps) } -// RedundancySettings returns the redundancy settings. -func (c *Client) RedundancySettings(ctx context.Context) (rs api.RedundancySettings, err error) { - err = c.c.WithContext(ctx).GET("/setting/redundancy", &rs) - return -} - -// UpdateRedundancySettings updates the given setting. -func (c *Client) UpdateRedundancySettings(ctx context.Context, rs api.RedundancySettings) error { - return c.c.WithContext(ctx).PUT("/setting/redundancy", rs) -} - -// S3AuthenticationSettings returns the S3 authentication settings. -func (c *Client) S3AuthenticationSettings(ctx context.Context) (as api.S3AuthenticationSettings, err error) { - err = c.c.WithContext(ctx).GET("/setting/s3authentication", &as) +// S3Settings returns the S3 settings. +func (c *Client) S3Settings(ctx context.Context) (as api.S3Settings, err error) { + err = c.c.WithContext(ctx).GET("/setting/s3", &as) return } -// UpdateS3AuthenticationSettings updates the given setting. -func (c *Client) UpdateS3AuthenticationSettings(ctx context.Context, as api.S3AuthenticationSettings) error { - return c.c.WithContext(ctx).PUT("/setting/s3authentication", as) +// UpdateS3Settings updates the given setting. +func (c *Client) UpdateS3Settings(ctx context.Context, as api.S3Settings) error { + return c.c.WithContext(ctx).PUT("/setting/s3", as) } -// UploadPackingSettings returns the upload packing settings. -func (c *Client) UploadPackingSettings(ctx context.Context) (ups api.UploadPackingSettings, err error) { - err = c.c.WithContext(ctx).GET("/setting/uploadpacking", &ups) +// UploadSettings returns the upload settings. +func (c *Client) UploadSettings(ctx context.Context) (css api.UploadSettings, err error) { + err = c.c.WithContext(ctx).GET("/setting/upload", &css) return } -// UpdateUploadPackingSettings updates the given setting. -func (c *Client) UpdateUploadPackingSettings(ctx context.Context, ups api.UploadPackingSettings) error { - return c.c.WithContext(ctx).PUT("/setting/uploadpacking", ups) +// UpdateUploadSettings update the given setting. +func (c *Client) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error { + return c.c.WithContext(ctx).PUT("/setting/upload", us) } diff --git a/bus/routes.go b/bus/routes.go index 76c7df4ec..a95d62ac3 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -1284,8 +1284,8 @@ func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { } func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { - var pps api.PricePinSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingPricePinning, &pps); errors.Is(err, api.ErrSettingNotFound) { + var pps api.PinnedSettings + if err := b.fetchSetting(jc.Request.Context(), api.SettingPinned, &pps); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) } else if jc.Check("failed to get price pinning settings", err) == nil { // populate the Autopilots map with the current autopilots @@ -1303,7 +1303,7 @@ func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { } func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { - var pps api.PricePinSettings + var pps api.PinnedSettings if jc.Decode(&pps) != nil { return } else if err := pps.Validate(); err != nil { @@ -1324,14 +1324,14 @@ func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { } // update the setting - if jc.Check("could not update price pinning settings", b.updateSetting(jc.Request.Context(), api.SettingPricePinning, string(data), true)) != nil { + if jc.Check("could not update price pinning settings", b.updateSetting(jc.Request.Context(), api.SettingPinned, string(data), true)) != nil { return } } func (b *Bus) settingsRedundancyHandlerGET(jc jape.Context) { var rs api.RedundancySettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingRedundancy, &rs); errors.Is(err, api.ErrSettingNotFound) { + if err := b.fetchSetting(jc.Request.Context(), api.SettingUploads, &rs); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) } else if jc.Check("failed to get redundancy settings", err) == nil { jc.Encode(rs) @@ -1355,22 +1355,22 @@ func (b *Bus) settingsRedundancyHandlerPUT(jc jape.Context) { } // update the setting - if jc.Check("could not update redundancy settings", b.updateSetting(jc.Request.Context(), api.SettingRedundancy, string(data), false)) != nil { + if jc.Check("could not update redundancy settings", b.updateSetting(jc.Request.Context(), api.SettingUploads, string(data), false)) != nil { return } } -func (b *Bus) settingsS3AuthenticationHandlerGET(jc jape.Context) { - var s3as api.S3AuthenticationSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingS3Authentication, &s3as); errors.Is(err, api.ErrSettingNotFound) { +func (b *Bus) settingsS3HandlerGET(jc jape.Context) { + var s3as api.S3Settings + if err := b.fetchSetting(jc.Request.Context(), api.SettingS3, &s3as); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) } else if jc.Check("failed to get s3 authentication settings", err) == nil { jc.Encode(s3as) } } -func (b *Bus) settingsS3AuthenticationHandlerPUT(jc jape.Context) { - var s3as api.S3AuthenticationSettings +func (b *Bus) settingsS3HandlerPUT(jc jape.Context) { + var s3as api.S3Settings if jc.Decode(&s3as) != nil { return } else if err := s3as.Validate(); err != nil { @@ -1386,38 +1386,7 @@ func (b *Bus) settingsS3AuthenticationHandlerPUT(jc jape.Context) { } // update the setting - if jc.Check("could not update s3 authentication settings", b.updateSetting(jc.Request.Context(), api.SettingS3Authentication, string(data), false)) != nil { - return - } -} - -func (b *Bus) settingsUploadPackingHandlerGET(jc jape.Context) { - var ups api.UploadPackingSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingUploadPacking, &ups); errors.Is(err, api.ErrSettingNotFound) { - jc.Error(err, http.StatusNotFound) - } else if jc.Check("failed to get upload packing settings", err) == nil { - jc.Encode(ups) - } -} - -func (b *Bus) settingsUploadPackingHandlerPUT(jc jape.Context) { - var ups api.UploadPackingSettings - if jc.Decode(&ups) != nil { - return - } else if err := ups.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update upload packing settings, error: %v", err), http.StatusBadRequest) - return - } - - // marshal the setting - data, err := json.Marshal(ups) - if err != nil { - jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) - return - } - - // update the setting - if jc.Check("could not update upload packing settings", b.updateSetting(jc.Request.Context(), api.SettingUploadPacking, string(data), false)) != nil { + if jc.Check("could not update s3 authentication settings", b.updateSetting(jc.Request.Context(), api.SettingS3, string(data), false)) != nil { return } } @@ -1555,14 +1524,14 @@ func (b *Bus) slabsPartialHandlerPOST(jc jape.Context) { if jc.Check("failed to add partial slab", err) != nil { return } - var pus api.UploadPackingSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingUploadPacking, &pus); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + var us api.UploadSettings + if err := b.fetchSetting(jc.Request.Context(), api.SettingUploads, &us); err != nil && !errors.Is(err, api.ErrSettingNotFound) { jc.Error(fmt.Errorf("could not get upload packing settings: %w", err), http.StatusInternalServerError) return } jc.Encode(api.AddPartialSlabResponse{ Slabs: slabs, - SlabBufferMaxSizeSoftReached: bufferSize >= pus.SlabBufferMaxSizeSoft, + SlabBufferMaxSizeSoftReached: bufferSize >= us.Packing.SlabBufferMaxSizeSoft, }) } @@ -1588,22 +1557,15 @@ func (b *Bus) paramsHandlerUploadGET(jc jape.Context) { return } - var contractSet string - var css api.ContractSetSetting - if err := b.fetchSetting(jc.Request.Context(), api.SettingContractSet, &css); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - jc.Error(fmt.Errorf("could not get contract set settings: %w", err), http.StatusInternalServerError) - return - } else if err == nil { - contractSet = css.Default - } - var uploadPacking bool - var pus api.UploadPackingSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingUploadPacking, &pus); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - jc.Error(fmt.Errorf("could not get upload packing settings: %w", err), http.StatusInternalServerError) + var contractSet string + var us api.UploadSettings + if err := b.fetchSetting(jc.Request.Context(), api.SettingUploads, &us); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + jc.Error(fmt.Errorf("could not get upload settings: %w", err), http.StatusInternalServerError) return } else if err == nil { - uploadPacking = pus.Enabled + contractSet = us.DefaultContractSet + uploadPacking = us.Packing.Enabled } jc.Encode(api.UploadParams{ @@ -1650,7 +1612,7 @@ func (b *Bus) gougingParams(ctx context.Context) (api.GougingParams, error) { } var rs api.RedundancySettings - if rss, err := b.ss.Setting(ctx, api.SettingRedundancy); err != nil { + if rss, err := b.ss.Setting(ctx, api.SettingUploads); err != nil { return api.GougingParams{}, err } else if err := json.Unmarshal([]byte(rss), &rs); err != nil { b.logger.Panicf("failed to unmarshal redundancy settings '%s': %v", rss, err) diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 5ab3bf13d..28d8e791d 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -413,30 +413,31 @@ func (n *node) Run() error { // set initial S3 keys if n.cfg.S3.Enabled && !n.cfg.S3.DisableAuth { - as, err := n.bus.S3AuthenticationSettings(context.Background()) + s3s, err := n.bus.S3Settings(context.Background()) if err != nil && !strings.Contains(err.Error(), api.ErrSettingNotFound.Error()) { - return fmt.Errorf("failed to fetch S3 authentication settings: %w", err) - } else if as.V4Keypairs == nil { - as.V4Keypairs = make(map[string]string) + return fmt.Errorf("failed to fetch S3 settings: %w", err) + } else if s3s.Authentication.V4Keypairs == nil { + s3s.Authentication.V4Keypairs = make(map[string]string) } // S3 key pair validation was broken at one point, we need to remove the // invalid key pairs here to ensure we don't fail when we update the // setting below. - for k, v := range as.V4Keypairs { + for k, v := range s3s.Authentication.V4Keypairs { if err := (api.S3AuthenticationSettings{V4Keypairs: map[string]string{k: v}}).Validate(); err != nil { n.logger.Infof("removing invalid S3 keypair for AccessKeyID %s, reason: %v", k, err) - delete(as.V4Keypairs, k) + delete(s3s.Authentication.V4Keypairs, k) } } // merge keys for k, v := range n.cfg.S3.KeypairsV4 { - as.V4Keypairs[k] = v + s3s.Authentication.V4Keypairs[k] = v } + // update settings - if err := n.bus.UpdateS3AuthenticationSettings(context.Background(), as); err != nil { - return fmt.Errorf("failed to update S3 authentication settings: %w", err) + if err := n.bus.UpdateS3Settings(context.Background(), s3s); err != nil { + return fmt.Errorf("failed to update S3 settings: %w", err) } } diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index c128a8392..9274f0920 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -103,10 +103,10 @@ func (pm *pinManager) averageRate() decimal.Decimal { return decimal.NewFromFloat(median) } -func (pm *pinManager) pinnedSettings(ctx context.Context) (api.PricePinSettings, error) { - var ps api.PricePinSettings - if pss, err := pm.s.Setting(ctx, api.SettingPricePinning); err != nil { - return api.PricePinSettings{}, err +func (pm *pinManager) pinnedSettings(ctx context.Context) (api.PinnedSettings, error) { + var ps api.PinnedSettings + if pss, err := pm.s.Setting(ctx, api.SettingPinned); err != nil { + return api.PinnedSettings{}, err } else if err := json.Unmarshal([]byte(pss), &ps); err != nil { pm.logger.Panicf("failed to unmarshal pinned settings '%s': %v", pss, err) } diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index e5158836d..33e4962eb 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -123,7 +123,7 @@ func newTestStore() *mockPinStore { // add default price pin - and gouging settings b, _ := json.Marshal(api.DefaultPricePinSettings) - s.settings[api.SettingPricePinning] = string(b) + s.settings[api.SettingPinned] = string(b) b, _ = json.Marshal(api.DefaultGougingSettings) s.settings[api.SettingGouging] = string(b) @@ -152,9 +152,9 @@ func (ms *mockPinStore) gougingSettings() api.GougingSettings { return gs } -func (ms *mockPinStore) updatPinnedSettings(pps api.PricePinSettings) { +func (ms *mockPinStore) updatPinnedSettings(pps api.PinnedSettings) { b, _ := json.Marshal(pps) - ms.UpdateSetting(context.Background(), api.SettingPricePinning, string(b)) + ms.UpdateSetting(context.Background(), api.SettingPinned, string(b)) time.Sleep(2 * testUpdateInterval) } diff --git a/internal/test/config.go b/internal/test/config.go index 1b5d926a0..877b6168e 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -33,10 +33,7 @@ var ( }, } - ContractSet = "testset" - ContractSetSettings = api.ContractSetSetting{ - Default: ContractSet, - } + ContractSet = "testset" GougingSettings = api.GougingSettings{ MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC @@ -59,6 +56,11 @@ var ( TotalShards: 3, } + UploadSettings = api.UploadSettings{ + DefaultContractSet: ContractSet, + Redundancy: RedundancySettings, + } + S3AccessKeyID = "TESTINGYNHUWCPKOPSYQ" S3SecretAccessKey = "Rh30BNyj+qNI4ftYRteoZbHJ3X4Ln71QtZkRXzJ9" S3Credentials = credentials.NewStaticV4(S3AccessKeyID, S3SecretAccessKey, "") diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 189a92e43..d84b87b17 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -435,18 +435,25 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { })) } + // Build upload settings. + us := test.UploadSettings + us.Packing = api.UploadPackingSettings{ + Enabled: enableUploadPacking, + SlabBufferMaxSizeSoft: api.DefaultUploadPackingSettings.SlabBufferMaxSizeSoft, + } + + // Build S3 settings. + s3 := api.S3Settings{ + Authentication: api.S3AuthenticationSettings{ + V4Keypairs: map[string]string{test.S3AccessKeyID: test.S3SecretAccessKey}, + }, + } + // Update the bus settings. tt.OK(busClient.UpdateGougingSettings(ctx, test.GougingSettings)) - tt.OK(busClient.UpdateContractSetSetting(ctx, test.ContractSetSettings)) tt.OK(busClient.UpdatePinnedSettings(ctx, test.PricePinSettings)) - tt.OK(busClient.UpdateRedundancySettings(ctx, test.RedundancySettings)) - tt.OK(busClient.UpdateS3AuthenticationSettings(ctx, api.S3AuthenticationSettings{ - V4Keypairs: map[string]string{test.S3AccessKeyID: test.S3SecretAccessKey}, - })) - tt.OK(busClient.UpdateUploadPackingSettings(ctx, api.UploadPackingSettings{ - Enabled: enableUploadPacking, - SlabBufferMaxSizeSoft: api.DefaultUploadPackingSettings.SlabBufferMaxSizeSoft, - })) + tt.OK(busClient.UpdateUploadSettings(ctx, us)) + tt.OK(busClient.UpdateS3Settings(ctx, s3)) // Fund the bus. if funding { diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 2c1c1e16e..902c922e4 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -178,10 +178,10 @@ func TestNewTestCluster(t *testing.T) { tt := cluster.tt // Upload packing should be disabled by default. - ups, err := b.UploadPackingSettings(context.Background()) + us, err := b.UploadSettings(context.Background()) tt.OK(err) - if ups.Enabled { - t.Fatalf("expected upload packing to be disabled by default, got %v", ups.Enabled) + if us.Packing.Enabled { + t.Fatalf("expected upload packing to be disabled by default, got %v", us.Packing.Enabled) } // PricePinningSettings should have default values @@ -1142,10 +1142,13 @@ func TestEphemeralAccounts(t *testing.T) { w := cluster.Worker tt := cluster.tt - tt.OK(b.UpdateRedundancySettings(context.Background(), api.RedundancySettings{ + us := test.UploadSettings + us.Redundancy = api.RedundancySettings{ MinShards: 1, TotalShards: 1, - })) + } + tt.OK(b.UpdateUploadSettings(context.Background(), us)) + // add a host hosts := cluster.AddHosts(1) h, err := b.Host(context.Background(), hosts[0].PublicKey()) diff --git a/internal/test/e2e/s3_test.go b/internal/test/e2e/s3_test.go index 5775d2292..272ead406 100644 --- a/internal/test/e2e/s3_test.go +++ b/internal/test/e2e/s3_test.go @@ -825,9 +825,11 @@ func TestS3SettingsValidate(t *testing.T) { }, } for i, test := range tests { - err := cluster.Bus.UpdateS3AuthenticationSettings(context.Background(), api.S3AuthenticationSettings{ - V4Keypairs: map[string]string{ - test.id: test.key, + err := cluster.Bus.UpdateS3Settings(context.Background(), api.S3Settings{ + Authentication: api.S3AuthenticationSettings{ + V4Keypairs: map[string]string{ + test.id: test.key, + }, }, }) if err != nil && !test.shouldFail { diff --git a/internal/worker/cache.go b/internal/worker/cache.go index 2ec207dc9..f6c1ea574 100644 --- a/internal/worker/cache.go +++ b/internal/worker/cache.go @@ -333,7 +333,7 @@ func (c *cache) handleSettingUpdate(e api.EventSettingUpdate) (err error) { gp.GougingSettings = gs c.cache.Set(cacheKeyGougingParams, gp) - case api.SettingRedundancy: + case api.SettingUploads: var rs api.RedundancySettings if err := json.Unmarshal(data, &rs); err != nil { return fmt.Errorf("couldn't update redundancy settings, invalid request body, %t", e.Update) diff --git a/worker/mocks_test.go b/worker/mocks_test.go index 13e5fd733..203afe7e1 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -678,8 +678,8 @@ func (*s3Mock) MultipartUploadParts(ctx context.Context, bucket, object string, return api.MultipartListPartsResponse{}, nil } -func (*s3Mock) S3AuthenticationSettings(context.Context) (as api.S3AuthenticationSettings, err error) { - return api.S3AuthenticationSettings{}, nil +func (*s3Mock) S3Settings(context.Context) (as api.S3Settings, err error) { + return api.S3Settings{}, nil } func (*s3Mock) UpdateSetting(context.Context, string, interface{}) error { diff --git a/worker/s3/authentication.go b/worker/s3/authentication.go index 58ebad677..066e27e53 100644 --- a/worker/s3/authentication.go +++ b/worker/s3/authentication.go @@ -117,11 +117,11 @@ func (b *authenticatedBackend) permsFromCtx(ctx context.Context, bucket string) } func (b *authenticatedBackend) reloadV4Keys(ctx context.Context) error { - as, err := b.backend.b.S3AuthenticationSettings(ctx) + s3, err := b.backend.b.S3Settings(ctx) if err != nil { return err } - signature.ReloadKeys(as.V4Keypairs) + signature.ReloadKeys(s3.Authentication.V4Keypairs) return nil } diff --git a/worker/s3/s3.go b/worker/s3/s3.go index efa921030..c0e2c054e 100644 --- a/worker/s3/s3.go +++ b/worker/s3/s3.go @@ -42,7 +42,7 @@ type Bus interface { MultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker string, maxUploads int) (resp api.MultipartListUploadsResponse, _ error) MultipartUploadParts(ctx context.Context, bucket, object string, uploadID string, marker int, limit int64) (resp api.MultipartListPartsResponse, _ error) - S3AuthenticationSettings(ctx context.Context) (as api.S3AuthenticationSettings, err error) + S3Settings(ctx context.Context) (as api.S3Settings, err error) UploadParams(ctx context.Context) (api.UploadParams, error) } From f43d5d7b86b90421a81ccc9ce19602d157d0ee3d Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 Aug 2024 09:34:12 +0200 Subject: [PATCH 03/33] stores: add compat code --- .github/ISSUE_TEMPLATE/bug_report.yml | 8 +- README.md | 40 +------ api/setting.go | 7 ++ bus/bus.go | 120 +++++++++++++------- bus/client/settings.go | 16 +-- bus/routes.go | 151 ++++++++++++-------------- internal/bus/pinmanager.go | 35 ++---- internal/bus/pinmanager_test.go | 72 ++++++------ stores/settingsdb.go | 115 +++++++++++++++++++- stores/settingsdb_test.go | 33 ------ stores/sql/database.go | 3 + stores/sql/main.go | 7 ++ stores/sql/mysql/main.go | 4 + stores/sql/sqlite/main.go | 4 + 14 files changed, 339 insertions(+), 276 deletions(-) delete mode 100644 stores/settingsdb_test.go diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 04504c086..5219b07cf 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -66,10 +66,10 @@ body: description: | The configuration of your bus ```bash - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/contractset - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/gouging - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/redundancy - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/uploadpacking + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/contractset + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/gouging + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/redundancy + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/uploadpacking ``` placeholder: Paste the output of the above commands here validations: diff --git a/README.md b/README.md index c20749935..7508620c5 100644 --- a/README.md +++ b/README.md @@ -558,49 +558,13 @@ formed. } ``` -### Contract Set - -The contract set settings on the bus allow specifying a default contract set. -This contract set will be returned by the `bus` through the upload parameters, -and decides what contracts data is upload or migrated to by default. This -setting does not have a default value, it can be updated using the settings API: - -- `GET /api/bus/setting/contractset` -- `PUT /api/bus/setting/contractset` - -```json -{ - "default": "autopilot" -} -``` - -In most cases the default set should match the set from your autopilot -configuration in order for migrations to work properly. The contract set can be -overridden by passing it as a query string parameter to the worker's upload and -migrate endpoints. - -- `PUT /api/worker/objects/foo?contractset=foo` - -### Redundancy - -The default redundancy on mainnet is 30-10, on testnet it is 6-2. The redundancy -can be updated using the settings API: - -- `GET /api/bus/setting/redundancy` -- `PUT /api/bus/setting/redundancy` - -The redundancy can also be passed through query string parameters on the upload -endpoint in the worker API: - -- `PUT /api/worker/objects/foo?minshards=2&totalshards=5` - ### Gouging The default gouging settings are listed below. The gouging settings can be updated using the settings API: -- `GET /api/bus/setting/gouging` -- `PUT /api/bus/setting/gouging` +- `GET /api/bus/settings/gouging` +- `PUT /api/bus/settings/gouging` ```json { diff --git a/api/setting.go b/api/setting.go index 3b81eb87e..f3e0f2b63 100644 --- a/api/setting.go +++ b/api/setting.go @@ -246,6 +246,13 @@ func (gs GougingSettings) Validate() error { return nil } +func (us UploadSettings) Validate() error { + return errors.Join( + us.Packing.Validate(), + us.Redundancy.Validate(), + ) +} + // Validate returns an error if the upload packing settings are not considered // valid. func (up UploadPackingSettings) Validate() error { diff --git a/bus/bus.go b/bus/bus.go index 81c52f99c..e1e5563f9 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -5,7 +5,6 @@ package bus import ( "context" - "encoding/json" "errors" "fmt" "net" @@ -278,8 +277,21 @@ type ( // A SettingStore stores settings. SettingStore interface { - Setting(ctx context.Context, key string) (string, error) - UpdateSetting(ctx context.Context, key, value string) error + GougingSettings(ctx context.Context) (api.GougingSettings, error) + UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error + + PinnedSettings(ctx context.Context) (api.PinnedSettings, error) + UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error + + UploadSettings(ctx context.Context) (api.UploadSettings, error) + UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error + + S3Settings(ctx context.Context) (api.S3Settings, error) + UpdateS3Settings(ctx context.Context, s3as api.S3Settings) error + + // required for compat + Setting(ctx context.Context, key string, out interface{}) error + DeleteSetting(ctx context.Context, key string) error } WalletMetricsRecorder interface { @@ -357,6 +369,11 @@ func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksMa // create wallet metrics recorder b.walletMetricsRecorder = ibus.NewWalletMetricRecorder(store, w, defaultWalletRecordMetricInterval, l) + // migrate settings to V2 types + if err := b.compatV2Settings(ctx); err != nil { + return nil, err + } + return b, nil } @@ -457,8 +474,8 @@ func (b *Bus) Handler() http.Handler { "PUT /settings/pinned": b.settingsPinnedHandlerPUT, "GET /settings/s3": b.settingsS3HandlerGET, "PUT /settings/s3": b.settingsS3HandlerPUT, - "GET /settings/uploads": b.settingsRedundancyHandlerGET, - "PUT /settings/uploads": b.settingsRedundancyHandlerPUT, + "GET /settings/uploads": b.settingsUploadsHandlerGET, + "PUT /settings/uploads": b.settingsUploadsHandlerPUT, "POST /slabs/migration": b.slabsMigrationHandlerPOST, "GET /slabs/partial/:key": b.slabsPartialHandlerGET, @@ -594,56 +611,75 @@ func (b *Bus) deriveSubKey(purpose string) types.PrivateKey { return pk } -func (b *Bus) fetchSetting(ctx context.Context, key string, value interface{}) error { - defaults := map[string]interface{}{ - api.SettingGouging: api.DefaultGougingSettings, - api.SettingPinned: api.DefaultPricePinSettings, - api.SettingUploads: api.DefaultUploadSettings, +func (b *Bus) compatV2Settings(ctx context.Context) error { + // escape early if all settings are present + if !errors.Is(errors.Join( + b.ss.Setting(ctx, api.SettingGouging, struct{}{}), + b.ss.Setting(ctx, api.SettingPinned, struct{}{}), + b.ss.Setting(ctx, api.SettingS3, struct{}{}), + b.ss.Setting(ctx, api.SettingUploads, struct{}{}), + ), api.ErrAutopilotNotFound) { + return nil } - // testnets have different redundancy settings - if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { - defaults[api.SettingUploads] = api.DefaultRedundancySettingsTestnet + // migrate S3 settings + var s3as api.S3AuthenticationSettings + if err := b.ss.Setting(ctx, "s3authentication", &s3as); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + return err + } else if err == nil { + s3s := api.S3Settings{Authentication: s3as} + if err := b.ss.UpdateS3Settings(ctx, s3s); err != nil { + return err + } } - setting, err := b.ss.Setting(ctx, key) - if errors.Is(err, api.ErrSettingNotFound) { - val, ok := defaults[key] - if !ok { - return fmt.Errorf("%w: unknown setting '%s'", api.ErrSettingNotFound, key) + // migrate pinned settings + var pps api.PinnedSettings + if err := b.ss.Setting(ctx, "pricepinning", &pps); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + return err + } else if errors.Is(err, api.ErrSettingNotFound) { + if err := b.ss.UpdatePinnedSettings(ctx, api.DefaultPricePinSettings); err != nil { + return err } - - bytes, _ := json.Marshal(val) - if err := b.ss.UpdateSetting(ctx, key, string(bytes)); err != nil { - b.logger.Warn(fmt.Sprintf("failed to update default setting '%s': %v", key, err)) + } else { + if err := b.ss.UpdatePinnedSettings(ctx, pps); err != nil { + return err } - return json.Unmarshal(bytes, &val) - } else if err != nil { - return err } - return json.Unmarshal([]byte(setting), &value) -} + // migrate upload settings + us := api.DefaultUploadSettings + var css struct { + Default string `json:"default"` + } -func (b *Bus) updateSetting(ctx context.Context, key string, value string, updatePinMgr bool) error { - err := b.ss.UpdateSetting(ctx, key, value) - if err != nil { + // override default contract set on default upload settings + if err := b.ss.Setting(ctx, "contractset", &css); err != nil && !errors.Is(err, api.ErrSettingNotFound) { return err + } else if err == nil { + us.DefaultContractSet = css.Default } - b.broadcastAction(webhooks.Event{ - Module: api.ModuleSetting, - Event: api.EventUpdate, - Payload: api.EventSettingUpdate{ - Key: key, - Update: value, - Timestamp: time.Now().UTC(), - }, - }) + // override redundancy settings on default upload settings + var rs api.RedundancySettings + if err := b.ss.Setting(ctx, "redundancy", &rs); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + return err + } else if errors.Is(err, api.ErrSettingNotFound) { + // default redundancy settings for testnet are different from mainnet + if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { + us.Redundancy = api.DefaultRedundancySettingsTestnet + } + } else { + us.Redundancy = rs + } - if updatePinMgr { - b.pinMgr.TriggerUpdate() + // override upload packing settings on default upload settings + var ups api.UploadPackingSettings + if err := b.ss.Setting(ctx, "uploadpacking", &ups); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + return err + } else if err == nil { + us.Packing = ups } - return nil + return b.ss.UpdateUploadSettings(ctx, us) } diff --git a/bus/client/settings.go b/bus/client/settings.go index 35c48eac2..5723c3cdd 100644 --- a/bus/client/settings.go +++ b/bus/client/settings.go @@ -8,44 +8,44 @@ import ( // GougingSettings returns the gouging settings. func (c *Client) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { - err = c.c.WithContext(ctx).GET("/setting/gouging", &gs) + err = c.c.WithContext(ctx).GET("/settings/gouging", &gs) return } // UpdateGougingSettings updates the given setting. func (c *Client) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { - return c.c.WithContext(ctx).PUT("/setting/gouging", gs) + return c.c.WithContext(ctx).PUT("/settings/gouging", gs) } // PricePinningSettings returns the contract set settings. func (c *Client) PricePinningSettings(ctx context.Context) (pps api.PinnedSettings, err error) { - err = c.c.WithContext(ctx).GET("/setting/pinned", &pps) + err = c.c.WithContext(ctx).GET("/settings/pinned", &pps) return } // UpdatePinnedSettings updates the given setting. func (c *Client) UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error { - return c.c.WithContext(ctx).PUT("/setting/pinned", pps) + return c.c.WithContext(ctx).PUT("/settings/pinned", pps) } // S3Settings returns the S3 settings. func (c *Client) S3Settings(ctx context.Context) (as api.S3Settings, err error) { - err = c.c.WithContext(ctx).GET("/setting/s3", &as) + err = c.c.WithContext(ctx).GET("/settings/s3", &as) return } // UpdateS3Settings updates the given setting. func (c *Client) UpdateS3Settings(ctx context.Context, as api.S3Settings) error { - return c.c.WithContext(ctx).PUT("/setting/s3", as) + return c.c.WithContext(ctx).PUT("/settings/s3", as) } // UploadSettings returns the upload settings. func (c *Client) UploadSettings(ctx context.Context) (css api.UploadSettings, err error) { - err = c.c.WithContext(ctx).GET("/setting/upload", &css) + err = c.c.WithContext(ctx).GET("/settings/upload", &css) return } // UpdateUploadSettings update the given setting. func (c *Client) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error { - return c.c.WithContext(ctx).PUT("/setting/upload", us) + return c.c.WithContext(ctx).PUT("/settings/upload", us) } diff --git a/bus/routes.go b/bus/routes.go index e4ba44a2d..593aa1112 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -1253,8 +1253,7 @@ func (b *Bus) packedSlabsHandlerDonePOST(jc jape.Context) { } func (b *Bus) settingsGougingHandlerGET(jc jape.Context) { - var gs api.GougingSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingGouging, &gs); errors.Is(err, api.ErrSettingNotFound) { + if gs, err := b.ss.GougingSettings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) } else if jc.Check("failed to get gouging settings", err) == nil { jc.Encode(gs) @@ -1268,24 +1267,22 @@ func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { } else if err := gs.Validate(); err != nil { jc.Error(fmt.Errorf("couldn't update gouging settings, error: %v", err), http.StatusBadRequest) return - } - - // marshal the setting - data, err := json.Marshal(gs) - if err != nil { - jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) - return - } - - // update the setting - if jc.Check("could not update gouging settings", b.updateSetting(jc.Request.Context(), api.SettingGouging, string(data), true)) != nil { - return + } else if jc.Check("could not update gouging settings", b.ss.UpdateGougingSettings(jc.Request.Context(), gs)) == nil { + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + Key: api.SettingGouging, + Update: gs, + Timestamp: time.Now().UTC(), + }, + }) + b.pinMgr.TriggerUpdate() } } func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { - var pps api.PinnedSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingPinned, &pps); errors.Is(err, api.ErrSettingNotFound) { + if pps, err := b.ss.PinnedSettings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) } else if jc.Check("failed to get price pinning settings", err) == nil { // populate the Autopilots map with the current autopilots @@ -1315,79 +1312,73 @@ func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { return } } - - // marshal the setting - data, err := json.Marshal(pps) - if err != nil { - jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) - return - } - - // update the setting - if jc.Check("could not update price pinning settings", b.updateSetting(jc.Request.Context(), api.SettingPinned, string(data), true)) != nil { - return + if jc.Check("could not update price pinning settings", b.ss.UpdatePinnedSettings(jc.Request.Context(), pps)) == nil { + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + Key: api.SettingPinned, + Update: pps, + Timestamp: time.Now().UTC(), + }, + }) + b.pinMgr.TriggerUpdate() } } -func (b *Bus) settingsRedundancyHandlerGET(jc jape.Context) { - var rs api.RedundancySettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingUploads, &rs); errors.Is(err, api.ErrSettingNotFound) { +func (b *Bus) settingsUploadsHandlerGET(jc jape.Context) { + if us, err := b.ss.UploadSettings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) - } else if jc.Check("failed to get redundancy settings", err) == nil { - jc.Encode(rs) + } else if jc.Check("failed to get upload settings", err) == nil { + jc.Encode(us) } } -func (b *Bus) settingsRedundancyHandlerPUT(jc jape.Context) { - var rs api.RedundancySettings - if jc.Decode(&rs) != nil { - return - } else if err := rs.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update redundancy settings, error: %v", err), http.StatusBadRequest) - return - } - - // marshal the setting - data, err := json.Marshal(rs) - if err != nil { - jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) +func (b *Bus) settingsUploadsHandlerPUT(jc jape.Context) { + var us api.UploadSettings + if jc.Decode(&us) != nil { return - } - - // update the setting - if jc.Check("could not update redundancy settings", b.updateSetting(jc.Request.Context(), api.SettingUploads, string(data), false)) != nil { + } else if err := us.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update upload settings, error: %v", err), http.StatusBadRequest) return + } else if jc.Check("could not update upload settings", b.ss.UpdateUploadSettings(jc.Request.Context(), us)) == nil { + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + Key: api.SettingUploads, + Update: us, + Timestamp: time.Now().UTC(), + }, + }) } } func (b *Bus) settingsS3HandlerGET(jc jape.Context) { - var s3as api.S3Settings - if err := b.fetchSetting(jc.Request.Context(), api.SettingS3, &s3as); errors.Is(err, api.ErrSettingNotFound) { + if s3s, err := b.ss.S3Settings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) - } else if jc.Check("failed to get s3 authentication settings", err) == nil { - jc.Encode(s3as) + } else if jc.Check("failed to get S3 settings", err) == nil { + jc.Encode(s3s) } } func (b *Bus) settingsS3HandlerPUT(jc jape.Context) { - var s3as api.S3Settings - if jc.Decode(&s3as) != nil { - return - } else if err := s3as.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update s3 authentication settings, error: %v", err), http.StatusBadRequest) - return - } - - // marshal the setting - data, err := json.Marshal(s3as) - if err != nil { - jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) + var s3s api.S3Settings + if jc.Decode(&s3s) != nil { return - } - - // update the setting - if jc.Check("could not update s3 authentication settings", b.updateSetting(jc.Request.Context(), api.SettingS3, string(data), false)) != nil { + } else if err := s3s.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update S3 settings, error: %v", err), http.StatusBadRequest) return + } else if jc.Check("could not update S3 settings", b.ss.UpdateS3Settings(jc.Request.Context(), s3s)) == nil { + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + Key: api.SettingS3, + Update: s3s, + Timestamp: time.Now().UTC(), + }, + }) } } @@ -1524,8 +1515,8 @@ func (b *Bus) slabsPartialHandlerPOST(jc jape.Context) { if jc.Check("failed to add partial slab", err) != nil { return } - var us api.UploadSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingUploads, &us); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + us, err := b.ss.UploadSettings(jc.Request.Context()) + if err != nil && !errors.Is(err, api.ErrSettingNotFound) { jc.Error(fmt.Errorf("could not get upload packing settings: %w", err), http.StatusInternalServerError) return } @@ -1559,8 +1550,8 @@ func (b *Bus) paramsHandlerUploadGET(jc jape.Context) { var uploadPacking bool var contractSet string - var us api.UploadSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingUploads, &us); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + us, err := b.ss.UploadSettings(jc.Request.Context()) + if err != nil && !errors.Is(err, api.ErrSettingNotFound) { jc.Error(fmt.Errorf("could not get upload settings: %w", err), http.StatusInternalServerError) return } else if err == nil { @@ -1604,18 +1595,14 @@ func (b *Bus) paramsHandlerGougingGET(jc jape.Context) { } func (b *Bus) gougingParams(ctx context.Context) (api.GougingParams, error) { - var gs api.GougingSettings - if gss, err := b.ss.Setting(ctx, api.SettingGouging); err != nil { + gs, err := b.ss.GougingSettings(ctx) + if err != nil { return api.GougingParams{}, err - } else if err := json.Unmarshal([]byte(gss), &gs); err != nil { - b.logger.Panicf("failed to unmarshal gouging settings '%s': %v", gss, err) } - var rs api.RedundancySettings - if rss, err := b.ss.Setting(ctx, api.SettingUploads); err != nil { + us, err := b.ss.UploadSettings(ctx) + if err != nil { return api.GougingParams{}, err - } else if err := json.Unmarshal([]byte(rss), &rs); err != nil { - b.logger.Panicf("failed to unmarshal redundancy settings '%s': %v", rss, err) } cs, err := b.consensusState(ctx) @@ -1626,7 +1613,7 @@ func (b *Bus) gougingParams(ctx context.Context) (api.GougingParams, error) { return api.GougingParams{ ConsensusState: cs, GougingSettings: gs, - RedundancySettings: rs, + RedundancySettings: us.Redundancy, TransactionFee: b.cm.RecommendedFee(), }, nil } diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index ff51812dc..3bc579f28 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -2,7 +2,6 @@ package bus import ( "context" - "encoding/json" "errors" "fmt" "sync" @@ -20,9 +19,13 @@ import ( type ( Store interface { Autopilot(ctx context.Context, id string) (api.Autopilot, error) - Setting(ctx context.Context, key string) (string, error) UpdateAutopilot(ctx context.Context, ap api.Autopilot) error - UpdateSetting(ctx context.Context, key, value string) error + + GougingSettings(ctx context.Context) (api.GougingSettings, error) + UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error + + PinnedSettings(ctx context.Context) (api.PinnedSettings, error) + UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error } ) @@ -106,16 +109,6 @@ func (pm *pinManager) averageRate() decimal.Decimal { return decimal.NewFromFloat(median) } -func (pm *pinManager) pinnedSettings(ctx context.Context) (api.PinnedSettings, error) { - var ps api.PinnedSettings - if pss, err := pm.s.Setting(ctx, api.SettingPinned); err != nil { - return api.PinnedSettings{}, err - } else if err := json.Unmarshal([]byte(pss), &ps); err != nil { - pm.logger.Panicf("failed to unmarshal pinned settings '%s': %v", pss, err) - } - return ps, nil -} - func (pm *pinManager) rateExceedsThreshold(threshold float64) bool { pm.mu.Lock() defer pm.mu.Unlock() @@ -241,11 +234,8 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin var updated bool // fetch gouging settings - var gs api.GougingSettings - if gss, err := pm.s.Setting(ctx, api.SettingGouging); err != nil { - return err - } else if err := json.Unmarshal([]byte(gss), &gs); err != nil { - pm.logger.Panicf("failed to unmarshal gouging settings '%s': %v", gss, err) + gs, err := pm.s.GougingSettings(ctx) + if err != nil { return err } @@ -292,15 +282,14 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin } // validate settings - err := gs.Validate() + err = gs.Validate() if err != nil { pm.logger.Warnw("failed to update gouging setting, new settings make the setting invalid", zap.Error(err)) return err } // update settings - bytes, _ := json.Marshal(gs) - err = pm.s.UpdateSetting(ctx, api.SettingGouging, string(bytes)) + err = pm.s.UpdateGougingSettings(ctx, gs) // broadcast event if err == nil { @@ -309,7 +298,7 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin Event: api.EventUpdate, Payload: api.EventSettingUpdate{ Key: api.SettingGouging, - Update: string(bytes), + Update: gs, Timestamp: time.Now().UTC(), }, }) @@ -322,7 +311,7 @@ func (pm *pinManager) updatePrices(ctx context.Context, forced bool) error { pm.logger.Debugw("updating prices", zap.Bool("forced", forced)) // fetch pinned settings - settings, err := pm.pinnedSettings(ctx) + settings, err := pm.s.PinnedSettings(ctx) if errors.Is(err, api.ErrSettingNotFound) { pm.logger.Debug("price pinning not configured, skipping price update") return nil diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index 33e4962eb..37523dd6e 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -111,22 +111,18 @@ func (api *mockForexAPI) setUnreachable(unreachable bool) { type mockPinStore struct { mu sync.Mutex - settings map[string]string + gs api.GougingSettings + ps api.PinnedSettings autopilots map[string]api.Autopilot } func newTestStore() *mockPinStore { s := &mockPinStore{ autopilots: make(map[string]api.Autopilot), - settings: make(map[string]string), + gs: api.DefaultGougingSettings, + ps: api.DefaultPricePinSettings, } - // add default price pin - and gouging settings - b, _ := json.Marshal(api.DefaultPricePinSettings) - s.settings[api.SettingPinned] = string(b) - b, _ = json.Marshal(api.DefaultGougingSettings) - s.settings[api.SettingGouging] = string(b) - // add default autopilot s.autopilots[testAutopilotID] = api.Autopilot{ ID: testAutopilotID, @@ -140,34 +136,30 @@ func newTestStore() *mockPinStore { return s } -func (ms *mockPinStore) gougingSettings() api.GougingSettings { - val, err := ms.Setting(context.Background(), api.SettingGouging) - if err != nil { - panic(err) - } - var gs api.GougingSettings - if err := json.Unmarshal([]byte(val), &gs); err != nil { - panic(err) - } - return gs +func (ms *mockPinStore) GougingSettings(ctx context.Context) (api.GougingSettings, error) { + ms.mu.Lock() + defer ms.mu.Unlock() + return ms.gs, nil } -func (ms *mockPinStore) updatPinnedSettings(pps api.PinnedSettings) { - b, _ := json.Marshal(pps) - ms.UpdateSetting(context.Background(), api.SettingPinned, string(b)) - time.Sleep(2 * testUpdateInterval) +func (ms *mockPinStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { + ms.mu.Lock() + defer ms.mu.Unlock() + ms.gs = gs + return nil } -func (ms *mockPinStore) Setting(ctx context.Context, key string) (string, error) { +func (ms *mockPinStore) PinnedSettings(ctx context.Context) (api.PinnedSettings, error) { ms.mu.Lock() defer ms.mu.Unlock() - return ms.settings[key], nil + return ms.ps, nil } -func (ms *mockPinStore) UpdateSetting(ctx context.Context, key, value string) error { +func (ms *mockPinStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { ms.mu.Lock() defer ms.mu.Unlock() - ms.settings[key] = value + ms.ps = ps + time.Sleep(2 * testUpdateInterval) return nil } @@ -221,7 +213,7 @@ func TestPinManager(t *testing.T) { pps.Currency = "usd" pps.Threshold = 0.5 pps.ForexEndpointURL = forex.s.URL - ms.updatPinnedSettings(pps) + ms.UpdatePinnedSettings(context.Background(), pps) // assert price manager is running now if cnt := len(rates()); cnt < 1 { @@ -230,30 +222,30 @@ func TestPinManager(t *testing.T) { // update exchange rate and fetch current gouging settings forex.setRate(2.5) - gs := ms.gougingSettings() + gs, _ := ms.GougingSettings(context.Background()) // configure all pins but disable them for now pps.GougingSettingsPins.MaxDownload = api.Pin{Value: 3, Pinned: false} pps.GougingSettingsPins.MaxStorage = api.Pin{Value: 3, Pinned: false} pps.GougingSettingsPins.MaxUpload = api.Pin{Value: 3, Pinned: false} - ms.updatPinnedSettings(pps) + ms.UpdatePinnedSettings(context.Background(), pps) // assert gouging settings are unchanged - if gss := ms.gougingSettings(); !reflect.DeepEqual(gs, gss) { + if gss, _ := ms.GougingSettings(context.Background()); !reflect.DeepEqual(gs, gss) { t.Fatalf("expected gouging settings to be the same, got %v", gss) } // enable the max download pin, with the threshold at 0.5 it should remain unchanged pps.GougingSettingsPins.MaxDownload.Pinned = true - ms.updatPinnedSettings(pps) - if gss := ms.gougingSettings(); !reflect.DeepEqual(gs, gss) { + ms.UpdatePinnedSettings(context.Background(), pps) + if gss, _ := ms.GougingSettings(context.Background()); !reflect.DeepEqual(gs, gss) { t.Fatalf("expected gouging settings to be the same, got %v", gss) } // lower the threshold, gouging settings should be updated pps.Threshold = 0.05 - ms.updatPinnedSettings(pps) - if gss := ms.gougingSettings(); gss.MaxContractPrice.Equals(gs.MaxDownloadPrice) { + ms.UpdatePinnedSettings(context.Background(), pps) + if gss, _ := ms.GougingSettings(context.Background()); gss.MaxContractPrice.Equals(gs.MaxDownloadPrice) { t.Fatalf("expected gouging settings to be updated, got %v = %v", gss.MaxDownloadPrice, gs.MaxDownloadPrice) } @@ -261,10 +253,10 @@ func TestPinManager(t *testing.T) { pps.GougingSettingsPins.MaxDownload.Pinned = true pps.GougingSettingsPins.MaxStorage.Pinned = true pps.GougingSettingsPins.MaxUpload.Pinned = true - ms.updatPinnedSettings(pps) + ms.UpdatePinnedSettings(context.Background(), pps) // assert they're all updated - if gss := ms.gougingSettings(); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) || + if gss, _ := ms.GougingSettings(context.Background()); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) || gss.MaxStoragePrice.Equals(gs.MaxStoragePrice) || gss.MaxUploadPrice.Equals(gs.MaxUploadPrice) { t.Fatalf("expected gouging settings to be updated, got %v = %v", gss, gs) @@ -284,7 +276,7 @@ func TestPinManager(t *testing.T) { }, } pps.Autopilots = map[string]api.AutopilotPins{testAutopilotID: pins} - ms.updatPinnedSettings(pps) + ms.UpdatePinnedSettings(context.Background(), pps) // assert autopilot was not updated if app, _ := ms.Autopilot(context.Background(), testAutopilotID); !app.Config.Contracts.Allowance.Equals(ap.Config.Contracts.Allowance) { @@ -294,7 +286,7 @@ func TestPinManager(t *testing.T) { // enable the pin pins.Allowance.Pinned = true pps.Autopilots[testAutopilotID] = pins - ms.updatPinnedSettings(pps) + ms.UpdatePinnedSettings(context.Background(), pps) // assert autopilot was updated if app, _ := ms.Autopilot(context.Background(), testAutopilotID); app.Config.Contracts.Allowance.Equals(ap.Config.Contracts.Allowance) { @@ -305,7 +297,7 @@ func TestPinManager(t *testing.T) { forex.setUnreachable(true) // assert alert was registered - ms.updatPinnedSettings(pps) + ms.UpdatePinnedSettings(context.Background(), pps) res, _ := a.Alerts(context.Background(), alerts.AlertsOpts{}) if len(res.Alerts) == 0 { t.Fatalf("expected 1 alert, got %d", len(a.alerts)) @@ -315,7 +307,7 @@ func TestPinManager(t *testing.T) { forex.setUnreachable(false) // assert alert was dismissed - ms.updatPinnedSettings(pps) + ms.UpdatePinnedSettings(context.Background(), pps) res, _ = a.Alerts(context.Background(), alerts.AlertsOpts{}) if len(res.Alerts) != 0 { t.Fatalf("expected 0 alerts, got %d", len(a.alerts)) diff --git a/stores/settingsdb.go b/stores/settingsdb.go index ea31b25bd..407a53efd 100644 --- a/stores/settingsdb.go +++ b/stores/settingsdb.go @@ -2,14 +2,118 @@ package stores import ( "context" + "encoding/json" "fmt" + "go.sia.tech/renterd/api" sql "go.sia.tech/renterd/stores/sql" ) -// Setting implements the bus.SettingStore interface. -func (s *SQLStore) Setting(ctx context.Context, key string) (string, error) { - // Check cache first. +func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, _ error) { + value, err := s.fetchSetting(ctx, api.SettingGouging) + if err != nil { + return api.GougingSettings{}, err + } + + if err := json.Unmarshal([]byte(value), &gs); err != nil { + s.logger.Panicf("failed to unmarshal gouging settings '%s': %v", value, err) + return api.GougingSettings{}, err + } + return +} + +func (s *SQLStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { + data, err := json.Marshal(gs) + if err != nil { + return fmt.Errorf("couldn't marshal the given value, error: %v", err) + } + return s.updateSetting(ctx, api.SettingGouging, string(data)) +} + +func (s *SQLStore) PinnedSettings(ctx context.Context) (pps api.PinnedSettings, _ error) { + value, err := s.fetchSetting(ctx, api.SettingPinned) + if err != nil { + return api.PinnedSettings{}, err + } + + if err := json.Unmarshal([]byte(value), &pps); err != nil { + s.logger.Panicf("failed to unmarshal pinned settings '%s': %v", value, err) + return api.PinnedSettings{}, err + } + return +} + +func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error { + data, err := json.Marshal(pps) + if err != nil { + return fmt.Errorf("couldn't marshal the given value, error: %v", err) + } + return s.updateSetting(ctx, api.SettingPinned, string(data)) +} + +func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, _ error) { + value, err := s.fetchSetting(ctx, api.SettingUploads) + if err != nil { + return api.UploadSettings{}, err + } + + if err := json.Unmarshal([]byte(value), &us); err != nil { + s.logger.Panicf("failed to unmarshal upload settings '%s': %v", value, err) + return api.UploadSettings{}, err + } + return +} + +func (s *SQLStore) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error { + data, err := json.Marshal(us) + if err != nil { + return fmt.Errorf("couldn't marshal the given value, error: %v", err) + } + return s.updateSetting(ctx, api.SettingUploads, string(data)) +} + +func (s *SQLStore) S3Settings(ctx context.Context) (ss api.S3Settings, _ error) { + value, err := s.fetchSetting(ctx, api.SettingS3) + if err != nil { + return api.S3Settings{}, err + } + + if err := json.Unmarshal([]byte(value), &ss); err != nil { + s.logger.Panicf("failed to unmarshal s3 settings '%s': %v", value, err) + return api.S3Settings{}, err + } + return +} + +func (s *SQLStore) UpdateS3Settings(ctx context.Context, ss api.S3Settings) error { + data, err := json.Marshal(ss) + if err != nil { + return fmt.Errorf("couldn't marshal the given value, error: %v", err) + } + return s.updateSetting(ctx, api.SettingS3, string(data)) +} + +func (s *SQLStore) DeleteSetting(ctx context.Context, key string) (err error) { + return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + return tx.DeleteSetting(ctx, key) + }) +} + +func (s *SQLStore) Setting(ctx context.Context, key string, out interface{}) (err error) { + var value string + err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + value, err = tx.Setting(ctx, key) + return err + }) + if err != nil { + return fmt.Errorf("failed to fetch setting from db: %w", err) + } + + return json.Unmarshal([]byte(value), &out) +} + +func (s *SQLStore) fetchSetting(ctx context.Context, key string) (string, error) { + // check cache first s.settingsMu.Lock() defer s.settingsMu.Unlock() value, ok := s.settings[key] @@ -17,7 +121,7 @@ func (s *SQLStore) Setting(ctx context.Context, key string) (string, error) { return value, nil } - // Check database. + // check database var err error err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { value, err = tx.Setting(ctx, key) @@ -30,8 +134,7 @@ func (s *SQLStore) Setting(ctx context.Context, key string) (string, error) { return value, nil } -// UpdateSetting implements the bus.SettingStore interface. -func (s *SQLStore) UpdateSetting(ctx context.Context, key, value string) error { +func (s *SQLStore) updateSetting(ctx context.Context, key, value string) error { // update db first s.settingsMu.Lock() defer s.settingsMu.Unlock() diff --git a/stores/settingsdb_test.go b/stores/settingsdb_test.go deleted file mode 100644 index 9eda8f546..000000000 --- a/stores/settingsdb_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package stores - -import ( - "context" - "testing" -) - -// TestSQLSettingStore tests the bus.SettingStore methods on the SQLSettingStore. -func TestSQLSettingStore(t *testing.T) { - ss := newTestSQLStore(t, defaultTestSQLStoreConfig) - defer ss.Close() - - // add a setting - if err := ss.UpdateSetting(context.Background(), "foo", "bar"); err != nil { - t.Fatal(err) - } - - // assert we can query the setting by key - if value, err := ss.Setting(context.Background(), "foo"); err != nil { - t.Fatal(err) - } else if value != "bar" { - t.Fatalf("unexpected value, %s != 'bar'", value) - } - - // assert we can update the setting - if err := ss.UpdateSetting(context.Background(), "foo", "barbaz"); err != nil { - t.Fatal(err) - } else if value, err := ss.Setting(context.Background(), "foo"); err != nil { - t.Fatal(err) - } else if value != "barbaz" { - t.Fatalf("unexpected value, %s != 'barbaz'", value) - } -} diff --git a/stores/sql/database.go b/stores/sql/database.go index e4a8ab967..b1f45ef70 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -150,6 +150,9 @@ type ( // prefix and returns 'true' if any object was deleted. DeleteObjects(ctx context.Context, bucket, prefix string, limit int64) (bool, error) + // DeleteSetting deletes the setting with the given key. + DeleteSetting(ctx context.Context, key string) error + // DeleteWebhook deletes the webhook with the matching module, event and // URL of the provided webhook. If the webhook doesn't exist, // webhooks.ErrWebhookNotFound is returned. diff --git a/stores/sql/main.go b/stores/sql/main.go index e0d158552..02f876214 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -548,6 +548,13 @@ func DeleteMetadata(ctx context.Context, tx sql.Tx, objID int64) error { return err } +func DeleteSetting(ctx context.Context, tx sql.Tx, key string) error { + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", key); err != nil { + return fmt.Errorf("failed to delete setting '%s': %w", key, err) + } + return nil +} + func DeleteWebhook(ctx context.Context, tx sql.Tx, wh webhooks.Webhook) error { res, err := tx.Exec(ctx, "DELETE FROM webhooks WHERE module = ? AND event = ? AND url = ?", wh.Module, wh.Event, wh.URL) if err != nil { diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index 5be1a9c3f..0c5b0dce0 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -340,6 +340,10 @@ func (tx *MainDatabaseTx) InsertMultipartUpload(ctx context.Context, bucket, key return ssql.InsertMultipartUpload(ctx, tx, bucket, key, ec, mimeType, metadata) } +func (tx *MainDatabaseTx) DeleteSetting(ctx context.Context, key string) error { + return ssql.DeleteSetting(ctx, tx, key) +} + func (tx *MainDatabaseTx) DeleteWebhook(ctx context.Context, wh webhooks.Webhook) error { return ssql.DeleteWebhook(ctx, tx, wh) } diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 50ea2619f..fa253a004 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -332,6 +332,10 @@ func (tx *MainDatabaseTx) DeleteHostSector(ctx context.Context, hk types.PublicK return ssql.DeleteHostSector(ctx, tx, hk, root) } +func (tx *MainDatabaseTx) DeleteSetting(ctx context.Context, key string) error { + return ssql.DeleteSetting(ctx, tx, key) +} + func (tx *MainDatabaseTx) DeleteWebhook(ctx context.Context, wh webhooks.Webhook) error { return ssql.DeleteWebhook(ctx, tx, wh) } From 0e43e8d03bf57b94401c790d8c352c9997b82d65 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 Aug 2024 09:42:44 +0200 Subject: [PATCH 04/33] all: update docs and defaults --- .github/ISSUE_TEMPLATE/bug_report.yml | 6 ++-- README.md | 23 -------------- api/setting.go | 45 ++++++++++----------------- bus/bus.go | 2 +- internal/bus/pinmanager_test.go | 4 +-- internal/test/config.go | 2 +- internal/test/e2e/cluster.go | 2 +- 7 files changed, 25 insertions(+), 59 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 5219b07cf..a5ccf759f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -66,10 +66,10 @@ body: description: | The configuration of your bus ```bash - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/contractset curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/gouging - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/redundancy - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/uploadpacking + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/pinned + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/s3 + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/uploads ``` placeholder: Paste the output of the above commands here validations: diff --git a/README.md b/README.md index 7508620c5..698a7c4b5 100644 --- a/README.md +++ b/README.md @@ -558,29 +558,6 @@ formed. } ``` -### Gouging - -The default gouging settings are listed below. The gouging settings can be -updated using the settings API: - -- `GET /api/bus/settings/gouging` -- `PUT /api/bus/settings/gouging` - -```json -{ - "hostBlockHeightLeeway": 6, // 6 blocks - "maxContractPrice": "15000000000000000000000000", // 15 SC per contract - "maxDownloadPrice": "3000000000000000000000000000", // 3000 SC per 1 TB - "maxRPCPrice": "1000000000000000000000", // 1mS per RPC - "maxStoragePrice": "631593542824", // 3000 SC per TB per month - "maxUploadPrice": "3000000000000000000000000000", // 3000 SC per 1 TB - "migrationSurchargeMultiplier": 10, // overpay up to 10x for sectors migrations on critical slabs - "minAccountExpiry": 86400000000000, // 1 day - "minMaxEphemeralAccountBalance": "1000000000000000000000000", // 1 SC - "minPriceTableValidity": 300000000000 // 5 minutes -} -``` - ### Blocklist Unfortunately the Sia blockchain is subject to hosts that announced themselves diff --git a/api/setting.go b/api/setting.go index f3e0f2b63..4627a31a7 100644 --- a/api/setting.go +++ b/api/setting.go @@ -32,9 +32,7 @@ var ( ErrSettingNotFound = errors.New("setting not found") // DefaultGougingSettings define the default gouging settings the bus is - // configured with on startup. These values can be adjusted using the - // settings API. - // + // configured with on startup. DefaultGougingSettings = GougingSettings{ MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC MaxContractPrice: types.Siacoins(15), // 15 SC per contract @@ -48,40 +46,31 @@ var ( MigrationSurchargeMultiplier: 10, // 10x } - // DefaultPricePinSettings define the default price pin settings the bus is - // configured with on startup. These values can be adjusted using the - // settings API. - DefaultPricePinSettings = PinnedSettings{ + // DefaultPinnedSettings define the default pin settings the bus is + // configured with on startup. + DefaultPinnedSettings = PinnedSettings{ Enabled: false, Currency: "usd", ForexEndpointURL: "https://api.siascan.com/exchange-rate/siacoin", Threshold: 0.05, } + // DefaultUploadSettings define the default upload settings the bus is + // configured with on startup. DefaultUploadSettings = UploadSettings{ - Packing: DefaultUploadPackingSettings, - Redundancy: DefaultRedundancySettings, + Packing: UploadPackingSettings{ + Enabled: true, + SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB + }, + Redundancy: RedundancySettings{ + MinShards: 10, + TotalShards: 30, + }, } - // DefaultUploadPackingSettings define the default upload packing settings - // the bus is configured with on startup. - DefaultUploadPackingSettings = UploadPackingSettings{ - Enabled: true, - SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB - } - - // DefaultRedundancySettings define the default redundancy settings the bus - // is configured with on startup. These values can be adjusted using the - // settings API. - // - // NOTE: default redundancy settings for testnet are different from mainnet. - DefaultRedundancySettings = RedundancySettings{ - MinShards: 10, - TotalShards: 30, - } - - // Same as DefaultRedundancySettings but for running on testnet networks due - // to their reduced number of hosts. + // DefaultRedundancySettingsTestnet defines redundancy settings for the + // testnet, these are lower due to the reduced number of hosts on the + // testnet. DefaultRedundancySettingsTestnet = RedundancySettings{ MinShards: 2, TotalShards: 6, diff --git a/bus/bus.go b/bus/bus.go index e1e5563f9..47de9c44f 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -638,7 +638,7 @@ func (b *Bus) compatV2Settings(ctx context.Context) error { if err := b.ss.Setting(ctx, "pricepinning", &pps); err != nil && !errors.Is(err, api.ErrSettingNotFound) { return err } else if errors.Is(err, api.ErrSettingNotFound) { - if err := b.ss.UpdatePinnedSettings(ctx, api.DefaultPricePinSettings); err != nil { + if err := b.ss.UpdatePinnedSettings(ctx, api.DefaultPinnedSettings); err != nil { return err } } else { diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index 37523dd6e..a90d76e38 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -120,7 +120,7 @@ func newTestStore() *mockPinStore { s := &mockPinStore{ autopilots: make(map[string]api.Autopilot), gs: api.DefaultGougingSettings, - ps: api.DefaultPricePinSettings, + ps: api.DefaultPinnedSettings, } // add default autopilot @@ -208,7 +208,7 @@ func TestPinManager(t *testing.T) { } // enable price pinning - pps := api.DefaultPricePinSettings + pps := api.DefaultPinnedSettings pps.Enabled = true pps.Currency = "usd" pps.Threshold = 0.5 diff --git a/internal/test/config.go b/internal/test/config.go index f55c8a9e0..b33c2b0d6 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -49,7 +49,7 @@ var ( MinMaxEphemeralAccountBalance: types.Siacoins(1), // 1SC } - PricePinSettings = api.DefaultPricePinSettings + PricePinSettings = api.DefaultPinnedSettings RedundancySettings = api.RedundancySettings{ MinShards: 2, diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index ab057e445..e79a1e605 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -446,7 +446,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { us := test.UploadSettings us.Packing = api.UploadPackingSettings{ Enabled: enableUploadPacking, - SlabBufferMaxSizeSoft: api.DefaultUploadPackingSettings.SlabBufferMaxSizeSoft, + SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB, } // Build S3 settings. From 8c76a212257f907fbd6507f90924f514dfc6c768 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 Aug 2024 10:14:25 +0200 Subject: [PATCH 05/33] all: cleanup PR --- api/setting.go | 2 +- bus/bus.go | 8 ++++---- bus/routes.go | 9 ++++++--- internal/worker/cache.go | 14 +++++++------- stores/settingsdb.go | 4 ++-- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/api/setting.go b/api/setting.go index 4627a31a7..dd2cb78af 100644 --- a/api/setting.go +++ b/api/setting.go @@ -13,7 +13,7 @@ const ( SettingGouging = "gouging" SettingPinned = "pinned" SettingS3 = "s3" - SettingUploads = "uploads" + SettingUpload = "upload" ) const ( diff --git a/bus/bus.go b/bus/bus.go index 47de9c44f..ff5320f70 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -474,8 +474,8 @@ func (b *Bus) Handler() http.Handler { "PUT /settings/pinned": b.settingsPinnedHandlerPUT, "GET /settings/s3": b.settingsS3HandlerGET, "PUT /settings/s3": b.settingsS3HandlerPUT, - "GET /settings/uploads": b.settingsUploadsHandlerGET, - "PUT /settings/uploads": b.settingsUploadsHandlerPUT, + "GET /settings/upload": b.settingsUploadHandlerGET, + "PUT /settings/upload": b.settingsUploadHandlerPUT, "POST /slabs/migration": b.slabsMigrationHandlerPOST, "GET /slabs/partial/:key": b.slabsPartialHandlerGET, @@ -617,8 +617,8 @@ func (b *Bus) compatV2Settings(ctx context.Context) error { b.ss.Setting(ctx, api.SettingGouging, struct{}{}), b.ss.Setting(ctx, api.SettingPinned, struct{}{}), b.ss.Setting(ctx, api.SettingS3, struct{}{}), - b.ss.Setting(ctx, api.SettingUploads, struct{}{}), - ), api.ErrAutopilotNotFound) { + b.ss.Setting(ctx, api.SettingUpload, struct{}{}), + ), api.ErrSettingNotFound) { return nil } diff --git a/bus/routes.go b/bus/routes.go index 593aa1112..e681b1b93 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -1290,6 +1290,9 @@ func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { if jc.Check("failed to fetch autopilots", err) != nil { return } + if pps.Autopilots == nil { + pps.Autopilots = make(map[string]api.AutopilotPins) + } for _, ap := range aps { if _, exists := pps.Autopilots[ap.ID]; !exists { pps.Autopilots[ap.ID] = api.AutopilotPins{} @@ -1326,7 +1329,7 @@ func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { } } -func (b *Bus) settingsUploadsHandlerGET(jc jape.Context) { +func (b *Bus) settingsUploadHandlerGET(jc jape.Context) { if us, err := b.ss.UploadSettings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) } else if jc.Check("failed to get upload settings", err) == nil { @@ -1334,7 +1337,7 @@ func (b *Bus) settingsUploadsHandlerGET(jc jape.Context) { } } -func (b *Bus) settingsUploadsHandlerPUT(jc jape.Context) { +func (b *Bus) settingsUploadHandlerPUT(jc jape.Context) { var us api.UploadSettings if jc.Decode(&us) != nil { return @@ -1346,7 +1349,7 @@ func (b *Bus) settingsUploadsHandlerPUT(jc jape.Context) { Module: api.ModuleSetting, Event: api.EventUpdate, Payload: api.EventSettingUpdate{ - Key: api.SettingUploads, + Key: api.SettingUpload, Update: us, Timestamp: time.Now().UTC(), }, diff --git a/internal/worker/cache.go b/internal/worker/cache.go index f6c1ea574..ebb071acd 100644 --- a/internal/worker/cache.go +++ b/internal/worker/cache.go @@ -333,15 +333,15 @@ func (c *cache) handleSettingUpdate(e api.EventSettingUpdate) (err error) { gp.GougingSettings = gs c.cache.Set(cacheKeyGougingParams, gp) - case api.SettingUploads: - var rs api.RedundancySettings - if err := json.Unmarshal(data, &rs); err != nil { - return fmt.Errorf("couldn't update redundancy settings, invalid request body, %t", e.Update) - } else if err := rs.Validate(); err != nil { - return fmt.Errorf("couldn't update redundancy settings, error: %v", err) + case api.SettingUpload: + var us api.UploadSettings + if err := json.Unmarshal(data, &us); err != nil { + return fmt.Errorf("couldn't update upload settings, invalid request body, %t", e.Update) + } else if err := us.Validate(); err != nil { + return fmt.Errorf("couldn't update upload settings, error: %v", err) } - gp.RedundancySettings = rs + gp.RedundancySettings = us.Redundancy c.cache.Set(cacheKeyGougingParams, gp) default: } diff --git a/stores/settingsdb.go b/stores/settingsdb.go index 407a53efd..658b6de1f 100644 --- a/stores/settingsdb.go +++ b/stores/settingsdb.go @@ -52,7 +52,7 @@ func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, pps api.PinnedSetti } func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, _ error) { - value, err := s.fetchSetting(ctx, api.SettingUploads) + value, err := s.fetchSetting(ctx, api.SettingUpload) if err != nil { return api.UploadSettings{}, err } @@ -69,7 +69,7 @@ func (s *SQLStore) UpdateUploadSettings(ctx context.Context, us api.UploadSettin if err != nil { return fmt.Errorf("couldn't marshal the given value, error: %v", err) } - return s.updateSetting(ctx, api.SettingUploads, string(data)) + return s.updateSetting(ctx, api.SettingUpload, string(data)) } func (s *SQLStore) S3Settings(ctx context.Context) (ss api.S3Settings, _ error) { From c45aa0beb485808e3d083fdb67465cd07675c12b Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 Aug 2024 10:27:35 +0200 Subject: [PATCH 06/33] all: cleanup PR --- api/setting.go | 46 ++++++++++++++++++++++----------------------- bus/bus.go | 29 +++++++++++++++++++++++++--- cmd/renterd/node.go | 32 ------------------------------- 3 files changed, 48 insertions(+), 59 deletions(-) diff --git a/api/setting.go b/api/setting.go index dd2cb78af..237a96f5f 100644 --- a/api/setting.go +++ b/api/setting.go @@ -30,7 +30,9 @@ var ( // ErrSettingNotFound is returned if a requested setting is not present in the // database. ErrSettingNotFound = errors.New("setting not found") +) +var ( // DefaultGougingSettings define the default gouging settings the bus is // configured with on startup. DefaultGougingSettings = GougingSettings{ @@ -55,6 +57,22 @@ var ( Threshold: 0.05, } + // DefaultRedundancySettingsTestnet defines redundancy settings for the + // testnet, these are lower due to the reduced number of hosts on the + // testnet. + DefaultRedundancySettingsTestnet = RedundancySettings{ + MinShards: 2, + TotalShards: 6, + } + + // DefaultS3Settings defines the 3 settings the bus is configured with on + // startup. + DefaultS3Settings = S3Settings{ + Authentication: S3AuthenticationSettings{ + V4Keypairs: map[string]string{}, + }, + } + // DefaultUploadSettings define the default upload settings the bus is // configured with on startup. DefaultUploadSettings = UploadSettings{ @@ -67,14 +85,6 @@ var ( TotalShards: 30, }, } - - // DefaultRedundancySettingsTestnet defines redundancy settings for the - // testnet, these are lower due to the reduced number of hosts on the - // testnet. - DefaultRedundancySettingsTestnet = RedundancySettings{ - MinShards: 2, - TotalShards: 6, - } ) type ( @@ -235,20 +245,12 @@ func (gs GougingSettings) Validate() error { return nil } +// Validate returns an error if the upload settings are not considered valid. func (us UploadSettings) Validate() error { - return errors.Join( - us.Packing.Validate(), - us.Redundancy.Validate(), - ) -} - -// Validate returns an error if the upload packing settings are not considered -// valid. -func (up UploadPackingSettings) Validate() error { - if up.Enabled && up.SlabBufferMaxSizeSoft <= 0 { + if us.Packing.Enabled && us.Packing.SlabBufferMaxSizeSoft <= 0 { return errors.New("SlabBufferMaxSizeSoft must be greater than zero when upload packing is enabled") } - return nil + return us.Redundancy.Validate() } // Redundancy returns the effective storage redundancy of the @@ -285,11 +287,7 @@ func (rs RedundancySettings) Validate() error { // Validate returns an error if the authentication settings are not considered // valid. func (s3s S3Settings) Validate() error { - return s3s.Authentication.Validate() -} - -func (s3a S3AuthenticationSettings) Validate() error { - for accessKeyID, secretAccessKey := range s3a.V4Keypairs { + for accessKeyID, secretAccessKey := range s3s.Authentication.V4Keypairs { if accessKeyID == "" { return fmt.Errorf("AccessKeyID cannot be empty") } else if len(accessKeyID) < S3MinAccessKeyLen || len(accessKeyID) > S3MaxAccessKeyLen { diff --git a/bus/bus.go b/bus/bus.go index ff5320f70..dc6dff267 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -622,13 +622,28 @@ func (b *Bus) compatV2Settings(ctx context.Context) error { return nil } + // migrate gouging settings + if _, err := b.ss.GougingSettings(ctx); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + return err + } else if errors.Is(err, api.ErrSettingNotFound) { + if err := b.ss.UpdateGougingSettings(ctx, api.DefaultGougingSettings); err != nil { + return err + } + } + // migrate S3 settings var s3as api.S3AuthenticationSettings if err := b.ss.Setting(ctx, "s3authentication", &s3as); err != nil && !errors.Is(err, api.ErrSettingNotFound) { return err + } else if errors.Is(err, api.ErrSettingNotFound) { + if err := b.ss.UpdateS3Settings(ctx, api.DefaultS3Settings); err != nil { + return err + } } else if err == nil { s3s := api.S3Settings{Authentication: s3as} - if err := b.ss.UpdateS3Settings(ctx, s3s); err != nil { + if err := s3s.Validate(); err != nil { + return fmt.Errorf("failed to migrate S3 setting: %w", err) + } else if err := b.ss.UpdateS3Settings(ctx, s3s); err != nil { return err } } @@ -642,7 +657,9 @@ func (b *Bus) compatV2Settings(ctx context.Context) error { return err } } else { - if err := b.ss.UpdatePinnedSettings(ctx, pps); err != nil { + if err := pps.Validate(); err != nil { + return fmt.Errorf("failed to migrate pinned setting: %w", err) + } else if err := b.ss.UpdatePinnedSettings(ctx, pps); err != nil { return err } } @@ -681,5 +698,11 @@ func (b *Bus) compatV2Settings(ctx context.Context) error { us.Packing = ups } - return b.ss.UpdateUploadSettings(ctx, us) + if err := us.Validate(); err != nil { + return fmt.Errorf("failed to migrate upload setting: %w", err) + } else if err := b.ss.UpdateUploadSettings(ctx, us); err != nil { + return err + } + + return nil } diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index e87116c93..5510788bb 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "runtime" - "strings" "time" "go.sia.tech/core/consensus" @@ -21,7 +20,6 @@ import ( "go.sia.tech/coreutils/wallet" "go.sia.tech/jape" "go.sia.tech/renterd/alerts" - "go.sia.tech/renterd/api" "go.sia.tech/renterd/autopilot" "go.sia.tech/renterd/build" "go.sia.tech/renterd/bus" @@ -411,36 +409,6 @@ func (n *node) Run() error { } } - // set initial S3 keys - if n.cfg.S3.Enabled && !n.cfg.S3.DisableAuth { - s3s, err := n.bus.S3Settings(context.Background()) - if err != nil && !strings.Contains(err.Error(), api.ErrSettingNotFound.Error()) { - return fmt.Errorf("failed to fetch S3 settings: %w", err) - } else if s3s.Authentication.V4Keypairs == nil { - s3s.Authentication.V4Keypairs = make(map[string]string) - } - - // S3 key pair validation was broken at one point, we need to remove the - // invalid key pairs here to ensure we don't fail when we update the - // setting below. - for k, v := range s3s.Authentication.V4Keypairs { - if err := (api.S3AuthenticationSettings{V4Keypairs: map[string]string{k: v}}).Validate(); err != nil { - n.logger.Infof("removing invalid S3 keypair for AccessKeyID %s, reason: %v", k, err) - delete(s3s.Authentication.V4Keypairs, k) - } - } - - // merge keys - for k, v := range n.cfg.S3.KeypairsV4 { - s3s.Authentication.V4Keypairs[k] = v - } - - // update settings - if err := n.bus.UpdateS3Settings(context.Background(), s3s); err != nil { - return fmt.Errorf("failed to update S3 settings: %w", err) - } - } - // start S3 server if n.s3Srv != nil { go n.s3Srv.Serve(n.s3Listener) From 54b3d34d377a588661392087c5c1f7042765d508 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 Aug 2024 10:49:30 +0200 Subject: [PATCH 07/33] api: get rid of setting constants --- api/events.go | 20 +++++++------- api/setting.go | 7 ----- bus/bus.go | 26 +++++++++++------- bus/client/settings.go | 8 +++--- bus/routes.go | 38 ++++++++++++-------------- internal/bus/pinmanager.go | 7 +++-- internal/test/e2e/events_test.go | 8 +----- internal/worker/cache.go | 46 ++++++++------------------------ stores/settingsdb.go | 31 ++++++++++++--------- 9 files changed, 82 insertions(+), 109 deletions(-) diff --git a/api/events.go b/api/events.go index 38d490506..bd6dcd776 100644 --- a/api/events.go +++ b/api/events.go @@ -50,22 +50,24 @@ type ( Timestamp time.Time `json:"timestamp"` } - EventHostUpdate struct { - HostKey types.PublicKey `json:"hostKey"` - NetAddr string `json:"netAddr"` - Timestamp time.Time `json:"timestamp"` - } - EventContractSetUpdate struct { Name string `json:"name"` ContractIDs []types.FileContractID `json:"contractIDs"` Timestamp time.Time `json:"timestamp"` } + EventHostUpdate struct { + HostKey types.PublicKey `json:"hostKey"` + NetAddr string `json:"netAddr"` + Timestamp time.Time `json:"timestamp"` + } + EventSettingUpdate struct { - Key string `json:"key"` - Update interface{} `json:"update"` - Timestamp time.Time `json:"timestamp"` + GougingSettings *GougingSettings `json:"gougingSettings,omitempty"` + PinnedSettings *PinnedSettings `json:"pinnedSettings,omitempty"` + S3Settings *S3Settings `json:"s3Settings,omitempty"` + UploadSettings *UploadSettings `json:"uploadSettings,omitempty"` + Timestamp time.Time `json:"timestamp"` } ) diff --git a/api/setting.go b/api/setting.go index 237a96f5f..e4741e5ff 100644 --- a/api/setting.go +++ b/api/setting.go @@ -9,13 +9,6 @@ import ( "go.sia.tech/core/types" ) -const ( - SettingGouging = "gouging" - SettingPinned = "pinned" - SettingS3 = "s3" - SettingUpload = "upload" -) - const ( S3MinAccessKeyLen = 16 S3MaxAccessKeyLen = 128 diff --git a/bus/bus.go b/bus/bus.go index dc6dff267..b7513177e 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -26,6 +26,7 @@ import ( "go.sia.tech/renterd/internal/rhp" rhp2 "go.sia.tech/renterd/internal/rhp/v2" "go.sia.tech/renterd/object" + "go.sia.tech/renterd/stores" "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" @@ -281,7 +282,7 @@ type ( UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error PinnedSettings(ctx context.Context) (api.PinnedSettings, error) - UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error + UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error UploadSettings(ctx context.Context) (api.UploadSettings, error) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error @@ -614,10 +615,10 @@ func (b *Bus) deriveSubKey(purpose string) types.PrivateKey { func (b *Bus) compatV2Settings(ctx context.Context) error { // escape early if all settings are present if !errors.Is(errors.Join( - b.ss.Setting(ctx, api.SettingGouging, struct{}{}), - b.ss.Setting(ctx, api.SettingPinned, struct{}{}), - b.ss.Setting(ctx, api.SettingS3, struct{}{}), - b.ss.Setting(ctx, api.SettingUpload, struct{}{}), + b.ss.Setting(ctx, stores.SettingGouging, nil), + b.ss.Setting(ctx, stores.SettingPinned, nil), + b.ss.Setting(ctx, stores.SettingS3, nil), + b.ss.Setting(ctx, stores.SettingUpload, nil), ), api.ErrSettingNotFound) { return nil } @@ -649,17 +650,17 @@ func (b *Bus) compatV2Settings(ctx context.Context) error { } // migrate pinned settings - var pps api.PinnedSettings - if err := b.ss.Setting(ctx, "pricepinning", &pps); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + var ps api.PinnedSettings + if err := b.ss.Setting(ctx, "pricepinning", &ps); err != nil && !errors.Is(err, api.ErrSettingNotFound) { return err } else if errors.Is(err, api.ErrSettingNotFound) { if err := b.ss.UpdatePinnedSettings(ctx, api.DefaultPinnedSettings); err != nil { return err } } else { - if err := pps.Validate(); err != nil { + if err := ps.Validate(); err != nil { return fmt.Errorf("failed to migrate pinned setting: %w", err) - } else if err := b.ss.UpdatePinnedSettings(ctx, pps); err != nil { + } else if err := b.ss.UpdatePinnedSettings(ctx, ps); err != nil { return err } } @@ -704,5 +705,10 @@ func (b *Bus) compatV2Settings(ctx context.Context) error { return err } - return nil + // delete old settings + return errors.Join( + b.ss.DeleteSetting(ctx, "contractset"), + b.ss.DeleteSetting(ctx, "pricepinning"), + b.ss.DeleteSetting(ctx, "uploadpacking"), + ) } diff --git a/bus/client/settings.go b/bus/client/settings.go index 5723c3cdd..46d3e8708 100644 --- a/bus/client/settings.go +++ b/bus/client/settings.go @@ -18,14 +18,14 @@ func (c *Client) UpdateGougingSettings(ctx context.Context, gs api.GougingSettin } // PricePinningSettings returns the contract set settings. -func (c *Client) PricePinningSettings(ctx context.Context) (pps api.PinnedSettings, err error) { - err = c.c.WithContext(ctx).GET("/settings/pinned", &pps) +func (c *Client) PricePinningSettings(ctx context.Context) (ps api.PinnedSettings, err error) { + err = c.c.WithContext(ctx).GET("/settings/pinned", &ps) return } // UpdatePinnedSettings updates the given setting. -func (c *Client) UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error { - return c.c.WithContext(ctx).PUT("/settings/pinned", pps) +func (c *Client) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { + return c.c.WithContext(ctx).PUT("/settings/pinned", ps) } // S3Settings returns the S3 settings. diff --git a/bus/routes.go b/bus/routes.go index e681b1b93..284609981 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -1272,9 +1272,8 @@ func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { Module: api.ModuleSetting, Event: api.EventUpdate, Payload: api.EventSettingUpdate{ - Key: api.SettingGouging, - Update: gs, - Timestamp: time.Now().UTC(), + GougingSettings: &gs, + Timestamp: time.Now().UTC(), }, }) b.pinMgr.TriggerUpdate() @@ -1284,7 +1283,7 @@ func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { if pps, err := b.ss.PinnedSettings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { jc.Error(err, http.StatusNotFound) - } else if jc.Check("failed to get price pinning settings", err) == nil { + } else if jc.Check("failed to get pinned settings", err) == nil { // populate the Autopilots map with the current autopilots aps, err := b.as.Autopilots(jc.Request.Context()) if jc.Check("failed to fetch autopilots", err) != nil { @@ -1303,26 +1302,25 @@ func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { } func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { - var pps api.PinnedSettings - if jc.Decode(&pps) != nil { + var ps api.PinnedSettings + if jc.Decode(&ps) != nil { return - } else if err := pps.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update price pinning settings, error: %v", err), http.StatusBadRequest) + } else if err := ps.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update pinned settings, error: %v", err), http.StatusBadRequest) return - } else if pps.Enabled { - if _, err := ibus.NewForexClient(pps.ForexEndpointURL).SiacoinExchangeRate(jc.Request.Context(), pps.Currency); err != nil { - jc.Error(fmt.Errorf("couldn't update price pinning settings, forex API unreachable,error: %v", err), http.StatusBadRequest) + } else if ps.Enabled { + if _, err := ibus.NewForexClient(ps.ForexEndpointURL).SiacoinExchangeRate(jc.Request.Context(), ps.Currency); err != nil { + jc.Error(fmt.Errorf("couldn't update pinned settings, forex API unreachable,error: %v", err), http.StatusBadRequest) return } } - if jc.Check("could not update price pinning settings", b.ss.UpdatePinnedSettings(jc.Request.Context(), pps)) == nil { + if jc.Check("could not update pinned settings", b.ss.UpdatePinnedSettings(jc.Request.Context(), ps)) == nil { b.broadcastAction(webhooks.Event{ Module: api.ModuleSetting, Event: api.EventUpdate, Payload: api.EventSettingUpdate{ - Key: api.SettingPinned, - Update: pps, - Timestamp: time.Now().UTC(), + PinnedSettings: &ps, + Timestamp: time.Now().UTC(), }, }) b.pinMgr.TriggerUpdate() @@ -1349,9 +1347,8 @@ func (b *Bus) settingsUploadHandlerPUT(jc jape.Context) { Module: api.ModuleSetting, Event: api.EventUpdate, Payload: api.EventSettingUpdate{ - Key: api.SettingUpload, - Update: us, - Timestamp: time.Now().UTC(), + UploadSettings: &us, + Timestamp: time.Now().UTC(), }, }) } @@ -1377,9 +1374,8 @@ func (b *Bus) settingsS3HandlerPUT(jc jape.Context) { Module: api.ModuleSetting, Event: api.EventUpdate, Payload: api.EventSettingUpdate{ - Key: api.SettingS3, - Update: s3s, - Timestamp: time.Now().UTC(), + S3Settings: &s3s, + Timestamp: time.Now().UTC(), }, }) } diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index 3bc579f28..716068565 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -25,7 +25,7 @@ type ( UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error PinnedSettings(ctx context.Context) (api.PinnedSettings, error) - UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error + UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error } ) @@ -297,9 +297,8 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin Module: api.ModuleSetting, Event: api.EventUpdate, Payload: api.EventSettingUpdate{ - Key: api.SettingGouging, - Update: gs, - Timestamp: time.Now().UTC(), + GougingSettings: &gs, + Timestamp: time.Now().UTC(), }, }) } diff --git a/internal/test/e2e/events_test.go b/internal/test/e2e/events_test.go index e1bc29df7..5f6d741ee 100644 --- a/internal/test/e2e/events_test.go +++ b/internal/test/e2e/events_test.go @@ -169,15 +169,9 @@ func TestEvents(t *testing.T) { t.Fatalf("unexpected event %+v", e) } case api.EventSettingUpdate: - if e.Key != api.SettingGouging || e.Timestamp.IsZero() { + if e.GougingSettings == nil || e.GougingSettings.HostBlockHeightLeeway != 100 || e.Timestamp.IsZero() { t.Fatalf("unexpected event %+v", e) } - var update api.GougingSettings - bytes, _ := json.Marshal(e.Update) - tt.OK(json.Unmarshal(bytes, &update)) - if update.HostBlockHeightLeeway != 100 { - t.Fatalf("unexpected update %+v", update) - } } } } diff --git a/internal/worker/cache.go b/internal/worker/cache.go index ebb071acd..1f5d28d22 100644 --- a/internal/worker/cache.go +++ b/internal/worker/cache.go @@ -181,8 +181,8 @@ func (c *cache) HandleEvent(event webhooks.Event) (err error) { log = log.With("hk", e.HostKey, "ts", e.Timestamp) c.handleHostUpdate(e) case api.EventSettingUpdate: - log = log.With("key", e.Key, "ts", e.Timestamp) - err = c.handleSettingUpdate(e) + log = log.With("gouging", e.GougingSettings != nil, "pinned", e.PinnedSettings != nil, "upload", e.UploadSettings != nil, "ts", e.Timestamp) + c.handleSettingUpdate(e) default: log.Info("unhandled event", e) return @@ -307,46 +307,22 @@ func (c *cache) handleHostUpdate(e api.EventHostUpdate) { c.cache.Set(cacheKeyDownloadContracts, contracts) } -func (c *cache) handleSettingUpdate(e api.EventSettingUpdate) (err error) { +func (c *cache) handleSettingUpdate(e api.EventSettingUpdate) { // return early if the cache doesn't have gouging params to update value, found, _ := c.cache.Get(cacheKeyGougingParams) if !found { - return nil + return } - gp := value.(api.GougingParams) - // marshal the updated value - data, err := json.Marshal(e.Update) - if err != nil { - return fmt.Errorf("couldn't marshal the given value, error: %v", err) + // update the cache + gp := value.(api.GougingParams) + if e.GougingSettings != nil { + gp.GougingSettings = *e.GougingSettings } - - // unmarshal into the appropriated setting and update the cache - switch e.Key { - case api.SettingGouging: - var gs api.GougingSettings - if err := json.Unmarshal(data, &gs); err != nil { - return fmt.Errorf("couldn't update gouging settings, invalid request body, %t", e.Update) - } else if err := gs.Validate(); err != nil { - return fmt.Errorf("couldn't update gouging settings, error: %v", err) - } - - gp.GougingSettings = gs - c.cache.Set(cacheKeyGougingParams, gp) - case api.SettingUpload: - var us api.UploadSettings - if err := json.Unmarshal(data, &us); err != nil { - return fmt.Errorf("couldn't update upload settings, invalid request body, %t", e.Update) - } else if err := us.Validate(); err != nil { - return fmt.Errorf("couldn't update upload settings, error: %v", err) - } - - gp.RedundancySettings = us.Redundancy - c.cache.Set(cacheKeyGougingParams, gp) - default: + if e.UploadSettings != nil { + gp.RedundancySettings = e.UploadSettings.Redundancy } - - return nil + c.cache.Set(cacheKeyGougingParams, gp) } func contractsEqual(x, y []api.ContractMetadata) bool { diff --git a/stores/settingsdb.go b/stores/settingsdb.go index 658b6de1f..31190f769 100644 --- a/stores/settingsdb.go +++ b/stores/settingsdb.go @@ -9,8 +9,15 @@ import ( sql "go.sia.tech/renterd/stores/sql" ) +const ( + SettingGouging = "gouging" + SettingPinned = "pinned" + SettingS3 = "s3" + SettingUpload = "upload" +) + func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, _ error) { - value, err := s.fetchSetting(ctx, api.SettingGouging) + value, err := s.fetchSetting(ctx, SettingGouging) if err != nil { return api.GougingSettings{}, err } @@ -27,32 +34,32 @@ func (s *SQLStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSett if err != nil { return fmt.Errorf("couldn't marshal the given value, error: %v", err) } - return s.updateSetting(ctx, api.SettingGouging, string(data)) + return s.updateSetting(ctx, SettingGouging, string(data)) } -func (s *SQLStore) PinnedSettings(ctx context.Context) (pps api.PinnedSettings, _ error) { - value, err := s.fetchSetting(ctx, api.SettingPinned) +func (s *SQLStore) PinnedSettings(ctx context.Context) (ps api.PinnedSettings, _ error) { + value, err := s.fetchSetting(ctx, SettingPinned) if err != nil { return api.PinnedSettings{}, err } - if err := json.Unmarshal([]byte(value), &pps); err != nil { + if err := json.Unmarshal([]byte(value), &ps); err != nil { s.logger.Panicf("failed to unmarshal pinned settings '%s': %v", value, err) return api.PinnedSettings{}, err } return } -func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, pps api.PinnedSettings) error { - data, err := json.Marshal(pps) +func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { + data, err := json.Marshal(ps) if err != nil { return fmt.Errorf("couldn't marshal the given value, error: %v", err) } - return s.updateSetting(ctx, api.SettingPinned, string(data)) + return s.updateSetting(ctx, SettingPinned, string(data)) } func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, _ error) { - value, err := s.fetchSetting(ctx, api.SettingUpload) + value, err := s.fetchSetting(ctx, SettingUpload) if err != nil { return api.UploadSettings{}, err } @@ -69,11 +76,11 @@ func (s *SQLStore) UpdateUploadSettings(ctx context.Context, us api.UploadSettin if err != nil { return fmt.Errorf("couldn't marshal the given value, error: %v", err) } - return s.updateSetting(ctx, api.SettingUpload, string(data)) + return s.updateSetting(ctx, SettingUpload, string(data)) } func (s *SQLStore) S3Settings(ctx context.Context) (ss api.S3Settings, _ error) { - value, err := s.fetchSetting(ctx, api.SettingS3) + value, err := s.fetchSetting(ctx, SettingS3) if err != nil { return api.S3Settings{}, err } @@ -90,7 +97,7 @@ func (s *SQLStore) UpdateS3Settings(ctx context.Context, ss api.S3Settings) erro if err != nil { return fmt.Errorf("couldn't marshal the given value, error: %v", err) } - return s.updateSetting(ctx, api.SettingS3, string(data)) + return s.updateSetting(ctx, SettingS3, string(data)) } func (s *SQLStore) DeleteSetting(ctx context.Context, key string) (err error) { From ed1679034330e69e48b61dda1897fa17e9171bd7 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 Aug 2024 10:52:08 +0200 Subject: [PATCH 08/33] testing: fix TestPinManager --- internal/bus/pinmanager_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index a90d76e38..1798a92a1 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -157,8 +157,8 @@ func (ms *mockPinStore) PinnedSettings(ctx context.Context) (api.PinnedSettings, func (ms *mockPinStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { ms.mu.Lock() - defer ms.mu.Unlock() ms.ps = ps + ms.mu.Unlock() time.Sleep(2 * testUpdateInterval) return nil } From f21c2cbe4a3a64f0b06c7f72f33d7992ab58696f Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 Aug 2024 11:09:24 +0200 Subject: [PATCH 09/33] testing: fix race in TestPinManager --- internal/bus/pinmanager_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index 1798a92a1..8716197ee 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -156,8 +156,16 @@ func (ms *mockPinStore) PinnedSettings(ctx context.Context) (api.PinnedSettings, } func (ms *mockPinStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { + b, err := json.Marshal(ps) + if err != nil { + return err + } + var cloned api.PinnedSettings + if err := json.Unmarshal(b, &cloned); err != nil { + return err + } ms.mu.Lock() - ms.ps = ps + ms.ps = cloned ms.mu.Unlock() time.Sleep(2 * testUpdateInterval) return nil From b2c5eeff4b6dacd8d8c387719fe5f335d75dc26e Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 3 Sep 2024 15:33:19 +0200 Subject: [PATCH 10/33] stores: unmarshal in store --- stores/settingsdb.go | 81 ++++++++++++++------------------------------ 1 file changed, 26 insertions(+), 55 deletions(-) diff --git a/stores/settingsdb.go b/stores/settingsdb.go index 31190f769..e4eeb20c9 100644 --- a/stores/settingsdb.go +++ b/stores/settingsdb.go @@ -16,16 +16,8 @@ const ( SettingUpload = "upload" ) -func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, _ error) { - value, err := s.fetchSetting(ctx, SettingGouging) - if err != nil { - return api.GougingSettings{}, err - } - - if err := json.Unmarshal([]byte(value), &gs); err != nil { - s.logger.Panicf("failed to unmarshal gouging settings '%s': %v", value, err) - return api.GougingSettings{}, err - } +func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { + err = s.fetchSetting(ctx, SettingGouging, &gs) return } @@ -37,16 +29,8 @@ func (s *SQLStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSett return s.updateSetting(ctx, SettingGouging, string(data)) } -func (s *SQLStore) PinnedSettings(ctx context.Context) (ps api.PinnedSettings, _ error) { - value, err := s.fetchSetting(ctx, SettingPinned) - if err != nil { - return api.PinnedSettings{}, err - } - - if err := json.Unmarshal([]byte(value), &ps); err != nil { - s.logger.Panicf("failed to unmarshal pinned settings '%s': %v", value, err) - return api.PinnedSettings{}, err - } +func (s *SQLStore) PinnedSettings(ctx context.Context) (ps api.PinnedSettings, err error) { + err = s.fetchSetting(ctx, SettingPinned, ps) return } @@ -58,16 +42,8 @@ func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettin return s.updateSetting(ctx, SettingPinned, string(data)) } -func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, _ error) { - value, err := s.fetchSetting(ctx, SettingUpload) - if err != nil { - return api.UploadSettings{}, err - } - - if err := json.Unmarshal([]byte(value), &us); err != nil { - s.logger.Panicf("failed to unmarshal upload settings '%s': %v", value, err) - return api.UploadSettings{}, err - } +func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, err error) { + err = s.fetchSetting(ctx, SettingUpload, us) return } @@ -79,16 +55,8 @@ func (s *SQLStore) UpdateUploadSettings(ctx context.Context, us api.UploadSettin return s.updateSetting(ctx, SettingUpload, string(data)) } -func (s *SQLStore) S3Settings(ctx context.Context) (ss api.S3Settings, _ error) { - value, err := s.fetchSetting(ctx, SettingS3) - if err != nil { - return api.S3Settings{}, err - } - - if err := json.Unmarshal([]byte(value), &ss); err != nil { - s.logger.Panicf("failed to unmarshal s3 settings '%s': %v", value, err) - return api.S3Settings{}, err - } +func (s *SQLStore) S3Settings(ctx context.Context) (ss api.S3Settings, err error) { + err = s.fetchSetting(ctx, SettingS3, ss) return } @@ -119,26 +87,29 @@ func (s *SQLStore) Setting(ctx context.Context, key string, out interface{}) (er return json.Unmarshal([]byte(value), &out) } -func (s *SQLStore) fetchSetting(ctx context.Context, key string) (string, error) { - // check cache first +func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{}) error { s.settingsMu.Lock() defer s.settingsMu.Unlock() - value, ok := s.settings[key] - if ok { - return value, nil - } - // check database - var err error - err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - value, err = tx.Setting(ctx, key) + value, ok := s.settings[key] + if !ok { + var err error + if err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + value, err = tx.Setting(ctx, key) + return err + }); err != nil { + return fmt.Errorf("failed to fetch setting from db: %w", err) + } + s.settings[key] = value + } + + // unmarshal setting + if err := json.Unmarshal([]byte(value), &out); err != nil { + s.logger.Panicf("failed to unmarshal %s setting '%s': %v", key, value, err) return err - }) - if err != nil { - return "", fmt.Errorf("failed to fetch setting from db: %w", err) } - s.settings[key] = value - return value, nil + + return nil } func (s *SQLStore) updateSetting(ctx context.Context, key, value string) error { From 1c4648d63a16b664285e52224082251bc5e07f28 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 3 Sep 2024 17:16:07 +0200 Subject: [PATCH 11/33] settings: update store --- api/setting.go | 15 +- bus/bus.go | 116 +-------------- bus/routes.go | 27 ++-- cmd/renterd/node.go | 2 +- internal/bus/pinmanager.go | 5 +- internal/test/e2e/cluster.go | 4 +- stores/settings.go | 281 +++++++++++++++++++++++++++++++++++ stores/settingsdb.go | 130 ---------------- stores/sql.go | 5 +- stores/sql/main.go | 7 +- stores/sql_test.go | 3 +- 11 files changed, 319 insertions(+), 276 deletions(-) create mode 100644 stores/settings.go delete mode 100644 stores/settingsdb.go diff --git a/api/setting.go b/api/setting.go index e4741e5ff..a9443d6be 100644 --- a/api/setting.go +++ b/api/setting.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "go.sia.tech/core/consensus" rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" ) @@ -19,10 +20,6 @@ var ( // ErrInvalidRedundancySettings is returned if the redundancy settings are // not valid ErrInvalidRedundancySettings = errors.New("invalid redundancy settings") - - // ErrSettingNotFound is returned if a requested setting is not present in the - // database. - ErrSettingNotFound = errors.New("setting not found") ) var ( @@ -65,10 +62,12 @@ var ( V4Keypairs: map[string]string{}, }, } +) - // DefaultUploadSettings define the default upload settings the bus is - // configured with on startup. - DefaultUploadSettings = UploadSettings{ +// DefaultUploadSettings define the default upload settings the bus is +// configured with on startup. +func DefaultUploadSettings(network *consensus.Network) UploadSettings { + return UploadSettings{ Packing: UploadPackingSettings{ Enabled: true, SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB @@ -78,7 +77,7 @@ var ( TotalShards: 30, }, } -) +} type ( // GougingSettings contain some price settings used in price gouging. diff --git a/bus/bus.go b/bus/bus.go index 718b4a091..4bb509d73 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -31,7 +31,6 @@ import ( rhp3 "go.sia.tech/renterd/internal/rhp/v3" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" - "go.sia.tech/renterd/stores" "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" @@ -297,9 +296,7 @@ type ( S3Settings(ctx context.Context) (api.S3Settings, error) UpdateS3Settings(ctx context.Context, s3as api.S3Settings) error - // required for compat - Setting(ctx context.Context, key string, out interface{}) error - DeleteSetting(ctx context.Context, key string) error + MigrateV2Settings(ctx context.Context) error } WalletMetricsRecorder interface { @@ -364,6 +361,11 @@ func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksMa rhp3: rhp3.New(rhp.NewFallbackDialer(store, net.Dialer{}, l), l), } + // migrate settings store + if err := store.MigrateV2Settings(ctx); err != nil { + return nil, err + } + // create contract locker b.contractLocker = ibus.NewContractLocker() @@ -379,11 +381,6 @@ func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksMa // create wallet metrics recorder b.walletMetricsRecorder = ibus.NewWalletMetricRecorder(store, w, defaultWalletRecordMetricInterval, l) - // migrate settings to V2 types - if err := b.compatV2Settings(ctx); err != nil { - return nil, err - } - return b, nil } @@ -574,107 +571,6 @@ func (b *Bus) addRenewedContract(ctx context.Context, renewedFrom types.FileCont return r, nil } -func (b *Bus) compatV2Settings(ctx context.Context) error { - // escape early if all settings are present - if !errors.Is(errors.Join( - b.ss.Setting(ctx, stores.SettingGouging, nil), - b.ss.Setting(ctx, stores.SettingPinned, nil), - b.ss.Setting(ctx, stores.SettingS3, nil), - b.ss.Setting(ctx, stores.SettingUpload, nil), - ), api.ErrSettingNotFound) { - return nil - } - - // migrate gouging settings - if _, err := b.ss.GougingSettings(ctx); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - return err - } else if errors.Is(err, api.ErrSettingNotFound) { - if err := b.ss.UpdateGougingSettings(ctx, api.DefaultGougingSettings); err != nil { - return err - } - } - - // migrate S3 settings - var s3as api.S3AuthenticationSettings - if err := b.ss.Setting(ctx, "s3authentication", &s3as); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - return err - } else if errors.Is(err, api.ErrSettingNotFound) { - if err := b.ss.UpdateS3Settings(ctx, api.DefaultS3Settings); err != nil { - return err - } - } else if err == nil { - s3s := api.S3Settings{Authentication: s3as} - if err := s3s.Validate(); err != nil { - return fmt.Errorf("failed to migrate S3 setting: %w", err) - } else if err := b.ss.UpdateS3Settings(ctx, s3s); err != nil { - return err - } - } - - // migrate pinned settings - var ps api.PinnedSettings - if err := b.ss.Setting(ctx, "pricepinning", &ps); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - return err - } else if errors.Is(err, api.ErrSettingNotFound) { - if err := b.ss.UpdatePinnedSettings(ctx, api.DefaultPinnedSettings); err != nil { - return err - } - } else { - if err := ps.Validate(); err != nil { - return fmt.Errorf("failed to migrate pinned setting: %w", err) - } else if err := b.ss.UpdatePinnedSettings(ctx, ps); err != nil { - return err - } - } - - // migrate upload settings - us := api.DefaultUploadSettings - var css struct { - Default string `json:"default"` - } - - // override default contract set on default upload settings - if err := b.ss.Setting(ctx, "contractset", &css); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - return err - } else if err == nil { - us.DefaultContractSet = css.Default - } - - // override redundancy settings on default upload settings - var rs api.RedundancySettings - if err := b.ss.Setting(ctx, "redundancy", &rs); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - return err - } else if errors.Is(err, api.ErrSettingNotFound) { - // default redundancy settings for testnet are different from mainnet - if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { - us.Redundancy = api.DefaultRedundancySettingsTestnet - } - } else { - us.Redundancy = rs - } - - // override upload packing settings on default upload settings - var ups api.UploadPackingSettings - if err := b.ss.Setting(ctx, "uploadpacking", &ups); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - return err - } else if err == nil { - us.Packing = ups - } - - if err := us.Validate(); err != nil { - return fmt.Errorf("failed to migrate upload setting: %w", err) - } else if err := b.ss.UpdateUploadSettings(ctx, us); err != nil { - return err - } - - // delete old settings - return errors.Join( - b.ss.DeleteSetting(ctx, "contractset"), - b.ss.DeleteSetting(ctx, "pricepinning"), - b.ss.DeleteSetting(ctx, "uploadpacking"), - ) -} - func (b *Bus) deriveRenterKey(hostKey types.PublicKey) types.PrivateKey { seed := blake2b.Sum256(append(b.deriveSubKey("renterkey"), hostKey[:]...)) pk := types.NewPrivateKeyFromSeed(seed[:]) diff --git a/bus/routes.go b/bus/routes.go index e4159dbeb..52b32ab5e 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -1348,9 +1348,8 @@ func (b *Bus) packedSlabsHandlerDonePOST(jc jape.Context) { } func (b *Bus) settingsGougingHandlerGET(jc jape.Context) { - if gs, err := b.ss.GougingSettings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { - jc.Error(err, http.StatusNotFound) - } else if jc.Check("failed to get gouging settings", err) == nil { + gs, err := b.ss.GougingSettings(jc.Request.Context()) + if jc.Check("failed to get gouging settings", err) == nil { jc.Encode(gs) } } @@ -1376,9 +1375,8 @@ func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { } func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { - if pps, err := b.ss.PinnedSettings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { - jc.Error(err, http.StatusNotFound) - } else if jc.Check("failed to get pinned settings", err) == nil { + pps, err := b.ss.PinnedSettings(jc.Request.Context()) + if jc.Check("failed to get pinned settings", err) == nil { // populate the Autopilots map with the current autopilots aps, err := b.as.Autopilots(jc.Request.Context()) if jc.Check("failed to fetch autopilots", err) != nil { @@ -1423,9 +1421,8 @@ func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { } func (b *Bus) settingsUploadHandlerGET(jc jape.Context) { - if us, err := b.ss.UploadSettings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { - jc.Error(err, http.StatusNotFound) - } else if jc.Check("failed to get upload settings", err) == nil { + us, err := b.ss.UploadSettings(jc.Request.Context()) + if jc.Check("failed to get upload settings", err) == nil { jc.Encode(us) } } @@ -1450,9 +1447,8 @@ func (b *Bus) settingsUploadHandlerPUT(jc jape.Context) { } func (b *Bus) settingsS3HandlerGET(jc jape.Context) { - if s3s, err := b.ss.S3Settings(jc.Request.Context()); errors.Is(err, api.ErrSettingNotFound) { - jc.Error(err, http.StatusNotFound) - } else if jc.Check("failed to get S3 settings", err) == nil { + s3s, err := b.ss.S3Settings(jc.Request.Context()) + if jc.Check("failed to get S3 settings", err) == nil { jc.Encode(s3s) } } @@ -1610,7 +1606,7 @@ func (b *Bus) slabsPartialHandlerPOST(jc jape.Context) { return } us, err := b.ss.UploadSettings(jc.Request.Context()) - if err != nil && !errors.Is(err, api.ErrSettingNotFound) { + if err != nil { jc.Error(fmt.Errorf("could not get upload packing settings: %w", err), http.StatusInternalServerError) return } @@ -1645,10 +1641,7 @@ func (b *Bus) paramsHandlerUploadGET(jc jape.Context) { var uploadPacking bool var contractSet string us, err := b.ss.UploadSettings(jc.Request.Context()) - if err != nil && !errors.Is(err, api.ErrSettingNotFound) { - jc.Error(fmt.Errorf("could not get upload settings: %w", err), http.StatusInternalServerError) - return - } else if err == nil { + if jc.Check("could not get upload settings", err) == nil { contractSet = us.DefaultContractSet uploadPacking = us.Packing.Enabled } diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 5510788bb..6721408ca 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -264,7 +264,7 @@ func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network if err != nil { return nil, nil, err } - sqlStore, err := stores.NewSQLStore(storeCfg) + sqlStore, err := stores.NewSQLStore(storeCfg, network) if err != nil { return nil, nil, err } diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index 716068565..8929b9064 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -311,10 +311,7 @@ func (pm *pinManager) updatePrices(ctx context.Context, forced bool) error { // fetch pinned settings settings, err := pm.s.PinnedSettings(ctx) - if errors.Is(err, api.ErrSettingNotFound) { - pm.logger.Debug("price pinning not configured, skipping price update") - return nil - } else if err != nil { + if err != nil { return fmt.Errorf("failed to fetch pinned settings: %w", err) } else if !settings.Enabled { pm.logger.Debug("price pinning is disabled, skipping price update") diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 1c59cddfe..f96440835 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -526,7 +526,8 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, return nil, nil, nil, nil, err } - sqlStore, err := stores.NewSQLStore(storeCfg) + network, genesis := testNetwork() + sqlStore, err := stores.NewSQLStore(storeCfg, network) if err != nil { return nil, nil, nil, nil, err } @@ -554,7 +555,6 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, } // create chain manager - network, genesis := testNetwork() store, state, err := chain.NewDBStore(bdb, network, genesis) if err != nil { return nil, nil, nil, nil, err diff --git a/stores/settings.go b/stores/settings.go new file mode 100644 index 000000000..4d13262c6 --- /dev/null +++ b/stores/settings.go @@ -0,0 +1,281 @@ +package stores + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "go.sia.tech/renterd/api" + sql "go.sia.tech/renterd/stores/sql" + "go.uber.org/zap" +) + +const ( + SettingGouging = "gouging" + SettingPinned = "pinned" + SettingS3 = "s3" + SettingUpload = "upload" +) + +func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { + err = s.fetchSetting(ctx, SettingPinned, &gs) + return +} + +func (s *SQLStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { + return s.updateSetting(ctx, SettingGouging, gs) +} + +func (s *SQLStore) PinnedSettings(ctx context.Context) (ps api.PinnedSettings, err error) { + err = s.fetchSetting(ctx, SettingPinned, &ps) + return +} + +func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { + return s.updateSetting(ctx, SettingPinned, ps) +} + +func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, err error) { + err = s.fetchSetting(ctx, SettingUpload, &us) + return +} + +func (s *SQLStore) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error { + return s.updateSetting(ctx, SettingUpload, us) +} + +func (s *SQLStore) S3Settings(ctx context.Context) (ss api.S3Settings, err error) { + err = s.fetchSetting(ctx, SettingS3, &ss) + return +} + +func (s *SQLStore) UpdateS3Settings(ctx context.Context, ss api.S3Settings) error { + return s.updateSetting(ctx, SettingS3, ss) +} + +// MigrateV2Settings migrates the settings from the old format to the new, +// migrating the existing settings over to the new types and removing the old +// settings. If a setting is not present in the database it will be set to its +// default setting. If an existing setting is not valid, the default will be +// used and a warning will get logged. +func (s *SQLStore) MigrateV2Settings(ctx context.Context) error { + return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + // escape early if none of the old settings are present + var found bool + for _, key := range []string{ + "pricepinning", + "s3authentication", + "contractset", + "redundancy", + "uploadpacking", + } { + if _, err := tx.Setting(ctx, key); err != nil && !errors.Is(err, sql.ErrSettingNotFound) { + return err + } else if err == nil { + found = true + break + } + } + if !found { + return nil + } + + s.logger.Info("migrating settings...") + + // migrate gouging settings + value, err := tx.Setting(ctx, "gouging") + if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { + return err + } else if errors.Is(err, sql.ErrSettingNotFound) { + if err := tx.UpdateSetting(ctx, SettingGouging, s.defaultSetting(SettingGouging)); err != nil { + return err + } + } + + // migrate pinned settings + value, err = tx.Setting(ctx, "pricepinning") + if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { + return err + } else if err == nil { + var ps api.PinnedSettings + if err := json.Unmarshal([]byte(value), &ps); err != nil { + s.logger.Warnw("failed to unmarshal pinned settings, using default", zap.Error(err)) + value = s.defaultSetting(SettingPinned) + } else if err := ps.Validate(); err != nil { + s.logger.Warnw("failed to migrate pinned settings, using default", zap.Error(err)) + value = s.defaultSetting(SettingPinned) + } + + // update setting and delete old value + if err := tx.UpdateSetting(ctx, SettingPinned, value); err != nil { + return err + } else if err := tx.DeleteSetting(ctx, "pricepinning"); err != nil { + return err + } + } + + // migrate s3 settings + value, err = tx.Setting(ctx, "s3authentication") + if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { + return err + } else if err == nil { + var s3s api.S3Settings + if err := json.Unmarshal([]byte(value), &s3s.Authentication); err != nil { + s.logger.Warnw("failed to unmarshal S3 authentication settings, using default", zap.Error(err)) + s3s = api.DefaultS3Settings + } else if err := s3s.Validate(); err != nil { + s.logger.Warnw("failed to migrate S3 settings, using default", zap.Error(err)) + s3s = api.DefaultS3Settings + } + + // update setting and delete old value + update, _ := json.Marshal(s3s) + if err := tx.UpdateSetting(ctx, SettingS3, string(update)); err != nil { + return err + } else if err := tx.DeleteSetting(ctx, "s3authentication"); err != nil { + return err + } + } + + us := api.DefaultUploadSettings(s.network) + + // migrate contractset settings + value, err = tx.Setting(ctx, "contractset") + if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { + return err + } else if err == nil { + var css struct { + Default string `json:"default"` + } + if err := json.Unmarshal([]byte(value), &css); err != nil { + s.logger.Warnw("failed to unmarshal contractset setting, using default", zap.Error(err)) + } else { + us.DefaultContractSet = css.Default + } + + // delete old value + if err := tx.DeleteSetting(ctx, "contractset"); err != nil { + return err + } + } + + // migrate redundancy settings + value, err = tx.Setting(ctx, "redundancy") + if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { + return err + } else if err == nil { + var rs api.RedundancySettings + if err := json.Unmarshal([]byte(value), &rs); err != nil { + s.logger.Warnw("failed to unmarshal redundancy settings, using default", zap.Error(err)) + } else if err := rs.Validate(); err != nil { + s.logger.Warnw("failed to migrate redundancy settings, using default", zap.Error(err)) + } else { + us.Redundancy = rs + } + + // delete old value + if err := tx.DeleteSetting(ctx, "redundancy"); err != nil { + return err + } + } + + // migrate uploadpacking settings + value, err = tx.Setting(ctx, "uploadpacking") + if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { + return err + } else if err == nil { + var ups api.UploadPackingSettings + if err := json.Unmarshal([]byte(value), &ups); err != nil { + s.logger.Warnw("failed to unmarshal uploadpacking settings, using default", zap.Error(err)) + } else { + us.Packing = ups + } + + // delete old value + if err := tx.DeleteSetting(ctx, "uploadpacking"); err != nil { + return err + } + } + + // update upload settings + if update, err := json.Marshal(us); err != nil { + return fmt.Errorf("failed to marshal upload settings: %w", err) + } else if err := tx.UpdateSetting(ctx, SettingUpload, string(update)); err != nil { + return err + } + + s.logger.Info("successfully migrated settings") + return nil + }) +} + +func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{}) error { + s.settingsMu.Lock() + defer s.settingsMu.Unlock() + + // fetch setting value + value, ok := s.settings[key] + if !ok { + var err error + if err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + value, err = tx.Setting(ctx, key) + return err + }); err != nil && !errors.Is(err, sql.ErrSettingNotFound) { + return fmt.Errorf("failed to fetch setting from db: %w", err) + } else if err != nil { + value = s.defaultSetting(key) + } + s.settings[key] = value + } + + // unmarshal setting + if err := json.Unmarshal([]byte(value), &out); err != nil { + s.logger.Warnf("failed to unmarshal %s setting '%s': %v, using default", key, value, err) + return json.Unmarshal([]byte(s.defaultSetting(key)), &out) + } + + return nil +} + +func (s *SQLStore) updateSetting(ctx context.Context, key string, value any) error { + s.settingsMu.Lock() + defer s.settingsMu.Unlock() + + // marshal the value + b, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("couldn't marshal the given value, error: %v", err) + } + + // update db first + err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + return tx.UpdateSetting(ctx, key, string(b)) + }) + if err != nil { + return err + } + + // update cache second + s.settings[key] = string(b) + return nil +} + +func (s *SQLStore) defaultSetting(key string) string { + switch key { + case SettingGouging: + b, _ := json.Marshal(api.DefaultGougingSettings) + return string(b) + case SettingPinned: + b, _ := json.Marshal(api.DefaultPinnedSettings) + return string(b) + case SettingS3: + b, _ := json.Marshal(api.DefaultS3Settings) + return string(b) + case SettingUpload: + b, _ := json.Marshal(api.DefaultUploadSettings(s.network)) + return string(b) + default: + panic("unknown setting") // developer error + } +} diff --git a/stores/settingsdb.go b/stores/settingsdb.go deleted file mode 100644 index e4eeb20c9..000000000 --- a/stores/settingsdb.go +++ /dev/null @@ -1,130 +0,0 @@ -package stores - -import ( - "context" - "encoding/json" - "fmt" - - "go.sia.tech/renterd/api" - sql "go.sia.tech/renterd/stores/sql" -) - -const ( - SettingGouging = "gouging" - SettingPinned = "pinned" - SettingS3 = "s3" - SettingUpload = "upload" -) - -func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { - err = s.fetchSetting(ctx, SettingGouging, &gs) - return -} - -func (s *SQLStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { - data, err := json.Marshal(gs) - if err != nil { - return fmt.Errorf("couldn't marshal the given value, error: %v", err) - } - return s.updateSetting(ctx, SettingGouging, string(data)) -} - -func (s *SQLStore) PinnedSettings(ctx context.Context) (ps api.PinnedSettings, err error) { - err = s.fetchSetting(ctx, SettingPinned, ps) - return -} - -func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { - data, err := json.Marshal(ps) - if err != nil { - return fmt.Errorf("couldn't marshal the given value, error: %v", err) - } - return s.updateSetting(ctx, SettingPinned, string(data)) -} - -func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, err error) { - err = s.fetchSetting(ctx, SettingUpload, us) - return -} - -func (s *SQLStore) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error { - data, err := json.Marshal(us) - if err != nil { - return fmt.Errorf("couldn't marshal the given value, error: %v", err) - } - return s.updateSetting(ctx, SettingUpload, string(data)) -} - -func (s *SQLStore) S3Settings(ctx context.Context) (ss api.S3Settings, err error) { - err = s.fetchSetting(ctx, SettingS3, ss) - return -} - -func (s *SQLStore) UpdateS3Settings(ctx context.Context, ss api.S3Settings) error { - data, err := json.Marshal(ss) - if err != nil { - return fmt.Errorf("couldn't marshal the given value, error: %v", err) - } - return s.updateSetting(ctx, SettingS3, string(data)) -} - -func (s *SQLStore) DeleteSetting(ctx context.Context, key string) (err error) { - return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - return tx.DeleteSetting(ctx, key) - }) -} - -func (s *SQLStore) Setting(ctx context.Context, key string, out interface{}) (err error) { - var value string - err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - value, err = tx.Setting(ctx, key) - return err - }) - if err != nil { - return fmt.Errorf("failed to fetch setting from db: %w", err) - } - - return json.Unmarshal([]byte(value), &out) -} - -func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{}) error { - s.settingsMu.Lock() - defer s.settingsMu.Unlock() - - value, ok := s.settings[key] - if !ok { - var err error - if err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - value, err = tx.Setting(ctx, key) - return err - }); err != nil { - return fmt.Errorf("failed to fetch setting from db: %w", err) - } - s.settings[key] = value - } - - // unmarshal setting - if err := json.Unmarshal([]byte(value), &out); err != nil { - s.logger.Panicf("failed to unmarshal %s setting '%s': %v", key, value, err) - return err - } - - return nil -} - -func (s *SQLStore) updateSetting(ctx context.Context, key, value string) error { - // update db first - s.settingsMu.Lock() - defer s.settingsMu.Unlock() - - err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - return tx.UpdateSetting(ctx, key, value) - }) - if err != nil { - return err - } - - // update cache second - s.settings[key] = value - return nil -} diff --git a/stores/sql.go b/stores/sql.go index 50533768d..cd0c54e08 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/stores/sql" @@ -39,6 +40,7 @@ type ( logger *zap.SugaredLogger walletAddress types.Address + network *consensus.Network // ObjectDB related fields slabBufferMgr *SlabBufferManager @@ -64,7 +66,7 @@ type ( // NewSQLStore uses a given Dialector to connect to a SQL database. NOTE: Only // pass migrate=true for the first instance of SQLHostDB if you connect via the // same Dialector multiple times. -func NewSQLStore(cfg Config) (*SQLStore, error) { +func NewSQLStore(cfg Config, network *consensus.Network) (*SQLStore, error) { if err := os.MkdirAll(cfg.PartialSlabDir, 0700); err != nil { return nil, fmt.Errorf("failed to create partial slab dir '%s': %v", cfg.PartialSlabDir, err) } @@ -97,6 +99,7 @@ func NewSQLStore(cfg Config) (*SQLStore, error) { settings: make(map[string]string), walletAddress: cfg.WalletAddress, + network: network, slabPruneSigChan: make(chan struct{}, 1), lastPrunedAt: time.Now(), diff --git a/stores/sql/main.go b/stores/sql/main.go index 02f876214..948c78a67 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -28,7 +28,10 @@ import ( "lukechampine.com/frand" ) -var ErrNegativeOffset = errors.New("offset can not be negative") +var ( + ErrNegativeOffset = errors.New("offset can not be negative") + ErrSettingNotFound = errors.New("setting not found") +) // helper types type ( @@ -2212,7 +2215,7 @@ func Setting(ctx context.Context, tx sql.Tx, key string) (string, error) { var value string err := tx.QueryRow(ctx, "SELECT value FROM settings WHERE `key` = ?", key).Scan((*BusSetting)(&value)) if errors.Is(err, dsql.ErrNoRows) { - return "", api.ErrSettingNotFound + return "", ErrSettingNotFound } else if err != nil { return "", fmt.Errorf("failed to fetch setting '%s': %w", key, err) } diff --git a/stores/sql_test.go b/stores/sql_test.go index 0846254cb..48fb7a6a9 100644 --- a/stores/sql_test.go +++ b/stores/sql_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" @@ -179,7 +180,7 @@ func newTestSQLStore(t *testing.T, cfg testSQLStoreConfig) *testSQLStore { LongQueryDuration: 100 * time.Millisecond, LongTxDuration: 100 * time.Millisecond, RetryTransactionIntervals: []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond}, - }) + }, &consensus.Network{}) if err != nil { t.Fatal("failed to create SQLStore", err) } From 36a73dbe27fd7beee7f873bd3695ec6210517a8f Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 3 Sep 2024 17:31:19 +0200 Subject: [PATCH 12/33] testing: fix setting key --- stores/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/settings.go b/stores/settings.go index 4d13262c6..2c9fc9a6b 100644 --- a/stores/settings.go +++ b/stores/settings.go @@ -19,7 +19,7 @@ const ( ) func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { - err = s.fetchSetting(ctx, SettingPinned, &gs) + err = s.fetchSetting(ctx, SettingGouging, &gs) return } From 2cab31c631e8121021710b810f2e2b9dbaf99296 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 5 Sep 2024 17:21:29 +0200 Subject: [PATCH 13/33] all: rename pin setting --- api/events.go | 2 +- api/setting.go | 10 +++++----- bus/bus.go | 31 +++++++------------------------ bus/client/settings.go | 6 +++--- bus/routes.go | 15 ++++++++------- internal/bus/pinmanager.go | 10 +++++----- internal/bus/pinmanager_test.go | 26 +++++++++++++------------- internal/test/e2e/cluster.go | 2 +- stores/settings.go | 32 ++++++++++++++++---------------- 9 files changed, 59 insertions(+), 75 deletions(-) diff --git a/api/events.go b/api/events.go index bd6dcd776..43620eda7 100644 --- a/api/events.go +++ b/api/events.go @@ -64,7 +64,7 @@ type ( EventSettingUpdate struct { GougingSettings *GougingSettings `json:"gougingSettings,omitempty"` - PinnedSettings *PinnedSettings `json:"pinnedSettings,omitempty"` + PinnedSettings *PinningSettings `json:"pinnedSettings,omitempty"` S3Settings *S3Settings `json:"s3Settings,omitempty"` UploadSettings *UploadSettings `json:"uploadSettings,omitempty"` Timestamp time.Time `json:"timestamp"` diff --git a/api/setting.go b/api/setting.go index a9443d6be..668acb243 100644 --- a/api/setting.go +++ b/api/setting.go @@ -40,7 +40,7 @@ var ( // DefaultPinnedSettings define the default pin settings the bus is // configured with on startup. - DefaultPinnedSettings = PinnedSettings{ + DefaultPinnedSettings = PinningSettings{ Enabled: false, Currency: "usd", ForexEndpointURL: "https://api.siascan.com/exchange-rate/siacoin", @@ -120,10 +120,10 @@ type ( MigrationSurchargeMultiplier uint64 `json:"migrationSurchargeMultiplier"` } - // PinnedSettings holds the configuration for pinning certain settings to a + // PinningSettings holds the configuration for pinning certain settings to a // specific currency (e.g., USD). It uses a Forex API to fetch the current // exchange rate, allowing users to set prices in USD instead of SC. - PinnedSettings struct { + PinningSettings struct { // Enabled can be used to either enable or temporarily disable price // pinning. If enabled, both the currency and the Forex endpoint URL // must be valid. @@ -137,7 +137,7 @@ type ( ForexEndpointURL string `json:"forexEndpointURL"` // Threshold is a percentage between 0 and 1 that determines when the - // pinned settings are updated based on the exchange rate at the time. + // pinning settings are updated based on the exchange rate at the time. Threshold float64 `json:"threshold"` // Autopilots contains the pinned settings for every autopilot. @@ -202,7 +202,7 @@ func (p Pin) IsPinned() bool { } // Validate returns an error if the price pin settings are not considered valid. -func (pps PinnedSettings) Validate() error { +func (pps PinningSettings) Validate() error { if pps.ForexEndpointURL == "" { return fmt.Errorf("price pin settings must have a forex endpoint URL") } diff --git a/bus/bus.go b/bus/bus.go index e26842664..a5e575d44 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -34,7 +34,6 @@ import ( "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" - "golang.org/x/crypto/blake2b" ) const ( @@ -292,8 +291,8 @@ type ( GougingSettings(ctx context.Context) (api.GougingSettings, error) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error - PinnedSettings(ctx context.Context) (api.PinnedSettings, error) - UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error + PinningSettings(ctx context.Context) (api.PinningSettings, error) + UpdatePinningSettings(ctx context.Context, ps api.PinningSettings) error UploadSettings(ctx context.Context) (api.UploadSettings, error) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error @@ -578,24 +577,6 @@ func (b *Bus) addRenewedContract(ctx context.Context, renewedFrom types.FileCont return r, nil } -func (b *Bus) deriveRenterKey(hostKey types.PublicKey) types.PrivateKey { - seed := blake2b.Sum256(append(b.deriveSubKey("renterkey"), hostKey[:]...)) - pk := types.NewPrivateKeyFromSeed(seed[:]) - for i := range seed { - seed[i] = 0 - } - return pk -} - -func (b *Bus) deriveSubKey(purpose string) types.PrivateKey { - seed := blake2b.Sum256(append(b.masterKey[:], []byte(purpose)...)) - pk := types.NewPrivateKeyFromSeed(seed[:]) - for i := range seed { - seed[i] = 0 - } - return pk -} - func (b *Bus) broadcastContract(ctx context.Context, fcid types.FileContractID) (txnID types.TransactionID, _ error) { // acquire contract lock indefinitely and defer the release lockID, err := b.contractLocker.Acquire(ctx, lockingPriorityRenew, fcid, time.Duration(math.MaxInt64)) @@ -615,7 +596,7 @@ func (b *Bus) broadcastContract(ctx context.Context, fcid types.FileContractID) } // derive the renter key - renterKey := b.deriveRenterKey(c.HostKey) + renterKey := b.masterKey.DeriveContractKey(c.HostKey) // fetch revision rev, err := b.rhp2.SignedRevision(ctx, c.HostIP, c.HostKey, renterKey, fcid, time.Minute) @@ -659,7 +640,7 @@ func (b *Bus) broadcastContract(ctx context.Context, fcid types.FileContractID) func (b *Bus) formContract(ctx context.Context, hostSettings rhpv2.HostSettings, renterAddress types.Address, renterFunds, hostCollateral types.Currency, hostKey types.PublicKey, hostIP string, endHeight uint64) (rhpv2.ContractRevision, error) { // derive the renter key - renterKey := b.deriveRenterKey(hostKey) + renterKey := b.masterKey.DeriveContractKey(hostKey) // prepare the transaction cs := b.cm.TipState() @@ -753,6 +734,9 @@ func (b *Bus) prepareRenew(cs consensus.State, revision types.FileContractRevisi } func (b *Bus) renewContract(ctx context.Context, cs consensus.State, gp api.GougingParams, c api.ContractMetadata, hs rhpv2.HostSettings, renterFunds, minNewCollateral, maxFundAmount types.Currency, endHeight, expectedNewStorage uint64) (rhpv2.ContractRevision, types.Currency, types.Currency, error) { + // derive the renter key + renterKey := b.masterKey.DeriveContractKey(c.HostKey) + // acquire contract lock indefinitely and defer the release lockID, err := b.contractLocker.Acquire(ctx, lockingPriorityRenew, c.ID, time.Duration(math.MaxInt64)) if err != nil { @@ -772,7 +756,6 @@ func (b *Bus) renewContract(ctx context.Context, cs consensus.State, gp api.Goug // renew contract gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState, gp.TransactionFee, nil, nil) - renterKey := b.deriveRenterKey(c.HostKey) prepareRenew := b.prepareRenew(cs, rev, hs.Address, b.w.Address(), renterFunds, minNewCollateral, maxFundAmount, endHeight, expectedNewStorage) newRevision, txnSet, contractPrice, fundAmount, err := b.rhp3.Renew(ctx, gc, rev, renterKey, c.HostKey, c.SiamuxAddr, prepareRenew, b.w.SignTransaction) if err != nil { diff --git a/bus/client/settings.go b/bus/client/settings.go index 46d3e8708..25f9ae4bb 100644 --- a/bus/client/settings.go +++ b/bus/client/settings.go @@ -18,13 +18,13 @@ func (c *Client) UpdateGougingSettings(ctx context.Context, gs api.GougingSettin } // PricePinningSettings returns the contract set settings. -func (c *Client) PricePinningSettings(ctx context.Context) (ps api.PinnedSettings, err error) { +func (c *Client) PricePinningSettings(ctx context.Context) (ps api.PinningSettings, err error) { err = c.c.WithContext(ctx).GET("/settings/pinned", &ps) return } -// UpdatePinnedSettings updates the given setting. -func (c *Client) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { +// UpdatePinningSettings updates the given setting. +func (c *Client) UpdatePinningSettings(ctx context.Context, ps api.PinningSettings) error { return c.c.WithContext(ctx).PUT("/settings/pinned", ps) } diff --git a/bus/routes.go b/bus/routes.go index 6334c24c1..188664e85 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -912,7 +912,8 @@ func (b *Bus) contractPruneHandlerPOST(jc jape.Context) { } // prune the contract - rev, spending, pruned, remaining, err := b.rhp2.PruneContract(pruneCtx, b.deriveRenterKey(c.HostKey), gc, c.HostIP, c.HostKey, fcid, c.RevisionNumber, func(fcid types.FileContractID, roots []types.Hash256) ([]uint64, error) { + rk := b.masterKey.DeriveContractKey(c.HostKey) + rev, spending, pruned, remaining, err := b.rhp2.PruneContract(pruneCtx, rk, gc, c.HostIP, c.HostKey, fcid, c.RevisionNumber, func(fcid types.FileContractID, roots []types.Hash256) ([]uint64, error) { indices, err := b.ms.PrunableContractRoots(ctx, fcid, roots) if err != nil { return nil, err @@ -1487,8 +1488,8 @@ func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { } func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { - pps, err := b.ss.PinnedSettings(jc.Request.Context()) - if jc.Check("failed to get pinned settings", err) == nil { + pps, err := b.ss.PinningSettings(jc.Request.Context()) + if jc.Check("failed to get pinning settings", err) == nil { // populate the Autopilots map with the current autopilots aps, err := b.as.Autopilots(jc.Request.Context()) if jc.Check("failed to fetch autopilots", err) != nil { @@ -1507,19 +1508,19 @@ func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { } func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { - var ps api.PinnedSettings + var ps api.PinningSettings if jc.Decode(&ps) != nil { return } else if err := ps.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update pinned settings, error: %v", err), http.StatusBadRequest) + jc.Error(fmt.Errorf("couldn't update pinning settings, error: %v", err), http.StatusBadRequest) return } else if ps.Enabled { if _, err := ibus.NewForexClient(ps.ForexEndpointURL).SiacoinExchangeRate(jc.Request.Context(), ps.Currency); err != nil { - jc.Error(fmt.Errorf("couldn't update pinned settings, forex API unreachable,error: %v", err), http.StatusBadRequest) + jc.Error(fmt.Errorf("couldn't update pinning settings, forex API unreachable,error: %v", err), http.StatusBadRequest) return } } - if jc.Check("could not update pinned settings", b.ss.UpdatePinnedSettings(jc.Request.Context(), ps)) == nil { + if jc.Check("could not update pinning settings", b.ss.UpdatePinningSettings(jc.Request.Context(), ps)) == nil { b.broadcastAction(webhooks.Event{ Module: api.ModuleSetting, Event: api.EventUpdate, diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index 8929b9064..a02f65dc3 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -24,8 +24,8 @@ type ( GougingSettings(ctx context.Context) (api.GougingSettings, error) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error - PinnedSettings(ctx context.Context) (api.PinnedSettings, error) - UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error + PinningSettings(ctx context.Context) (api.PinningSettings, error) + UpdatePinningSettings(ctx context.Context, ps api.PinningSettings) error } ) @@ -309,10 +309,10 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin func (pm *pinManager) updatePrices(ctx context.Context, forced bool) error { pm.logger.Debugw("updating prices", zap.Bool("forced", forced)) - // fetch pinned settings - settings, err := pm.s.PinnedSettings(ctx) + // fetch pinning settings + settings, err := pm.s.PinningSettings(ctx) if err != nil { - return fmt.Errorf("failed to fetch pinned settings: %w", err) + return fmt.Errorf("failed to fetch pinning settings: %w", err) } else if !settings.Enabled { pm.logger.Debug("price pinning is disabled, skipping price update") return nil diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index 8716197ee..85272610c 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -112,7 +112,7 @@ func (api *mockForexAPI) setUnreachable(unreachable bool) { type mockPinStore struct { mu sync.Mutex gs api.GougingSettings - ps api.PinnedSettings + ps api.PinningSettings autopilots map[string]api.Autopilot } @@ -149,18 +149,18 @@ func (ms *mockPinStore) UpdateGougingSettings(ctx context.Context, gs api.Gougin return nil } -func (ms *mockPinStore) PinnedSettings(ctx context.Context) (api.PinnedSettings, error) { +func (ms *mockPinStore) PinningSettings(ctx context.Context) (api.PinningSettings, error) { ms.mu.Lock() defer ms.mu.Unlock() return ms.ps, nil } -func (ms *mockPinStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { +func (ms *mockPinStore) UpdatePinningSettings(ctx context.Context, ps api.PinningSettings) error { b, err := json.Marshal(ps) if err != nil { return err } - var cloned api.PinnedSettings + var cloned api.PinningSettings if err := json.Unmarshal(b, &cloned); err != nil { return err } @@ -221,7 +221,7 @@ func TestPinManager(t *testing.T) { pps.Currency = "usd" pps.Threshold = 0.5 pps.ForexEndpointURL = forex.s.URL - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) // assert price manager is running now if cnt := len(rates()); cnt < 1 { @@ -236,7 +236,7 @@ func TestPinManager(t *testing.T) { pps.GougingSettingsPins.MaxDownload = api.Pin{Value: 3, Pinned: false} pps.GougingSettingsPins.MaxStorage = api.Pin{Value: 3, Pinned: false} pps.GougingSettingsPins.MaxUpload = api.Pin{Value: 3, Pinned: false} - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) // assert gouging settings are unchanged if gss, _ := ms.GougingSettings(context.Background()); !reflect.DeepEqual(gs, gss) { @@ -245,14 +245,14 @@ func TestPinManager(t *testing.T) { // enable the max download pin, with the threshold at 0.5 it should remain unchanged pps.GougingSettingsPins.MaxDownload.Pinned = true - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) if gss, _ := ms.GougingSettings(context.Background()); !reflect.DeepEqual(gs, gss) { t.Fatalf("expected gouging settings to be the same, got %v", gss) } // lower the threshold, gouging settings should be updated pps.Threshold = 0.05 - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) if gss, _ := ms.GougingSettings(context.Background()); gss.MaxContractPrice.Equals(gs.MaxDownloadPrice) { t.Fatalf("expected gouging settings to be updated, got %v = %v", gss.MaxDownloadPrice, gs.MaxDownloadPrice) } @@ -261,7 +261,7 @@ func TestPinManager(t *testing.T) { pps.GougingSettingsPins.MaxDownload.Pinned = true pps.GougingSettingsPins.MaxStorage.Pinned = true pps.GougingSettingsPins.MaxUpload.Pinned = true - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) // assert they're all updated if gss, _ := ms.GougingSettings(context.Background()); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) || @@ -284,7 +284,7 @@ func TestPinManager(t *testing.T) { }, } pps.Autopilots = map[string]api.AutopilotPins{testAutopilotID: pins} - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) // assert autopilot was not updated if app, _ := ms.Autopilot(context.Background(), testAutopilotID); !app.Config.Contracts.Allowance.Equals(ap.Config.Contracts.Allowance) { @@ -294,7 +294,7 @@ func TestPinManager(t *testing.T) { // enable the pin pins.Allowance.Pinned = true pps.Autopilots[testAutopilotID] = pins - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) // assert autopilot was updated if app, _ := ms.Autopilot(context.Background(), testAutopilotID); app.Config.Contracts.Allowance.Equals(ap.Config.Contracts.Allowance) { @@ -305,7 +305,7 @@ func TestPinManager(t *testing.T) { forex.setUnreachable(true) // assert alert was registered - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) res, _ := a.Alerts(context.Background(), alerts.AlertsOpts{}) if len(res.Alerts) == 0 { t.Fatalf("expected 1 alert, got %d", len(a.alerts)) @@ -315,7 +315,7 @@ func TestPinManager(t *testing.T) { forex.setUnreachable(false) // assert alert was dismissed - ms.UpdatePinnedSettings(context.Background(), pps) + ms.UpdatePinningSettings(context.Background(), pps) res, _ = a.Alerts(context.Background(), alerts.AlertsOpts{}) if len(res.Alerts) != 0 { t.Fatalf("expected 0 alerts, got %d", len(a.alerts)) diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index c1afeed35..9f2b2c0ac 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -502,7 +502,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { // Update the bus settings. tt.OK(busClient.UpdateGougingSettings(ctx, test.GougingSettings)) - tt.OK(busClient.UpdatePinnedSettings(ctx, test.PricePinSettings)) + tt.OK(busClient.UpdatePinningSettings(ctx, test.PricePinSettings)) tt.OK(busClient.UpdateUploadSettings(ctx, us)) tt.OK(busClient.UpdateS3Settings(ctx, s3)) diff --git a/stores/settings.go b/stores/settings.go index 2c9fc9a6b..843d8e986 100644 --- a/stores/settings.go +++ b/stores/settings.go @@ -12,10 +12,10 @@ import ( ) const ( - SettingGouging = "gouging" - SettingPinned = "pinned" - SettingS3 = "s3" - SettingUpload = "upload" + SettingGouging = "gouging" + SettingPricePinning = "pricepinning" + SettingS3 = "s3" + SettingUpload = "upload" ) func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { @@ -27,13 +27,13 @@ func (s *SQLStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSett return s.updateSetting(ctx, SettingGouging, gs) } -func (s *SQLStore) PinnedSettings(ctx context.Context) (ps api.PinnedSettings, err error) { - err = s.fetchSetting(ctx, SettingPinned, &ps) +func (s *SQLStore) PinningSettings(ctx context.Context) (ps api.PinningSettings, err error) { + err = s.fetchSetting(ctx, SettingPricePinning, &ps) return } -func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { - return s.updateSetting(ctx, SettingPinned, ps) +func (s *SQLStore) UpdatePinningSettings(ctx context.Context, ps api.PinningSettings) error { + return s.updateSetting(ctx, SettingPricePinning, ps) } func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, err error) { @@ -93,22 +93,22 @@ func (s *SQLStore) MigrateV2Settings(ctx context.Context) error { } } - // migrate pinned settings + // migrate pinning settings value, err = tx.Setting(ctx, "pricepinning") if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { return err } else if err == nil { - var ps api.PinnedSettings + var ps api.PinningSettings if err := json.Unmarshal([]byte(value), &ps); err != nil { - s.logger.Warnw("failed to unmarshal pinned settings, using default", zap.Error(err)) - value = s.defaultSetting(SettingPinned) + s.logger.Warnw("failed to unmarshal pinning settings, using default", zap.Error(err)) + value = s.defaultSetting(SettingPricePinning) } else if err := ps.Validate(); err != nil { - s.logger.Warnw("failed to migrate pinned settings, using default", zap.Error(err)) - value = s.defaultSetting(SettingPinned) + s.logger.Warnw("failed to migrate pinning settings, using default", zap.Error(err)) + value = s.defaultSetting(SettingPricePinning) } // update setting and delete old value - if err := tx.UpdateSetting(ctx, SettingPinned, value); err != nil { + if err := tx.UpdateSetting(ctx, SettingPricePinning, value); err != nil { return err } else if err := tx.DeleteSetting(ctx, "pricepinning"); err != nil { return err @@ -266,7 +266,7 @@ func (s *SQLStore) defaultSetting(key string) string { case SettingGouging: b, _ := json.Marshal(api.DefaultGougingSettings) return string(b) - case SettingPinned: + case SettingPricePinning: b, _ := json.Marshal(api.DefaultPinnedSettings) return string(b) case SettingS3: From 3f82d0bb89b3e930d5acafd776171cbdbcbe66c3 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 6 Sep 2024 10:40:07 +0200 Subject: [PATCH 14/33] stores: move migration --- api/setting.go | 15 +- bus/bus.go | 7 - cmd/renterd/node.go | 8 +- internal/sql/migrations.go | 149 ++++++++++++++++ internal/test/e2e/cluster.go | 15 +- stores/bench_test.go | 2 +- stores/settings.go | 159 +----------------- stores/sql/mysql/main.go | 10 +- stores/sql/mysql/metrics.go | 2 +- .../main/migration_00017_settings.sql | 1 + stores/sql/sqlite/main.go | 10 +- stores/sql/sqlite/metrics.go | 2 +- .../main/migration_00017_settings.sql | 1 + stores/sql_test.go | 13 +- 14 files changed, 201 insertions(+), 193 deletions(-) create mode 100644 stores/sql/mysql/migrations/main/migration_00017_settings.sql create mode 100644 stores/sql/sqlite/migrations/main/migration_00017_settings.sql diff --git a/api/setting.go b/api/setting.go index 668acb243..bf97581b4 100644 --- a/api/setting.go +++ b/api/setting.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "go.sia.tech/core/consensus" rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" ) @@ -66,16 +65,20 @@ var ( // DefaultUploadSettings define the default upload settings the bus is // configured with on startup. -func DefaultUploadSettings(network *consensus.Network) UploadSettings { +func DefaultUploadSettings(network string) UploadSettings { + rs := RedundancySettings{ + MinShards: 10, + TotalShards: 30, + } + if network != "mainnet" { + rs = DefaultRedundancySettingsTestnet + } return UploadSettings{ Packing: UploadPackingSettings{ Enabled: true, SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB }, - Redundancy: RedundancySettings{ - MinShards: 10, - TotalShards: 30, - }, + Redundancy: rs, } } diff --git a/bus/bus.go b/bus/bus.go index a5e575d44..d1d7b6dbd 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -299,8 +299,6 @@ type ( S3Settings(ctx context.Context) (api.S3Settings, error) UpdateS3Settings(ctx context.Context, s3as api.S3Settings) error - - MigrateV2Settings(ctx context.Context) error } WalletMetricsRecorder interface { @@ -365,11 +363,6 @@ func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksMa rhp3: rhp3.New(rhp.NewFallbackDialer(store, net.Dialer{}, l), l), } - // migrate settings store - if err := store.MigrateV2Settings(ctx); err != nil { - return nil, err - } - // create contract locker b.contractLocker = ibus.NewContractLocker() diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 6721408ca..9af925bf9 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -491,11 +491,11 @@ func buildStoreConfig(am alerts.Alerter, cfg config.Config, pk types.PrivateKey, if err != nil { return stores.Config{}, fmt.Errorf("failed to open MySQL metrics database: %w", err) } - dbMain, err = mysql.NewMainDatabase(connMain, logger, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + dbMain, err = mysql.NewMainDatabase(connMain, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold, cfg.Network, logger) if err != nil { return stores.Config{}, fmt.Errorf("failed to create MySQL main database: %w", err) } - dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold, logger) if err != nil { return stores.Config{}, fmt.Errorf("failed to create MySQL metrics database: %w", err) } @@ -511,7 +511,7 @@ func buildStoreConfig(am alerts.Alerter, cfg config.Config, pk types.PrivateKey, if err != nil { return stores.Config{}, fmt.Errorf("failed to open SQLite main database: %w", err) } - dbMain, err = sqlite.NewMainDatabase(db, logger, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + dbMain, err = sqlite.NewMainDatabase(db, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold, cfg.Network, logger) if err != nil { return stores.Config{}, fmt.Errorf("failed to create SQLite main database: %w", err) } @@ -520,7 +520,7 @@ func buildStoreConfig(am alerts.Alerter, cfg config.Config, pk types.PrivateKey, if err != nil { return stores.Config{}, fmt.Errorf("failed to open SQLite metrics database: %w", err) } - dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + dbMetrics, err = sqlite.NewMetricsDatabase(dbm, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold, logger) if err != nil { return stores.Config{}, fmt.Errorf("failed to create SQLite metrics database: %w", err) } diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index 9b98be300..a7368ba72 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -3,10 +3,13 @@ package sql import ( "context" "embed" + "encoding/json" "fmt" "strings" + "time" "unicode/utf8" + "go.sia.tech/renterd/api" "go.sia.tech/renterd/internal/utils" "go.uber.org/zap" ) @@ -28,6 +31,7 @@ type ( MainMigrator interface { Migrator MakeDirsForPath(ctx context.Context, tx Tx, path string) (int64, error) + Network() string } ) @@ -211,6 +215,151 @@ var ( return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00016_account_owner", log) }, }, + { + ID: "00017_settings", + Migrate: func(tx Tx) error { + log.Infof("performing %s migration '00017_settings'", dbIdentifier) + + // fetch all settings + rows, err := tx.Query(ctx, "SELECT key, value FROM settings") + if err != nil { + return fmt.Errorf("failed to fetch settings: %v", err) + } + defer rows.Close() + + settings := make(map[string]string) + for rows.Next() { + var k, v string + if err := rows.Scan(&k, &v); err != nil { + _ = rows.Close() + return fmt.Errorf("failed to scan setting: %v", err) + } + settings[k] = v + } + + // migrate gouging settings + if v, ok := settings["gouging"]; ok { + var gs api.GougingSettings + err := json.Unmarshal([]byte(v), &gs) + if err == nil { + err = gs.Validate() + } + if err != nil { + log.Warnf("gouging settings are not being migrated, err: %v", err) + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "gouging"); err != nil { + return fmt.Errorf("failed to delete gouging settings: %v", err) + } + } + } else { + log.Warn("no pricepinning settings found") + } + + // migrate pinning settings + if v, ok := settings["pricepinning"]; ok { + var ps api.PinningSettings + err := json.Unmarshal([]byte(v), &ps) + if err == nil { + err = ps.Validate() + } + if err != nil { + log.Warnf("pricepinning settings are not being migrated, err: %v", err) + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "pricepinning"); err != nil { + return fmt.Errorf("failed to delete pricepinning settings: %v", err) + } + } else { + b, _ := json.Marshal(ps) + if _, err := tx.Exec(ctx, "UPDATE settings SET value = ? WHERE `key` = ?", string(b), "pricepinning"); err != nil { + return fmt.Errorf("failed to update pricepinning settings: %v", err) + } + } + } else { + log.Warn("no pricepinning settings found") + } + + // migrate S3 authentication settings + if v, ok := settings["s3authentication"]; ok { + var s3s api.S3Settings + err := json.Unmarshal([]byte(v), &s3s.Authentication) + if err == nil { + err = s3s.Validate() + } + if err == nil { + b, _ := json.Marshal(s3s) + if _, err := tx.Exec(ctx, "INSERT INTO settings (created_at, `key`, value) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE value = VALUES(value)", + time.Now(), "s3", string(b)); err != nil { + return fmt.Errorf("failed to insert s3 settings: %v", err) + } + } else { + log.Warnf("s3authentication settings are not being migrated, err: %v", err) + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "s3authentication"); err != nil { + log.Warnf("failed to delete s3authentication settings: %v", err) + } + } + } else { + log.Warn("no s3authentication setting found") + } + + // migrate upload settings + us := api.DefaultUploadSettings(m.Network()) + + if v, ok := settings["contractset"]; ok { + var css struct { + Default string `json:"default"` + } + if err := json.Unmarshal([]byte(v), &css); err != nil { + log.Warnf("contractset settings are not being migrated, err: %v", err) + } else { + us.DefaultContractSet = css.Default + } + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "contractset"); err != nil { + return err + } + } + + if v, ok := settings["uploadpacking"]; ok { + var ups api.UploadPackingSettings + if err := json.Unmarshal([]byte(v), &ups); err != nil { + log.Warnf("uploadpacking settings are not being migrated, err: %v", err) + } else { + us.Packing = ups + } + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "uploadpacking"); err != nil { + return err + } + } + + if v, ok := settings["redundancy"]; ok { + var rs api.RedundancySettings + err := json.Unmarshal([]byte(v), &rs) + if err == nil { + err = rs.Validate() + } + if err != nil { + log.Warnf("redundancy settings are not being migrated, err: %v", err) + } else { + us.Redundancy = rs + } + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "redundancy"); err != nil { + return err + } + } + + // update upload settings + if err := us.Validate(); err != nil { + log.Warnf("upload settings are not being migrated, err: %v", err) + return err // developer error + } else { + b, _ := json.Marshal(us) + if _, err := tx.Exec(ctx, "INSERT INTO settings (created_at, `key`, value) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE value = VALUES(value)", + time.Now(), "upload", string(b)); err != nil { + return fmt.Errorf("failed to insert s3 settings: %v", err) + } + } + + log.Info("migration '00017_settings' complete") + return nil + }, + }, } } MetricsMigrations = func(ctx context.Context, migrationsFs embed.FS, log *zap.SugaredLogger) []Migration { diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 9f2b2c0ac..2af725d02 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -545,14 +545,15 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { } func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, pk types.PrivateKey, logger *zap.Logger) (*bus.Bus, func(ctx context.Context) error, *chain.Manager, bus.Store, error) { + network, genesis := testNetwork() + // create store alertsMgr := alerts.NewManager() - storeCfg, err := buildStoreConfig(alertsMgr, dir, cfg.SlabBufferCompletionThreshold, cfgDb, pk, logger) + storeCfg, err := buildStoreConfig(alertsMgr, dir, network.Name, cfg.SlabBufferCompletionThreshold, cfgDb, pk, logger) if err != nil { return nil, nil, nil, nil, err } - network, genesis := testNetwork() sqlStore, err := stores.NewSQLStore(storeCfg, network) if err != nil { return nil, nil, nil, nil, err @@ -1119,7 +1120,7 @@ func testApCfg() config.Autopilot { } } -func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThreshold int64, cfg dbConfig, pk types.PrivateKey, logger *zap.Logger) (stores.Config, error) { +func buildStoreConfig(am alerts.Alerter, dir, network string, slabBufferCompletionThreshold int64, cfg dbConfig, pk types.PrivateKey, logger *zap.Logger) (stores.Config, error) { // create database connections var dbMain sql.Database var dbMetrics sql.MetricsDatabase @@ -1143,11 +1144,11 @@ func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThresho if err != nil { return stores.Config{}, fmt.Errorf("failed to open MySQL metrics database: %w", err) } - dbMain, err = mysql.NewMainDatabase(connMain, logger, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + dbMain, err = mysql.NewMainDatabase(connMain, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold, network, logger) if err != nil { return stores.Config{}, fmt.Errorf("failed to create MySQL main database: %w", err) } - dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold, logger) if err != nil { return stores.Config{}, fmt.Errorf("failed to create MySQL metrics database: %w", err) } @@ -1163,7 +1164,7 @@ func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThresho if err != nil { return stores.Config{}, fmt.Errorf("failed to open SQLite main database: %w", err) } - dbMain, err = sqlite.NewMainDatabase(db, logger, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + dbMain, err = sqlite.NewMainDatabase(db, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold, network, logger) if err != nil { return stores.Config{}, fmt.Errorf("failed to create SQLite main database: %w", err) } @@ -1172,7 +1173,7 @@ func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThresho if err != nil { return stores.Config{}, fmt.Errorf("failed to open SQLite metrics database: %w", err) } - dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + dbMetrics, err = sqlite.NewMetricsDatabase(dbm, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold, logger) if err != nil { return stores.Config{}, fmt.Errorf("failed to create SQLite metrics database: %w", err) } diff --git a/stores/bench_test.go b/stores/bench_test.go index 60f75b52f..65e58caa5 100644 --- a/stores/bench_test.go +++ b/stores/bench_test.go @@ -153,7 +153,7 @@ func newTestDB(ctx context.Context, dir string) (*sqlite.MainDatabase, error) { return nil, err } - dbMain, err := sqlite.NewMainDatabase(db, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) + dbMain, err := sqlite.NewMainDatabase(db, 100*time.Millisecond, 100*time.Millisecond, "mainnet", zap.NewNop()) if err != nil { return nil, err } diff --git a/stores/settings.go b/stores/settings.go index 843d8e986..ac3c4d312 100644 --- a/stores/settings.go +++ b/stores/settings.go @@ -8,7 +8,6 @@ import ( "go.sia.tech/renterd/api" sql "go.sia.tech/renterd/stores/sql" - "go.uber.org/zap" ) const ( @@ -54,162 +53,6 @@ func (s *SQLStore) UpdateS3Settings(ctx context.Context, ss api.S3Settings) erro return s.updateSetting(ctx, SettingS3, ss) } -// MigrateV2Settings migrates the settings from the old format to the new, -// migrating the existing settings over to the new types and removing the old -// settings. If a setting is not present in the database it will be set to its -// default setting. If an existing setting is not valid, the default will be -// used and a warning will get logged. -func (s *SQLStore) MigrateV2Settings(ctx context.Context) error { - return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - // escape early if none of the old settings are present - var found bool - for _, key := range []string{ - "pricepinning", - "s3authentication", - "contractset", - "redundancy", - "uploadpacking", - } { - if _, err := tx.Setting(ctx, key); err != nil && !errors.Is(err, sql.ErrSettingNotFound) { - return err - } else if err == nil { - found = true - break - } - } - if !found { - return nil - } - - s.logger.Info("migrating settings...") - - // migrate gouging settings - value, err := tx.Setting(ctx, "gouging") - if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { - return err - } else if errors.Is(err, sql.ErrSettingNotFound) { - if err := tx.UpdateSetting(ctx, SettingGouging, s.defaultSetting(SettingGouging)); err != nil { - return err - } - } - - // migrate pinning settings - value, err = tx.Setting(ctx, "pricepinning") - if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { - return err - } else if err == nil { - var ps api.PinningSettings - if err := json.Unmarshal([]byte(value), &ps); err != nil { - s.logger.Warnw("failed to unmarshal pinning settings, using default", zap.Error(err)) - value = s.defaultSetting(SettingPricePinning) - } else if err := ps.Validate(); err != nil { - s.logger.Warnw("failed to migrate pinning settings, using default", zap.Error(err)) - value = s.defaultSetting(SettingPricePinning) - } - - // update setting and delete old value - if err := tx.UpdateSetting(ctx, SettingPricePinning, value); err != nil { - return err - } else if err := tx.DeleteSetting(ctx, "pricepinning"); err != nil { - return err - } - } - - // migrate s3 settings - value, err = tx.Setting(ctx, "s3authentication") - if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { - return err - } else if err == nil { - var s3s api.S3Settings - if err := json.Unmarshal([]byte(value), &s3s.Authentication); err != nil { - s.logger.Warnw("failed to unmarshal S3 authentication settings, using default", zap.Error(err)) - s3s = api.DefaultS3Settings - } else if err := s3s.Validate(); err != nil { - s.logger.Warnw("failed to migrate S3 settings, using default", zap.Error(err)) - s3s = api.DefaultS3Settings - } - - // update setting and delete old value - update, _ := json.Marshal(s3s) - if err := tx.UpdateSetting(ctx, SettingS3, string(update)); err != nil { - return err - } else if err := tx.DeleteSetting(ctx, "s3authentication"); err != nil { - return err - } - } - - us := api.DefaultUploadSettings(s.network) - - // migrate contractset settings - value, err = tx.Setting(ctx, "contractset") - if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { - return err - } else if err == nil { - var css struct { - Default string `json:"default"` - } - if err := json.Unmarshal([]byte(value), &css); err != nil { - s.logger.Warnw("failed to unmarshal contractset setting, using default", zap.Error(err)) - } else { - us.DefaultContractSet = css.Default - } - - // delete old value - if err := tx.DeleteSetting(ctx, "contractset"); err != nil { - return err - } - } - - // migrate redundancy settings - value, err = tx.Setting(ctx, "redundancy") - if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { - return err - } else if err == nil { - var rs api.RedundancySettings - if err := json.Unmarshal([]byte(value), &rs); err != nil { - s.logger.Warnw("failed to unmarshal redundancy settings, using default", zap.Error(err)) - } else if err := rs.Validate(); err != nil { - s.logger.Warnw("failed to migrate redundancy settings, using default", zap.Error(err)) - } else { - us.Redundancy = rs - } - - // delete old value - if err := tx.DeleteSetting(ctx, "redundancy"); err != nil { - return err - } - } - - // migrate uploadpacking settings - value, err = tx.Setting(ctx, "uploadpacking") - if err != nil && !errors.Is(err, sql.ErrSettingNotFound) { - return err - } else if err == nil { - var ups api.UploadPackingSettings - if err := json.Unmarshal([]byte(value), &ups); err != nil { - s.logger.Warnw("failed to unmarshal uploadpacking settings, using default", zap.Error(err)) - } else { - us.Packing = ups - } - - // delete old value - if err := tx.DeleteSetting(ctx, "uploadpacking"); err != nil { - return err - } - } - - // update upload settings - if update, err := json.Marshal(us); err != nil { - return fmt.Errorf("failed to marshal upload settings: %w", err) - } else if err := tx.UpdateSetting(ctx, SettingUpload, string(update)); err != nil { - return err - } - - s.logger.Info("successfully migrated settings") - return nil - }) -} - func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{}) error { s.settingsMu.Lock() defer s.settingsMu.Unlock() @@ -273,7 +116,7 @@ func (s *SQLStore) defaultSetting(key string) string { b, _ := json.Marshal(api.DefaultS3Settings) return string(b) case SettingUpload: - b, _ := json.Marshal(api.DefaultUploadSettings(s.network)) + b, _ := json.Marshal(api.DefaultUploadSettings(s.network.Name)) return string(b) default: panic("unknown setting") // developer error diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index af47d7a22..12e3fba14 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -32,6 +32,8 @@ const ( type ( MainDatabase struct { + network string + db *sql.DB log *zap.SugaredLogger } @@ -43,10 +45,12 @@ type ( ) // NewMainDatabase creates a new MySQL backend. -func NewMainDatabase(db *dsql.DB, log *zap.Logger, lqd, ltd time.Duration) (*MainDatabase, error) { +func NewMainDatabase(db *dsql.DB, lqd, ltd time.Duration, network string, log *zap.Logger) (*MainDatabase, error) { log = log.Named("main") store, err := sql.NewDB(db, log, deadlockMsgs, lqd, ltd) return &MainDatabase{ + network: network, + db: store, log: log.Sugar(), }, err @@ -81,6 +85,10 @@ func (b *MainDatabase) Migrate(ctx context.Context) error { return sql.PerformMigrations(ctx, b, migrationsFs, "main", sql.MainMigrations(ctx, b, migrationsFs, b.log)) } +func (b *MainDatabase) Network() string { + return b.network +} + func (b *MainDatabase) Transaction(ctx context.Context, fn func(tx ssql.DatabaseTx) error) error { return b.db.Transaction(ctx, func(tx sql.Tx) error { return fn(b.wrapTxn(tx)) diff --git a/stores/sql/mysql/metrics.go b/stores/sql/mysql/metrics.go index e7ef23813..3f642d878 100644 --- a/stores/sql/mysql/metrics.go +++ b/stores/sql/mysql/metrics.go @@ -30,7 +30,7 @@ type ( var _ ssql.MetricsDatabaseTx = (*MetricsDatabaseTx)(nil) // NewMetricsDatabase creates a new MySQL backend. -func NewMetricsDatabase(db *dsql.DB, log *zap.Logger, lqd, ltd time.Duration) (*MetricsDatabase, error) { +func NewMetricsDatabase(db *dsql.DB, lqd, ltd time.Duration, log *zap.Logger) (*MetricsDatabase, error) { log = log.Named("metrics") store, err := sql.NewDB(db, log, deadlockMsgs, lqd, ltd) return &MetricsDatabase{ diff --git a/stores/sql/mysql/migrations/main/migration_00017_settings.sql b/stores/sql/mysql/migrations/main/migration_00017_settings.sql new file mode 100644 index 000000000..09a7b0592 --- /dev/null +++ b/stores/sql/mysql/migrations/main/migration_00017_settings.sql @@ -0,0 +1 @@ +-- placeholder diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index b8bfe2771..d6ceafe6d 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -31,6 +31,8 @@ const ( type ( MainDatabase struct { + network string + db *sql.DB log *zap.SugaredLogger } @@ -42,10 +44,12 @@ type ( ) // NewMainDatabase creates a new SQLite backend. -func NewMainDatabase(db *dsql.DB, log *zap.Logger, lqd, ltd time.Duration) (*MainDatabase, error) { +func NewMainDatabase(db *dsql.DB, lqd, ltd time.Duration, network string, log *zap.Logger) (*MainDatabase, error) { log = log.Named("main") store, err := sql.NewDB(db, log, deadlockMsgs, lqd, ltd) return &MainDatabase{ + network: network, + db: store, log: log.Sugar(), }, err @@ -80,6 +84,10 @@ func (b *MainDatabase) Migrate(ctx context.Context) error { return sql.PerformMigrations(ctx, b, migrationsFs, "main", sql.MainMigrations(ctx, b, migrationsFs, b.log)) } +func (b *MainDatabase) Network() string { + return b.network +} + func (b *MainDatabase) Transaction(ctx context.Context, fn func(tx ssql.DatabaseTx) error) error { return b.db.Transaction(ctx, func(tx sql.Tx) error { return fn(b.wrapTxn(tx)) diff --git a/stores/sql/sqlite/metrics.go b/stores/sql/sqlite/metrics.go index df912d7c7..fd46c1222 100644 --- a/stores/sql/sqlite/metrics.go +++ b/stores/sql/sqlite/metrics.go @@ -29,7 +29,7 @@ type ( var _ ssql.MetricsDatabaseTx = (*MetricsDatabaseTx)(nil) // NewSQLiteDatabase creates a new SQLite backend. -func NewMetricsDatabase(db *dsql.DB, log *zap.Logger, lqd, ltd time.Duration) (*MetricsDatabase, error) { +func NewMetricsDatabase(db *dsql.DB, lqd, ltd time.Duration, log *zap.Logger) (*MetricsDatabase, error) { log = log.Named("metrics") store, err := sql.NewDB(db, log, deadlockMsgs, lqd, ltd) return &MetricsDatabase{ diff --git a/stores/sql/sqlite/migrations/main/migration_00017_settings.sql b/stores/sql/sqlite/migrations/main/migration_00017_settings.sql new file mode 100644 index 000000000..09a7b0592 --- /dev/null +++ b/stores/sql/sqlite/migrations/main/migration_00017_settings.sql @@ -0,0 +1 @@ +-- placeholder diff --git a/stores/sql_test.go b/stores/sql_test.go index 48fb7a6a9..fba4b8327 100644 --- a/stores/sql_test.go +++ b/stores/sql_test.go @@ -25,6 +25,7 @@ import ( ) const ( + testNetwork = "zen" testContractSet = "test" testMimeType = "application/octet-stream" testETag = "d34db33f" @@ -97,11 +98,11 @@ func (cfg *testSQLStoreConfig) dbConnections() (sql.Database, sql.MetricsDatabas if err != nil { return nil, nil, fmt.Errorf("failed to open MySQL metrics database: %w", err) } - dbMain, err = mysql.NewMainDatabase(connMain, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) + dbMain, err = mysql.NewMainDatabase(connMain, 100*time.Millisecond, 100*time.Millisecond, testNetwork, zap.NewNop()) if err != nil { return nil, nil, fmt.Errorf("failed to create MySQL main database: %w", err) } - dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) + dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, 100*time.Millisecond, 100*time.Millisecond, zap.NewNop()) if err != nil { return nil, nil, fmt.Errorf("failed to create MySQL metrics database: %w", err) } @@ -115,11 +116,11 @@ func (cfg *testSQLStoreConfig) dbConnections() (sql.Database, sql.MetricsDatabas if err != nil { return nil, nil, fmt.Errorf("failed to open SQLite metrics database: %w", err) } - dbMain, err = sqlite.NewMainDatabase(connMain, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) + dbMain, err = sqlite.NewMainDatabase(connMain, 100*time.Millisecond, 100*time.Millisecond, testNetwork, zap.NewNop()) if err != nil { return nil, nil, fmt.Errorf("failed to create SQLite main database: %w", err) } - dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) + dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, 100*time.Millisecond, 100*time.Millisecond, zap.NewNop()) if err != nil { return nil, nil, fmt.Errorf("failed to create SQLite metrics database: %w", err) } @@ -133,11 +134,11 @@ func (cfg *testSQLStoreConfig) dbConnections() (sql.Database, sql.MetricsDatabas if err != nil { return nil, nil, fmt.Errorf("failed to open ephemeral SQLite metrics database: %w", err) } - dbMain, err = sqlite.NewMainDatabase(connMain, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) + dbMain, err = sqlite.NewMainDatabase(connMain, 100*time.Millisecond, 100*time.Millisecond, testNetwork, zap.NewNop()) if err != nil { return nil, nil, fmt.Errorf("failed to create ephemeral SQLite main database: %w", err) } - dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) + dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, 100*time.Millisecond, 100*time.Millisecond, zap.NewNop()) if err != nil { return nil, nil, fmt.Errorf("failed to create ephemeral SQLite metrics database: %w", err) } From 829e6bf4fd3d51c688547373ebf23dbf674cf4d3 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 6 Sep 2024 12:04:12 +0200 Subject: [PATCH 15/33] sql: fix migration --- internal/sql/migrations.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index aeedc3d54..b6bc8b435 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -226,7 +226,7 @@ var ( log.Infof("performing %s migration '00018_settings'", dbIdentifier) // fetch all settings - rows, err := tx.Query(ctx, "SELECT key, value FROM settings") + rows, err := tx.Query(ctx, "SELECT `key`, value FROM settings") if err != nil { return fmt.Errorf("failed to fetch settings: %v", err) } @@ -297,9 +297,9 @@ var ( } } else { log.Warnf("s3authentication settings are not being migrated, err: %v", err) - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "s3authentication"); err != nil { - log.Warnf("failed to delete s3authentication settings: %v", err) - } + } + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "s3authentication"); err != nil { + log.Warnf("failed to delete s3authentication settings: %v", err) } } else { log.Warn("no s3authentication setting found") From 8aa18a03b504b9d3cdc899b61a83f1f5b51fbf9d Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 6 Sep 2024 12:30:38 +0200 Subject: [PATCH 16/33] db: update migration --- internal/sql/migrations.go | 42 ++++++++++++++++++++++---------------- stores/sql/mysql/main.go | 5 +++++ stores/sql/sqlite/main.go | 5 +++++ 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index b6bc8b435..12e21cb4b 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "strings" - "time" "unicode/utf8" "go.sia.tech/renterd/api" @@ -31,6 +30,7 @@ type ( MainMigrator interface { Migrator MakeDirsForPath(ctx context.Context, tx Tx, path string) (int64, error) + UpdateSetting(ctx context.Context, tx Tx, key, value string) error } ) @@ -236,7 +236,6 @@ var ( for rows.Next() { var k, v string if err := rows.Scan(&k, &v); err != nil { - _ = rows.Close() return fmt.Errorf("failed to scan setting: %v", err) } settings[k] = v @@ -266,17 +265,18 @@ var ( if err == nil { err = ps.Validate() } - if err != nil { - log.Warnf("pricepinning settings are not being migrated, err: %v", err) - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "pricepinning"); err != nil { - return fmt.Errorf("failed to delete pricepinning settings: %v", err) - } - } else { - b, _ := json.Marshal(ps) - if _, err := tx.Exec(ctx, "INSERT INTO settings (created_at, `key`, value) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE value = VALUES(value)", - time.Now(), "pinned", string(b)); err != nil { + if err == nil { + updated, _ := json.Marshal(ps) + if err := m.UpdateSetting(ctx, tx, "pinned", string(updated)); err != nil { return fmt.Errorf("failed to insert pinned settings: %v", err) } + } else { + log.Warnf("pricepinning settings are not being migrated, err: %v", err) + } + + // always delete because it got renamed + if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "pricepinning"); err != nil { + log.Warnf("failed to delete pricepinning settings: %v", err) } } else { log.Warn("no pricepinning settings found") @@ -290,14 +290,15 @@ var ( err = s3s.Validate() } if err == nil { - b, _ := json.Marshal(s3s) - if _, err := tx.Exec(ctx, "INSERT INTO settings (created_at, `key`, value) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE value = VALUES(value)", - time.Now(), "s3", string(b)); err != nil { + updated, _ := json.Marshal(s3s) + if err := m.UpdateSetting(ctx, tx, "s3", string(updated)); err != nil { return fmt.Errorf("failed to insert s3 settings: %v", err) } } else { log.Warnf("s3authentication settings are not being migrated, err: %v", err) } + + // always delete because it got renamed if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "s3authentication"); err != nil { log.Warnf("failed to delete s3authentication settings: %v", err) } @@ -317,6 +318,8 @@ var ( } else { us.DefaultContractSet = css.Default } + + // always delete because it got replaced if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "contractset"); err != nil { return err } @@ -329,6 +332,8 @@ var ( } else { us.Packing = ups } + + // always delete because it got replaced if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "uploadpacking"); err != nil { return err } @@ -345,6 +350,8 @@ var ( } else { us.Redundancy = rs } + + // always delete because it got replaced if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "redundancy"); err != nil { return err } @@ -355,10 +362,9 @@ var ( log.Warnf("upload settings are not being migrated, err: %v", err) return err // developer error } else { - b, _ := json.Marshal(us) - if _, err := tx.Exec(ctx, "INSERT INTO settings (created_at, `key`, value) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE value = VALUES(value)", - time.Now(), "upload", string(b)); err != nil { - return fmt.Errorf("failed to insert s3 settings: %v", err) + updated, _ := json.Marshal(us) + if err := m.UpdateSetting(ctx, tx, "upload", string(updated)); err != nil { + return fmt.Errorf("failed to insert upload settings: %v", err) } } diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index 457db78b7..71625a343 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -87,6 +87,11 @@ func (b *MainDatabase) Transaction(ctx context.Context, fn func(tx ssql.Database }) } +func (b *MainDatabase) UpdateSetting(ctx context.Context, tx sql.Tx, key, value string) error { + mtx := b.wrapTxn(tx) + return mtx.UpdateSetting(ctx, key, value) +} + func (b *MainDatabase) Version(ctx context.Context) (string, string, error) { return version(ctx, b.db) } diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 7adaaaeab..f7e839a4a 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -86,6 +86,11 @@ func (b *MainDatabase) Transaction(ctx context.Context, fn func(tx ssql.Database }) } +func (b *MainDatabase) UpdateSetting(ctx context.Context, tx sql.Tx, key, value string) error { + mtx := b.wrapTxn(tx) + return mtx.UpdateSetting(ctx, key, value) +} + func (b *MainDatabase) Version(ctx context.Context) (string, string, error) { return version(ctx, b.db) } From ca1c0776c729f3b8ddd5912f7bbcdfdf83db650a Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 6 Sep 2024 15:13:56 +0200 Subject: [PATCH 17/33] stores: use bool --- cmd/renterd/node.go | 14 +++++++------- internal/test/e2e/cluster.go | 2 +- stores/settings.go | 2 +- stores/sql.go | 16 ++++++++-------- stores/sql_test.go | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 5787bae7f..58e96c9c8 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -258,19 +258,13 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) } func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network *consensus.Network, genesis types.Block, logger *zap.Logger) (*bus.Bus, func(ctx context.Context) error, error) { - // get explorer URL - var explorerURL string - if !cfg.Explorer.Disable { - explorerURL = cfg.Explorer.URL - } - // create store alertsMgr := alerts.NewManager() storeCfg, err := buildStoreConfig(alertsMgr, cfg, pk, logger) if err != nil { return nil, nil, err } - sqlStore, err := stores.NewSQLStore(storeCfg, explorerURL, network) + sqlStore, err := stores.NewSQLStore(storeCfg, cfg.Explorer.Disable, network) if err != nil { return nil, nil, err } @@ -384,6 +378,12 @@ func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network // to ensure contracts formed by the bus can be renewed by the autopilot masterKey := blake2b.Sum256(append([]byte("worker"), pk...)) + // get explorer URL + var explorerURL string + if !cfg.Explorer.Disable { + explorerURL = cfg.Explorer.URL + } + // create bus announcementMaxAgeHours := time.Duration(cfg.Bus.AnnouncementMaxAgeHours) * time.Hour b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, explorerURL, logger) diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 3a75a7ee7..13c66520b 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -554,7 +554,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, // create store network, genesis := testNetwork() - sqlStore, err := stores.NewSQLStore(storeCfg, "", network) + sqlStore, err := stores.NewSQLStore(storeCfg, true, network) if err != nil { return nil, nil, nil, nil, err } diff --git a/stores/settings.go b/stores/settings.go index 3a24cb5bc..fab149a11 100644 --- a/stores/settings.go +++ b/stores/settings.go @@ -76,7 +76,7 @@ func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{} return fmt.Errorf("failed to fetch setting from db: %w", err) } else if err != nil { value = s.defaultSetting(key) - } else if key == SettingPinned && s.explorerURL == "" { + } else if key == SettingPinned && !s.explorerDisabled { var ps api.PinnedSettings if err := json.Unmarshal([]byte(value), &ps); err == nil && ps.Enabled() { s.logger.Warn("pinned settings are enabled but explorer is disabled, using default settings") diff --git a/stores/sql.go b/stores/sql.go index 5a3c188fb..dee0a416f 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -43,9 +43,9 @@ type ( dbMetrics sql.MetricsDatabase logger *zap.SugaredLogger - explorerURL string - network *consensus.Network - walletAddress types.Address + explorerDisabled bool + network *consensus.Network + walletAddress types.Address // ObjectDB related fields slabBufferMgr *SlabBufferManager @@ -71,7 +71,7 @@ type ( // NewSQLStore uses a given Dialector to connect to a SQL database. NOTE: Only // pass migrate=true for the first instance of SQLHostDB if you connect via the // same Dialector multiple times. -func NewSQLStore(cfg Config, explorerURL string, network *consensus.Network) (*SQLStore, error) { +func NewSQLStore(cfg Config, explorerDisabled bool, network *consensus.Network) (*SQLStore, error) { if err := os.MkdirAll(cfg.PartialSlabDir, 0700); err != nil { return nil, fmt.Errorf("failed to create partial slab dir '%s': %v", cfg.PartialSlabDir, err) } @@ -102,10 +102,10 @@ func NewSQLStore(cfg Config, explorerURL string, network *consensus.Network) (*S dbMetrics: dbMetrics, logger: l.Sugar(), - settings: make(map[string]string), - walletAddress: cfg.WalletAddress, - explorerURL: explorerURL, - network: network, + settings: make(map[string]string), + walletAddress: cfg.WalletAddress, + explorerDisabled: explorerDisabled, + network: network, slabPruneSigChan: make(chan struct{}, 1), lastPrunedAt: time.Now(), diff --git a/stores/sql_test.go b/stores/sql_test.go index 1b2689f03..7a676a5b3 100644 --- a/stores/sql_test.go +++ b/stores/sql_test.go @@ -180,7 +180,7 @@ func newTestSQLStore(t *testing.T, cfg testSQLStoreConfig) *testSQLStore { LongQueryDuration: 100 * time.Millisecond, LongTxDuration: 100 * time.Millisecond, RetryTransactionIntervals: []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond}, - }, "", &consensus.Network{}) + }, false, &consensus.Network{}) if err != nil { t.Fatal("failed to create SQLStore", err) } From 151fb9ce48abe7bc1b1d2330cd6de13c6b8d738a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 01:19:31 +0000 Subject: [PATCH 18/33] build(deps): bump the all-dependencies group with 4 updates Bumps the all-dependencies group with 4 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), [golang.org/x/crypto](https://github.com/golang/crypto), [golang.org/x/sys](https://github.com/golang/sys) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.22 to 1.14.23 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.22...v1.14.23) Updates `golang.org/x/crypto` from 0.26.0 to 0.27.0 - [Commits](https://github.com/golang/crypto/compare/v0.26.0...v0.27.0) Updates `golang.org/x/sys` from 0.24.0 to 0.25.0 - [Commits](https://github.com/golang/sys/compare/v0.24.0...v0.25.0) Updates `golang.org/x/term` from 0.23.0 to 0.24.0 - [Commits](https://github.com/golang/term/compare/v0.23.0...v0.24.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 77870cda4..a422773e6 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/gotd/contrib v0.20.0 github.com/klauspost/reedsolomon v1.12.3 - github.com/mattn/go-sqlite3 v1.14.22 + github.com/mattn/go-sqlite3 v1.14.23 github.com/minio/minio-go/v7 v7.0.76 github.com/montanaflynn/stats v0.7.1 github.com/shopspring/decimal v1.4.0 @@ -21,9 +21,9 @@ require ( go.sia.tech/mux v1.2.0 go.sia.tech/web/renterd v0.60.1 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.26.0 - golang.org/x/sys v0.24.0 - golang.org/x/term v0.23.0 + golang.org/x/crypto v0.27.0 + golang.org/x/sys v0.25.0 + golang.org/x/term v0.24.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/frand v1.4.2 ) @@ -50,7 +50,7 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.28.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.22.0 // indirect nhooyr.io/websocket v1.8.17 // indirect diff --git a/go.sum b/go.sum index f393bfbd4..e2c613525 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= +github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.76 h1:9nxHH2XDai61cT/EFhyIw/wW4vJfpPNvl7lSFpRt+Ng= @@ -95,8 +95,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -108,13 +108,13 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From fe695fad735c7bca38c75033a4007f4d690d83d9 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 9 Sep 2024 09:28:00 +0200 Subject: [PATCH 19/33] stores: remove explorer enabled check --- cmd/renterd/node.go | 2 +- internal/test/e2e/cluster.go | 2 +- stores/settings.go | 6 ------ stores/sql.go | 14 ++++++-------- stores/sql_test.go | 2 +- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 58e96c9c8..92c62ae9f 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -264,7 +264,7 @@ func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network if err != nil { return nil, nil, err } - sqlStore, err := stores.NewSQLStore(storeCfg, cfg.Explorer.Disable, network) + sqlStore, err := stores.NewSQLStore(storeCfg, network) if err != nil { return nil, nil, err } diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 13c66520b..ec07a465a 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -554,7 +554,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, // create store network, genesis := testNetwork() - sqlStore, err := stores.NewSQLStore(storeCfg, true, network) + sqlStore, err := stores.NewSQLStore(storeCfg, network) if err != nil { return nil, nil, nil, nil, err } diff --git a/stores/settings.go b/stores/settings.go index fab149a11..5801de1a9 100644 --- a/stores/settings.go +++ b/stores/settings.go @@ -76,12 +76,6 @@ func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{} return fmt.Errorf("failed to fetch setting from db: %w", err) } else if err != nil { value = s.defaultSetting(key) - } else if key == SettingPinned && !s.explorerDisabled { - var ps api.PinnedSettings - if err := json.Unmarshal([]byte(value), &ps); err == nil && ps.Enabled() { - s.logger.Warn("pinned settings are enabled but explorer is disabled, using default settings") - value = s.defaultSetting(key) - } } // unmarshal setting diff --git a/stores/sql.go b/stores/sql.go index dee0a416f..424016927 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -43,9 +43,8 @@ type ( dbMetrics sql.MetricsDatabase logger *zap.SugaredLogger - explorerDisabled bool - network *consensus.Network - walletAddress types.Address + network *consensus.Network + walletAddress types.Address // ObjectDB related fields slabBufferMgr *SlabBufferManager @@ -71,7 +70,7 @@ type ( // NewSQLStore uses a given Dialector to connect to a SQL database. NOTE: Only // pass migrate=true for the first instance of SQLHostDB if you connect via the // same Dialector multiple times. -func NewSQLStore(cfg Config, explorerDisabled bool, network *consensus.Network) (*SQLStore, error) { +func NewSQLStore(cfg Config, network *consensus.Network) (*SQLStore, error) { if err := os.MkdirAll(cfg.PartialSlabDir, 0700); err != nil { return nil, fmt.Errorf("failed to create partial slab dir '%s': %v", cfg.PartialSlabDir, err) } @@ -102,10 +101,9 @@ func NewSQLStore(cfg Config, explorerDisabled bool, network *consensus.Network) dbMetrics: dbMetrics, logger: l.Sugar(), - settings: make(map[string]string), - walletAddress: cfg.WalletAddress, - explorerDisabled: explorerDisabled, - network: network, + settings: make(map[string]string), + walletAddress: cfg.WalletAddress, + network: network, slabPruneSigChan: make(chan struct{}, 1), lastPrunedAt: time.Now(), diff --git a/stores/sql_test.go b/stores/sql_test.go index 7a676a5b3..0fd280873 100644 --- a/stores/sql_test.go +++ b/stores/sql_test.go @@ -180,7 +180,7 @@ func newTestSQLStore(t *testing.T, cfg testSQLStoreConfig) *testSQLStore { LongQueryDuration: 100 * time.Millisecond, LongTxDuration: 100 * time.Millisecond, RetryTransactionIntervals: []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond}, - }, false, &consensus.Network{}) + }, &consensus.Network{}) if err != nil { t.Fatal("failed to create SQLStore", err) } From 1d576bc52d4a3b6747ec49a2aa8de15b62a10b82 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 9 Sep 2024 10:53:11 +0200 Subject: [PATCH 20/33] testing: fix TestPinManager NDF --- internal/bus/pinmanager.go | 9 +++---- internal/bus/pinmanager_test.go | 44 ++++++++++++++++----------------- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index 4f38b8ba8..548e6b708 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -47,7 +47,7 @@ type ( updateInterval time.Duration rateWindow time.Duration - triggerChan chan struct{} + triggerChan chan bool closedChan chan struct{} wg sync.WaitGroup @@ -74,7 +74,7 @@ func NewPinManager(alerts alerts.Alerter, broadcaster webhooks.Broadcaster, e Ex updateInterval: updateInterval, rateWindow: rateWindow, - triggerChan: make(chan struct{}, 1), + triggerChan: make(chan bool, 1), closedChan: make(chan struct{}), } @@ -109,7 +109,7 @@ func (pm *pinManager) Shutdown(ctx context.Context) error { func (pm *pinManager) TriggerUpdate() { select { - case pm.triggerChan <- struct{}{}: + case pm.triggerChan <- true: default: } } @@ -174,8 +174,7 @@ func (pm *pinManager) run() { select { case <-pm.closedChan: return - case <-pm.triggerChan: - forced = true + case forced = <-pm.triggerChan: case <-t.C: } } diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index 8283144b7..d7800ed71 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -152,6 +152,9 @@ func (ms *mockPinStore) PinnedSettings(ctx context.Context) (api.PinnedSettings, } func (ms *mockPinStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { + ms.mu.Lock() + defer ms.mu.Unlock() + b, err := json.Marshal(ps) if err != nil { return err @@ -160,10 +163,7 @@ func (ms *mockPinStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedS if err := json.Unmarshal(b, &cloned); err != nil { return err } - ms.mu.Lock() ms.ps = cloned - ms.mu.Unlock() - time.Sleep(2 * testUpdateInterval) return nil } @@ -195,17 +195,11 @@ func TestPinManager(t *testing.T) { } }() - // define a small helper to fetch the price manager's rates - rates := func() []float64 { + // waitForUpdate waits for the price manager to update + waitForUpdate := func() { t.Helper() - pm.mu.Lock() - defer pm.mu.Unlock() - return pm.rates - } - - // assert price manager is disabled by default - if cnt := len(rates()); cnt != 0 { - t.Fatalf("expected no rates, got %d", cnt) + pm.triggerChan <- false + time.Sleep(testUpdateInterval) } // enable price pinning @@ -214,7 +208,7 @@ func TestPinManager(t *testing.T) { ps.Threshold = 0.5 s.UpdatePinnedSettings(context.Background(), ps) - // update exchange rate and fetch current gouging settings + // fetch current gouging settings gs, _ := s.GougingSettings(context.Background()) // configure all pins but disable them for now @@ -231,19 +225,20 @@ func TestPinManager(t *testing.T) { // enable the max download pin ps.GougingSettingsPins.MaxDownload.Pinned = true s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() - // adjust the rate - e.setRate(1.5) - time.Sleep(2 * testUpdateInterval) - - // at threshold of .5 the prices should not be updated + // assert prices are not updated if gss, _ := s.GougingSettings(context.Background()); !reflect.DeepEqual(gs, gss) { - t.Fatalf("expected gouging settings to be the same, got %v", gss) + t.Fatalf("expected gouging settings to be the same, got %v expected %v", gss, gs) } - // lower the threshold, gouging settings should be updated + // adjust and lower the threshold + e.setRate(1.5) ps.Threshold = 0.05 s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() + + // assert prices are updated if gss, _ := s.GougingSettings(context.Background()); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) { t.Fatalf("expected gouging settings to be updated, got %v = %v", gss.MaxDownloadPrice, gs.MaxDownloadPrice) } @@ -253,6 +248,7 @@ func TestPinManager(t *testing.T) { ps.GougingSettingsPins.MaxStorage.Pinned = true ps.GougingSettingsPins.MaxUpload.Pinned = true s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() // assert they're all updated if gss, _ := s.GougingSettings(context.Background()); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) || @@ -276,6 +272,7 @@ func TestPinManager(t *testing.T) { } ps.Autopilots = map[string]api.AutopilotPins{testAutopilotID: pins} s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() // assert autopilot was not updated if app, _ := s.Autopilot(context.Background(), testAutopilotID); !app.Config.Contracts.Allowance.Equals(ap.Config.Contracts.Allowance) { @@ -286,6 +283,7 @@ func TestPinManager(t *testing.T) { pins.Allowance.Pinned = true ps.Autopilots[testAutopilotID] = pins s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() // assert autopilot was updated if app, _ := s.Autopilot(context.Background(), testAutopilotID); app.Config.Contracts.Allowance.Equals(ap.Config.Contracts.Allowance) { @@ -294,9 +292,9 @@ func TestPinManager(t *testing.T) { // make explorer return an error e.setUnreachable(true) + waitForUpdate() // assert alert was registered - s.UpdatePinnedSettings(context.Background(), ps) res, _ := a.Alerts(context.Background(), alerts.AlertsOpts{}) if len(res.Alerts) == 0 { t.Fatalf("expected 1 alert, got %d", len(a.alerts)) @@ -304,9 +302,9 @@ func TestPinManager(t *testing.T) { // make explorer return a valid response e.setUnreachable(false) + waitForUpdate() // assert alert was dismissed - s.UpdatePinnedSettings(context.Background(), ps) res, _ = a.Alerts(context.Background(), alerts.AlertsOpts{}) if len(res.Alerts) != 0 { t.Fatalf("expected 0 alerts, got %d", len(a.alerts)) From a07d3ce80174d89866ff4e84ea8f39f643e16c8d Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 9 Sep 2024 11:45:56 +0200 Subject: [PATCH 21/33] testing: add MineTransactions --- internal/test/e2e/cluster.go | 14 ++++++++++++++ internal/test/e2e/cluster_test.go | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 133868029..332215159 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -936,6 +936,20 @@ func (c *TestCluster) AddHostsBlocking(n int) []*Host { return hosts } +// MineTransactions tries to mine the transactions in the transaction pool until +// it is empty. +func (c *TestCluster) MineTransactions(ctx context.Context) error { + return test.Retry(100, 100*time.Millisecond, func() error { + txns, err := c.Bus.TransactionPool(ctx) + if err != nil { + return err + } else if len(txns) > 0 { + c.MineBlocks(1) + } + return nil + }) +} + // Shutdown shuts down a TestCluster. func (c *TestCluster) Shutdown() { c.tt.Helper() diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 12c839a82..ef81e8d39 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -1351,6 +1351,11 @@ func TestEphemeralAccountSync(t *testing.T) { } acc := accounts[0] + // stop autopilot and mine transactions, this prevents an NDF where we + // double spend outputs after restarting the bus + cluster.ShutdownAutopilot(context.Background()) + tt.OK(cluster.MineTransactions(context.Background())) + // stop the cluster host := cluster.hosts[0] cluster.hosts = nil // exclude hosts from shutdown From f1748cdf84e2ca7dfc508aac40ca9c9a026dc862 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 9 Sep 2024 11:46:51 +0200 Subject: [PATCH 22/33] tmp: run TestEphemeralAccountSync in a loop --- .github/workflows/test.yml | 50 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6aee77227..ebdb1bd34 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,34 +49,34 @@ jobs: host port: 3800 mysql version: '8' mysql root password: test - - name: Test Stores - uses: n8maninger/action-golang-test@v1 - with: - args: "-race;-short" - - name: Test Stores - MySQL - if: matrix.os == 'ubuntu-latest' - uses: n8maninger/action-golang-test@v1 - env: - RENTERD_DB_URI: 127.0.0.1:3800 - RENTERD_DB_USER: root - RENTERD_DB_PASSWORD: test - with: - package: "./stores" - args: "-race;-short" + # - name: Test Stores + # uses: n8maninger/action-golang-test@v1 + # with: + # args: "-race;-short" + # - name: Test Stores - MySQL + # if: matrix.os == 'ubuntu-latest' + # uses: n8maninger/action-golang-test@v1 + # env: + # RENTERD_DB_URI: 127.0.0.1:3800 + # RENTERD_DB_USER: root + # RENTERD_DB_PASSWORD: test + # with: + # package: "./stores" + # args: "-race;-short" - name: Test Integration uses: n8maninger/action-golang-test@v1 with: package: "./internal/test/e2e/..." - args: "-failfast;-race;-timeout=60m" - - name: Test Integration - MySQL - if: matrix.os == 'ubuntu-latest' - uses: n8maninger/action-golang-test@v1 - env: - RENTERD_DB_URI: 127.0.0.1:3800 - RENTERD_DB_USER: root - RENTERD_DB_PASSWORD: test - with: - package: "./internal/test/e2e/..." - args: "-failfast;-race;-timeout=60m" + args: "-failfast;-race;-timeout=60m;-count=100;-run=TestEphemeralAccountSync" + # - name: Test Integration - MySQL + # if: matrix.os == 'ubuntu-latest' + # uses: n8maninger/action-golang-test@v1 + # env: + # RENTERD_DB_URI: 127.0.0.1:3800 + # RENTERD_DB_USER: root + # RENTERD_DB_PASSWORD: test + # with: + # package: "./internal/test/e2e/..." + # args: "-failfast;-race;-timeout=60m" - name: Build run: go build -o bin/ ./cmd/renterd From 414a239ef8880060d61bd18c86e5e5264214d67c Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 9 Sep 2024 12:07:37 +0200 Subject: [PATCH 23/33] Revert "tmp: run TestEphemeralAccountSync in a loop" This reverts commit 88259c6e40bfbd9eeb7389283155fe9c44e5690b. --- .github/workflows/test.yml | 50 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ebdb1bd34..6aee77227 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,34 +49,34 @@ jobs: host port: 3800 mysql version: '8' mysql root password: test - # - name: Test Stores - # uses: n8maninger/action-golang-test@v1 - # with: - # args: "-race;-short" - # - name: Test Stores - MySQL - # if: matrix.os == 'ubuntu-latest' - # uses: n8maninger/action-golang-test@v1 - # env: - # RENTERD_DB_URI: 127.0.0.1:3800 - # RENTERD_DB_USER: root - # RENTERD_DB_PASSWORD: test - # with: - # package: "./stores" - # args: "-race;-short" + - name: Test Stores + uses: n8maninger/action-golang-test@v1 + with: + args: "-race;-short" + - name: Test Stores - MySQL + if: matrix.os == 'ubuntu-latest' + uses: n8maninger/action-golang-test@v1 + env: + RENTERD_DB_URI: 127.0.0.1:3800 + RENTERD_DB_USER: root + RENTERD_DB_PASSWORD: test + with: + package: "./stores" + args: "-race;-short" - name: Test Integration uses: n8maninger/action-golang-test@v1 with: package: "./internal/test/e2e/..." - args: "-failfast;-race;-timeout=60m;-count=100;-run=TestEphemeralAccountSync" - # - name: Test Integration - MySQL - # if: matrix.os == 'ubuntu-latest' - # uses: n8maninger/action-golang-test@v1 - # env: - # RENTERD_DB_URI: 127.0.0.1:3800 - # RENTERD_DB_USER: root - # RENTERD_DB_PASSWORD: test - # with: - # package: "./internal/test/e2e/..." - # args: "-failfast;-race;-timeout=60m" + args: "-failfast;-race;-timeout=60m" + - name: Test Integration - MySQL + if: matrix.os == 'ubuntu-latest' + uses: n8maninger/action-golang-test@v1 + env: + RENTERD_DB_URI: 127.0.0.1:3800 + RENTERD_DB_USER: root + RENTERD_DB_PASSWORD: test + with: + package: "./internal/test/e2e/..." + args: "-failfast;-race;-timeout=60m" - name: Build run: go build -o bin/ ./cmd/renterd From d9237294913ce90f7773631e3c5b5debfd600abe Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Mon, 9 Sep 2024 14:07:22 +0000 Subject: [PATCH 24/33] ui: v0.61.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a422773e6..908c20ad3 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( go.sia.tech/hostd v1.1.3-0.20240903081107-6e044db95238 go.sia.tech/jape v0.12.1 go.sia.tech/mux v1.2.0 - go.sia.tech/web/renterd v0.60.1 + go.sia.tech/web/renterd v0.61.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.27.0 golang.org/x/sys v0.25.0 diff --git a/go.sum b/go.sum index e2c613525..f32b39629 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/renterd v0.60.1 h1:KJ/DgYKES29HoRd4/XY/G9CzTrHpMANCRCffIYc6Sxg= -go.sia.tech/web/renterd v0.60.1/go.mod h1:SWwKoAJvLxiHjTXsNPKX3RLiQzJb/vxwcpku3F78MO8= +go.sia.tech/web/renterd v0.61.0 h1:DmSGkpbaqodKvP4Mn79lLeZF2xqcWFQRrT2xPuLf8Uo= +go.sia.tech/web/renterd v0.61.0/go.mod h1:VWfvYtmdJGfrqSoNRO3NoOjUij+RB/xNO4M0HqIf1+M= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From 0d0321f8cf8b9c5e6694aeededa75eccde77dd23 Mon Sep 17 00:00:00 2001 From: PJ Date: Fri, 6 Sep 2024 14:49:18 +0200 Subject: [PATCH 25/33] utils: add SendRequest --- bus/client/client.go | 20 +++------------- bus/client/metrics.go | 42 ++++++---------------------------- bus/client/slabs.go | 14 ++---------- internal/bus/forex.go | 26 +++------------------ internal/utils/web.go | 24 +++++++++++++++++++ internal/worker/events_test.go | 17 +++----------- webhooks/webhooks.go | 17 +++----------- worker/client/client.go | 40 +++++++++----------------------- 8 files changed, 56 insertions(+), 144 deletions(-) diff --git a/bus/client/client.go b/bus/client/client.go index b082e5d9e..dd04f6664 100644 --- a/bus/client/client.go +++ b/bus/client/client.go @@ -1,13 +1,11 @@ package client import ( - "encoding/json" - "errors" - "io" "net/http" "go.sia.tech/jape" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" ) // A Client provides methods for interacting with a bus. @@ -34,18 +32,6 @@ func (c *Client) do(req *http.Request, resp interface{}) error { if c.c.Password != "" { req.SetBasicAuth("", c.c.Password) } - r, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer io.Copy(io.Discard, r.Body) - defer r.Body.Close() - if !(200 <= r.StatusCode && r.StatusCode < 300) { - err, _ := io.ReadAll(r.Body) - return errors.New(string(err)) - } - if resp == nil { - return nil - } - return json.NewDecoder(r.Body).Decode(resp) + _, _, err := utils.SendRequest(req, &resp) + return err } diff --git a/bus/client/metrics.go b/bus/client/metrics.go index 10bc2fbca..c35937214 100644 --- a/bus/client/metrics.go +++ b/bus/client/metrics.go @@ -4,15 +4,14 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" - "io" "net/http" "net/url" "time" "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" ) func (c *Client) ContractMetrics(ctx context.Context, start time.Time, n uint64, interval time.Duration, opts api.ContractMetricsQueryOpts) ([]api.ContractMetric, error) { @@ -130,16 +129,8 @@ func (c *Client) PruneMetrics(ctx context.Context, metric string, cutoff time.Ti panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return errors.New(string(err)) - } - return nil + _, _, err = utils.SendRequest(req, nil) + return err } func (c *Client) recordMetric(ctx context.Context, key string, d interface{}) error { @@ -159,17 +150,8 @@ func (c *Client) recordMetric(ctx context.Context, key string, d interface{}) er panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return errors.New(string(err)) - } - return nil + _, _, err = utils.SendRequest(req, nil) + return err } func (c *Client) metric(ctx context.Context, key string, values url.Values, res interface{}) error { @@ -185,16 +167,6 @@ func (c *Client) metric(ctx context.Context, key string, values url.Values, res panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - - if resp.StatusCode != 200 && resp.StatusCode != 206 { - err, _ := io.ReadAll(resp.Body) - return errors.New(string(err)) - } - return json.NewDecoder(resp.Body).Decode(&res) + _, _, err = utils.SendRequest(req, &res) + return err } diff --git a/bus/client/slabs.go b/bus/client/slabs.go index db5c0023a..571eaa3e7 100644 --- a/bus/client/slabs.go +++ b/bus/client/slabs.go @@ -3,7 +3,6 @@ package client import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -12,6 +11,7 @@ import ( "time" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" ) @@ -33,18 +33,8 @@ func (c *Client) AddPartialSlab(ctx context.Context, data []byte, minShards, tot panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, false, err - } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return nil, false, errors.New(string(err)) - } var apsr api.AddPartialSlabResponse - err = json.NewDecoder(resp.Body).Decode(&apsr) + _, _, err = utils.SendRequest(req, &apsr) if err != nil { return nil, false, err } diff --git a/internal/bus/forex.go b/internal/bus/forex.go index b6544b911..b1849b343 100644 --- a/internal/bus/forex.go +++ b/internal/bus/forex.go @@ -2,11 +2,10 @@ package bus import ( "context" - "encoding/json" - "errors" "fmt" - "io" "net/http" + + "go.sia.tech/renterd/internal/utils" ) type ( @@ -27,25 +26,6 @@ func (f *client) SiacoinExchangeRate(ctx context.Context, currency string) (rate } req.Header.Set("Accept", "application/json") - // create http client - resp, err := http.DefaultClient.Do(req) - if err != nil { - return 0, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - // check status code - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - var errorMessage string - if err := json.NewDecoder(io.LimitReader(resp.Body, 1024)).Decode(&errorMessage); err != nil { - return 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - return 0, errors.New(errorMessage) - } - - // decode exchange rate - if err := json.NewDecoder(resp.Body).Decode(&rate); err != nil { - return 0, fmt.Errorf("failed to decode response: %w", err) - } + _, _, err = utils.SendRequest(req, &rate) return } diff --git a/internal/utils/web.go b/internal/utils/web.go index 6f0caa571..415afb8a1 100644 --- a/internal/utils/web.go +++ b/internal/utils/web.go @@ -1,8 +1,10 @@ package utils import ( + "encoding/json" "errors" "fmt" + "io" "net" "net/http" _ "net/http/pprof" @@ -80,3 +82,25 @@ func OpenBrowser(url string) error { return fmt.Errorf("unsupported platform %q", runtime.GOOS) } } + +func SendRequest(req *http.Request, resp interface{}) (header http.Header, statusCode int, err error) { + r, err := http.DefaultClient.Do(req) + if err != nil { + return nil, 0, err + } + defer r.Body.Close() + defer io.Copy(io.Discard, r.Body) + + header = r.Header + statusCode = r.StatusCode + + if statusCode < 200 || statusCode >= 300 { + lr := io.LimitReader(r.Body, 1<<20) // 1MiB + errMsg, _ := io.ReadAll(lr) + err = fmt.Errorf("HTTP error: %s (status: %d)", string(errMsg), statusCode) + } else if resp != nil { + err = json.NewDecoder(r.Body).Decode(resp) + } + + return +} diff --git a/internal/worker/events_test.go b/internal/worker/events_test.go index cab65c62d..5b64c5b5f 100644 --- a/internal/worker/events_test.go +++ b/internal/worker/events_test.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "net/http/httptest" "sync" @@ -16,6 +15,7 @@ import ( "go.sia.tech/jape" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" "go.uber.org/zap/zaptest/observer" @@ -202,18 +202,7 @@ func sendEvent(url string, event webhooks.Event) error { if err != nil { return err } - defer io.ReadAll(req.Body) // always drain body - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - errStr, err := io.ReadAll(req.Body) - if err != nil { - return fmt.Errorf("failed to read response body: %w", err) - } - return fmt.Errorf("Webhook returned unexpected status %v: %v", resp.StatusCode, string(errStr)) - } - return nil + _, _, err = utils.SendRequest(req, nil) + return err } diff --git a/webhooks/webhooks.go b/webhooks/webhooks.go index 0f1eb636f..e9a6e256f 100644 --- a/webhooks/webhooks.go +++ b/webhooks/webhooks.go @@ -7,11 +7,11 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "sync" "time" + "go.sia.tech/renterd/internal/utils" "go.uber.org/zap" ) @@ -268,18 +268,7 @@ func sendEvent(ctx context.Context, url string, headers map[string]string, actio for k, v := range headers { req.Header.Set(k, v) } - defer io.ReadAll(req.Body) // always drain body - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - errStr, err := io.ReadAll(req.Body) - if err != nil { - return fmt.Errorf("failed to read response body: %w", err) - } - return fmt.Errorf("Webhook returned unexpected status %v: %v", resp.StatusCode, string(errStr)) - } - return nil + _, _, err = utils.SendRequest(req, nil) + return err } diff --git a/worker/client/client.go b/worker/client/client.go index 7df0a6052..82f061684 100644 --- a/worker/client/client.go +++ b/worker/client/client.go @@ -15,6 +15,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/jape" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/webhooks" ) @@ -108,21 +109,14 @@ func (c *Client) HeadObject(ctx context.Context, bucket, path string, opts api.H req.SetBasicAuth("", c.c.WithContext(ctx).Password) opts.ApplyHeaders(req.Header) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - if resp.StatusCode != 200 && resp.StatusCode != 206 { - _ = resp.Body.Close() - switch resp.StatusCode { - case http.StatusNotFound: - return nil, api.ErrObjectNotFound - default: - return nil, errors.New(http.StatusText(resp.StatusCode)) - } + headers, statusCode, err := utils.SendRequest(req, nil) + if err != nil && statusCode == http.StatusNotFound { + return nil, api.ErrObjectNotFound + } else if err != nil { + return nil, errors.New(http.StatusText(statusCode)) } - head, err := parseObjectResponseHeaders(resp.Header) + head, err := parseObjectResponseHeaders(headers) if err != nil { return nil, err } @@ -225,17 +219,11 @@ func (c *Client) UploadMultipartUploadPart(ctx context.Context, r io.Reader, buc } else if req.ContentLength, err = sizeFromSeeker(r); err != nil { return nil, fmt.Errorf("failed to get content length from seeker: %w", err) } - resp, err := http.DefaultClient.Do(req) + header, _, err := utils.SendRequest(req, nil) if err != nil { return nil, err } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return nil, errors.New(string(err)) - } - return &api.UploadMultipartUploadPartResponse{ETag: resp.Header.Get("ETag")}, nil + return &api.UploadMultipartUploadPartResponse{ETag: header.Get("ETag")}, nil } // UploadObject uploads the data in r, creating an object at the given path. @@ -262,17 +250,11 @@ func (c *Client) UploadObject(ctx context.Context, r io.Reader, bucket, path str } else if req.ContentLength, err = sizeFromSeeker(r); err != nil { return nil, fmt.Errorf("failed to get content length from seeker: %w", err) } - resp, err := http.DefaultClient.Do(req) + header, _, err := utils.SendRequest(req, nil) if err != nil { return nil, err } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return nil, errors.New(string(err)) - } - return &api.UploadObjectResponse{ETag: resp.Header.Get("ETag")}, nil + return &api.UploadObjectResponse{ETag: header.Get("ETag")}, nil } // UploadStats returns the upload stats. From 33c19cb14e8d752fd48036176fb8a4b82134b469 Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 9 Sep 2024 15:45:30 +0200 Subject: [PATCH 26/33] util: rename method --- bus/client/client.go | 2 +- bus/client/metrics.go | 6 +++--- bus/client/slabs.go | 2 +- internal/bus/forex.go | 2 +- internal/utils/web.go | 2 +- internal/worker/events_test.go | 2 +- webhooks/webhooks.go | 2 +- worker/client/client.go | 6 +++--- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bus/client/client.go b/bus/client/client.go index dd04f6664..c31b6d4a2 100644 --- a/bus/client/client.go +++ b/bus/client/client.go @@ -32,6 +32,6 @@ func (c *Client) do(req *http.Request, resp interface{}) error { if c.c.Password != "" { req.SetBasicAuth("", c.c.Password) } - _, _, err := utils.SendRequest(req, &resp) + _, _, err := utils.DoRequest(req, &resp) return err } diff --git a/bus/client/metrics.go b/bus/client/metrics.go index c35937214..3923145f0 100644 --- a/bus/client/metrics.go +++ b/bus/client/metrics.go @@ -129,7 +129,7 @@ func (c *Client) PruneMetrics(ctx context.Context, metric string, cutoff time.Ti panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - _, _, err = utils.SendRequest(req, nil) + _, _, err = utils.DoRequest(req, nil) return err } @@ -150,7 +150,7 @@ func (c *Client) recordMetric(ctx context.Context, key string, d interface{}) er panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - _, _, err = utils.SendRequest(req, nil) + _, _, err = utils.DoRequest(req, nil) return err } @@ -167,6 +167,6 @@ func (c *Client) metric(ctx context.Context, key string, values url.Values, res panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - _, _, err = utils.SendRequest(req, &res) + _, _, err = utils.DoRequest(req, &res) return err } diff --git a/bus/client/slabs.go b/bus/client/slabs.go index 571eaa3e7..b0fc8837e 100644 --- a/bus/client/slabs.go +++ b/bus/client/slabs.go @@ -34,7 +34,7 @@ func (c *Client) AddPartialSlab(ctx context.Context, data []byte, minShards, tot } req.SetBasicAuth("", c.c.WithContext(ctx).Password) var apsr api.AddPartialSlabResponse - _, _, err = utils.SendRequest(req, &apsr) + _, _, err = utils.DoRequest(req, &apsr) if err != nil { return nil, false, err } diff --git a/internal/bus/forex.go b/internal/bus/forex.go index b1849b343..122056949 100644 --- a/internal/bus/forex.go +++ b/internal/bus/forex.go @@ -26,6 +26,6 @@ func (f *client) SiacoinExchangeRate(ctx context.Context, currency string) (rate } req.Header.Set("Accept", "application/json") - _, _, err = utils.SendRequest(req, &rate) + _, _, err = utils.DoRequest(req, &rate) return } diff --git a/internal/utils/web.go b/internal/utils/web.go index 415afb8a1..48a8be969 100644 --- a/internal/utils/web.go +++ b/internal/utils/web.go @@ -83,7 +83,7 @@ func OpenBrowser(url string) error { } } -func SendRequest(req *http.Request, resp interface{}) (header http.Header, statusCode int, err error) { +func DoRequest(req *http.Request, resp interface{}) (header http.Header, statusCode int, err error) { r, err := http.DefaultClient.Do(req) if err != nil { return nil, 0, err diff --git a/internal/worker/events_test.go b/internal/worker/events_test.go index 5b64c5b5f..95a74da91 100644 --- a/internal/worker/events_test.go +++ b/internal/worker/events_test.go @@ -203,6 +203,6 @@ func sendEvent(url string, event webhooks.Event) error { return err } - _, _, err = utils.SendRequest(req, nil) + _, _, err = utils.DoRequest(req, nil) return err } diff --git a/webhooks/webhooks.go b/webhooks/webhooks.go index e9a6e256f..ce643835c 100644 --- a/webhooks/webhooks.go +++ b/webhooks/webhooks.go @@ -269,6 +269,6 @@ func sendEvent(ctx context.Context, url string, headers map[string]string, actio req.Header.Set(k, v) } - _, _, err = utils.SendRequest(req, nil) + _, _, err = utils.DoRequest(req, nil) return err } diff --git a/worker/client/client.go b/worker/client/client.go index 82f061684..2bac4f99f 100644 --- a/worker/client/client.go +++ b/worker/client/client.go @@ -109,7 +109,7 @@ func (c *Client) HeadObject(ctx context.Context, bucket, path string, opts api.H req.SetBasicAuth("", c.c.WithContext(ctx).Password) opts.ApplyHeaders(req.Header) - headers, statusCode, err := utils.SendRequest(req, nil) + headers, statusCode, err := utils.DoRequest(req, nil) if err != nil && statusCode == http.StatusNotFound { return nil, api.ErrObjectNotFound } else if err != nil { @@ -219,7 +219,7 @@ func (c *Client) UploadMultipartUploadPart(ctx context.Context, r io.Reader, buc } else if req.ContentLength, err = sizeFromSeeker(r); err != nil { return nil, fmt.Errorf("failed to get content length from seeker: %w", err) } - header, _, err := utils.SendRequest(req, nil) + header, _, err := utils.DoRequest(req, nil) if err != nil { return nil, err } @@ -250,7 +250,7 @@ func (c *Client) UploadObject(ctx context.Context, r io.Reader, bucket, path str } else if req.ContentLength, err = sizeFromSeeker(r); err != nil { return nil, fmt.Errorf("failed to get content length from seeker: %w", err) } - header, _, err := utils.SendRequest(req, nil) + header, _, err := utils.DoRequest(req, nil) if err != nil { return nil, err } From 94a3a694307ed95022c37c4d404ce6c7eefdabc3 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 9 Sep 2024 16:41:30 +0200 Subject: [PATCH 27/33] utils: remove named return vars --- internal/utils/web.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/internal/utils/web.go b/internal/utils/web.go index 48a8be969..471270deb 100644 --- a/internal/utils/web.go +++ b/internal/utils/web.go @@ -83,7 +83,7 @@ func OpenBrowser(url string) error { } } -func DoRequest(req *http.Request, resp interface{}) (header http.Header, statusCode int, err error) { +func DoRequest(req *http.Request, resp interface{}) (http.Header, int, error) { r, err := http.DefaultClient.Do(req) if err != nil { return nil, 0, err @@ -91,16 +91,12 @@ func DoRequest(req *http.Request, resp interface{}) (header http.Header, statusC defer r.Body.Close() defer io.Copy(io.Discard, r.Body) - header = r.Header - statusCode = r.StatusCode - - if statusCode < 200 || statusCode >= 300 { + if r.StatusCode < 200 || r.StatusCode >= 300 { lr := io.LimitReader(r.Body, 1<<20) // 1MiB errMsg, _ := io.ReadAll(lr) - err = fmt.Errorf("HTTP error: %s (status: %d)", string(errMsg), statusCode) + return http.Header{}, 0, fmt.Errorf("HTTP error: %s (status: %d)", string(errMsg), r.StatusCode) } else if resp != nil { - err = json.NewDecoder(r.Body).Decode(resp) + return http.Header{}, 0, json.NewDecoder(r.Body).Decode(resp) } - - return + return r.Header, r.StatusCode, nil } From e082fdfd92cc32e02dfc5aad7f2346ec94da1ebd Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 10 Sep 2024 23:17:38 +0200 Subject: [PATCH 28/33] stores: use raw SQL in the migration --- internal/sql/migrations.go | 150 +----------------- .../main/migration_00018_settings.sql | 84 +++++++++- .../main/migration_00018_settings.sql | 46 +++++- 3 files changed, 129 insertions(+), 151 deletions(-) diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index 12e21cb4b..029aba4ea 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -3,12 +3,10 @@ package sql import ( "context" "embed" - "encoding/json" "fmt" "strings" "unicode/utf8" - "go.sia.tech/renterd/api" "go.sia.tech/renterd/internal/utils" "go.uber.org/zap" ) @@ -223,153 +221,7 @@ var ( { ID: "00018_settings", Migrate: func(tx Tx) error { - log.Infof("performing %s migration '00018_settings'", dbIdentifier) - - // fetch all settings - rows, err := tx.Query(ctx, "SELECT `key`, value FROM settings") - if err != nil { - return fmt.Errorf("failed to fetch settings: %v", err) - } - defer rows.Close() - - settings := make(map[string]string) - for rows.Next() { - var k, v string - if err := rows.Scan(&k, &v); err != nil { - return fmt.Errorf("failed to scan setting: %v", err) - } - settings[k] = v - } - - // migrate gouging settings - if v, ok := settings["gouging"]; ok { - var gs api.GougingSettings - err := json.Unmarshal([]byte(v), &gs) - if err == nil { - err = gs.Validate() - } - if err != nil { - log.Warnf("gouging settings are not being migrated, err: %v", err) - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "gouging"); err != nil { - return fmt.Errorf("failed to delete gouging settings: %v", err) - } - } - } else { - log.Warn("no gouging settings found") - } - - // migrate pinning settings - if v, ok := settings["pricepinning"]; ok { - var ps api.PinnedSettings - err := json.Unmarshal([]byte(v), &ps) - if err == nil { - err = ps.Validate() - } - if err == nil { - updated, _ := json.Marshal(ps) - if err := m.UpdateSetting(ctx, tx, "pinned", string(updated)); err != nil { - return fmt.Errorf("failed to insert pinned settings: %v", err) - } - } else { - log.Warnf("pricepinning settings are not being migrated, err: %v", err) - } - - // always delete because it got renamed - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "pricepinning"); err != nil { - log.Warnf("failed to delete pricepinning settings: %v", err) - } - } else { - log.Warn("no pricepinning settings found") - } - - // migrate S3 authentication settings - if v, ok := settings["s3authentication"]; ok { - var s3s api.S3Settings - err := json.Unmarshal([]byte(v), &s3s.Authentication) - if err == nil { - err = s3s.Validate() - } - if err == nil { - updated, _ := json.Marshal(s3s) - if err := m.UpdateSetting(ctx, tx, "s3", string(updated)); err != nil { - return fmt.Errorf("failed to insert s3 settings: %v", err) - } - } else { - log.Warnf("s3authentication settings are not being migrated, err: %v", err) - } - - // always delete because it got renamed - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "s3authentication"); err != nil { - log.Warnf("failed to delete s3authentication settings: %v", err) - } - } else { - log.Warn("no s3authentication setting found") - } - - // migrate upload settings - us := api.DefaultUploadSettings("mainnet") - - if v, ok := settings["contractset"]; ok { - var css struct { - Default string `json:"default"` - } - if err := json.Unmarshal([]byte(v), &css); err != nil { - log.Warnf("contractset settings are not being migrated, err: %v", err) - } else { - us.DefaultContractSet = css.Default - } - - // always delete because it got replaced - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "contractset"); err != nil { - return err - } - } - - if v, ok := settings["uploadpacking"]; ok { - var ups api.UploadPackingSettings - if err := json.Unmarshal([]byte(v), &ups); err != nil { - log.Warnf("uploadpacking settings are not being migrated, err: %v", err) - } else { - us.Packing = ups - } - - // always delete because it got replaced - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "uploadpacking"); err != nil { - return err - } - } - - if v, ok := settings["redundancy"]; ok { - var rs api.RedundancySettings - err := json.Unmarshal([]byte(v), &rs) - if err == nil { - err = rs.Validate() - } - if err != nil { - log.Warnf("redundancy settings are not being migrated, err: %v", err) - } else { - us.Redundancy = rs - } - - // always delete because it got replaced - if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", "redundancy"); err != nil { - return err - } - } - - // update upload settings - if err := us.Validate(); err != nil { - log.Warnf("upload settings are not being migrated, err: %v", err) - return err // developer error - } else { - updated, _ := json.Marshal(us) - if err := m.UpdateSetting(ctx, tx, "upload", string(updated)); err != nil { - return fmt.Errorf("failed to insert upload settings: %v", err) - } - } - - log.Info("migration '00018_settings' complete") - return nil + return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00018_settings", log) }, }, } diff --git a/stores/sql/mysql/migrations/main/migration_00018_settings.sql b/stores/sql/mysql/migrations/main/migration_00018_settings.sql index 09a7b0592..0ac46b266 100644 --- a/stores/sql/mysql/migrations/main/migration_00018_settings.sql +++ b/stores/sql/mysql/migrations/main/migration_00018_settings.sql @@ -1 +1,83 @@ --- placeholder +-- avoid duplicate key errors +DELETE FROM settings WHERE `key` IN ("s3", "upload", "pinned"); + +-- migrate settings +INSERT INTO settings (created_at, `key`, value) +SELECT NOW(), k, v +FROM ( + -- upload is a combination of uploadpacking, redundancy, and contractset + SELECT + "upload" as k, + JSON_MERGE_PATCH( + JSON_OBJECT("packing", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "uploadpacking")), + JSON_MERGE_PATCH( + JSON_OBJECT("redundancy", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "redundancy")), + JSON_OBJECT("defaultContractSet", (SELECT JSON_EXTRACT(value, "$.default") FROM settings WHERE `key` = "contractset")) + ) + ) as v + WHERE JSON_EXTRACT( + JSON_MERGE_PATCH( + JSON_OBJECT("packing", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "uploadpacking")), + JSON_MERGE_PATCH( + JSON_OBJECT("redundancy", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "redundancy")), + JSON_OBJECT("defaultContractSet", (SELECT JSON_EXTRACT(value, "$.default") FROM settings WHERE `key` = "contractset")) + ) + ), "$.packing" + ) IS NOT NULL + AND JSON_EXTRACT( + JSON_MERGE_PATCH( + JSON_OBJECT("packing", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "uploadpacking")), + JSON_MERGE_PATCH( + JSON_OBJECT("redundancy", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "redundancy")), + JSON_OBJECT("defaultContractSet", (SELECT JSON_EXTRACT(value, "$.default") FROM settings WHERE `key` = "contractset")) + ) + ), "$.redundancy" + ) IS NOT NULL + + UNION ALL + + -- s3 wraps the s3authentication setting + SELECT + "s3" as k, + JSON_OBJECT("authentication", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "s3authentication")) as v + WHERE JSON_EXTRACT( + JSON_OBJECT("authentication", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "s3authentication")), + "$.authentication" + ) IS NOT NULL + + UNION ALL + + -- pinning renames pricepinning and removes the 'enabled' and 'forexEndpointURL' fields + SELECT + "pinned" as k, + JSON_REMOVE( + JSON_REMOVE( + (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "pricepinning"), + "$.enabled" + ), + "$.forexEndpointURL" + ) as v + WHERE JSON_EXTRACT( + JSON_REMOVE( + JSON_REMOVE( + (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "pricepinning"), + "$.enabled" + ), + "$.forexEndpointURL" + ), + "$.currency" + ) IS NOT NULL + AND JSON_EXTRACT( + JSON_REMOVE( + JSON_REMOVE( + (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "pricepinning"), + "$.enabled" + ), + "$.forexEndpointURL" + ), + "$.threshold" + ) IS NOT NULL +) as migration; + +-- delete old settings (TODO: should we?) +DELETE FROM settings WHERE `key` IN ("uploadpacking", "redundancy", "contractset", "s3authentication", "pricepinning"); diff --git a/stores/sql/sqlite/migrations/main/migration_00018_settings.sql b/stores/sql/sqlite/migrations/main/migration_00018_settings.sql index 09a7b0592..aef6d775a 100644 --- a/stores/sql/sqlite/migrations/main/migration_00018_settings.sql +++ b/stores/sql/sqlite/migrations/main/migration_00018_settings.sql @@ -1 +1,45 @@ --- placeholder +-- avoid duplicate key errors +DELETE FROM settings WHERE `key` IN ("s3", "upload", "pinned"); + +-- migrate settings +INSERT INTO settings (created_at, `key`, value) +SELECT DATETIME('now'), k, v +FROM ( + -- upload is a combination of uploadpacking, redundancy, and contractset + SELECT + "upload" as k, + json_patch( + json_object("packing", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE key = "uploadpacking")), + json_patch( + json_object("redundancy", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE key = "redundancy")), + json_object("defaultContractSet", (SELECT JSON_EXTRACT(value, "$.default") FROM settings WHERE key = "contractset")) + ) + ) as v + WHERE json_extract(v, "$.packing") IS NOT NULL + AND json_extract(v, "$.redundancy") IS NOT NULL + + UNION ALL + + -- s3 wraps the s3authentication setting + SELECT + "s3" as k, + json_object("authentication", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE key = "s3authentication")) as v + WHERE json_extract(v, "$.authentication") IS NOT NULL + + UNION ALL + + -- pinning renames pricepinning and removes the 'enabled' and 'forexEndpointURL' fields + SELECT + "pinned" as k, + json_remove( + json_remove( + (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE key = "pricepinning"), + "$.enabled" + ), + "$.forexEndpointURL" + ) as v + WHERE json_extract(v, "$.currency") IS NOT NULL AND json_extract(v, "$.threshold") IS NOT NULL +) + +-- delete old settings +DELETE FROM settings WHERE `key` IN ("uploadpacking", "redundancy", "contractset", "s3authentication", "pricepinning"); \ No newline at end of file From dafe545d655b5b7dd5fce781b8492e96cd90d416 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 10 Sep 2024 23:19:40 +0200 Subject: [PATCH 29/33] stores: cleanup migration --- .../main/migration_00018_settings.sql | 62 +++++++++---------- .../main/migration_00018_settings.sql | 10 +-- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/stores/sql/mysql/migrations/main/migration_00018_settings.sql b/stores/sql/mysql/migrations/main/migration_00018_settings.sql index 0ac46b266..006e97784 100644 --- a/stores/sql/mysql/migrations/main/migration_00018_settings.sql +++ b/stores/sql/mysql/migrations/main/migration_00018_settings.sql @@ -8,28 +8,28 @@ FROM ( -- upload is a combination of uploadpacking, redundancy, and contractset SELECT "upload" as k, - JSON_MERGE_PATCH( - JSON_OBJECT("packing", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "uploadpacking")), - JSON_MERGE_PATCH( - JSON_OBJECT("redundancy", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "redundancy")), - JSON_OBJECT("defaultContractSet", (SELECT JSON_EXTRACT(value, "$.default") FROM settings WHERE `key` = "contractset")) + json_merge_patch( + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "uploadpacking")), + json_merge_patch( + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE `key` = "contractset")) ) ) as v - WHERE JSON_EXTRACT( - JSON_MERGE_PATCH( - JSON_OBJECT("packing", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "uploadpacking")), - JSON_MERGE_PATCH( - JSON_OBJECT("redundancy", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "redundancy")), - JSON_OBJECT("defaultContractSet", (SELECT JSON_EXTRACT(value, "$.default") FROM settings WHERE `key` = "contractset")) + WHERE json_extract( + json_merge_patch( + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "uploadpacking")), + json_merge_patch( + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE `key` = "contractset")) ) ), "$.packing" ) IS NOT NULL - AND JSON_EXTRACT( - JSON_MERGE_PATCH( - JSON_OBJECT("packing", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "uploadpacking")), - JSON_MERGE_PATCH( - JSON_OBJECT("redundancy", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "redundancy")), - JSON_OBJECT("defaultContractSet", (SELECT JSON_EXTRACT(value, "$.default") FROM settings WHERE `key` = "contractset")) + AND json_extract( + json_merge_patch( + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "uploadpacking")), + json_merge_patch( + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE `key` = "contractset")) ) ), "$.redundancy" ) IS NOT NULL @@ -39,9 +39,9 @@ FROM ( -- s3 wraps the s3authentication setting SELECT "s3" as k, - JSON_OBJECT("authentication", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "s3authentication")) as v - WHERE JSON_EXTRACT( - JSON_OBJECT("authentication", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "s3authentication")), + json_object("authentication", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "s3authentication")) as v + WHERE json_extract( + json_object("authentication", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "s3authentication")), "$.authentication" ) IS NOT NULL @@ -50,27 +50,27 @@ FROM ( -- pinning renames pricepinning and removes the 'enabled' and 'forexEndpointURL' fields SELECT "pinned" as k, - JSON_REMOVE( - JSON_REMOVE( - (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "pricepinning"), + json_remove( + json_remove( + (SELECT json_extract(value, "$") FROM settings WHERE `key` = "pricepinning"), "$.enabled" ), "$.forexEndpointURL" ) as v - WHERE JSON_EXTRACT( - JSON_REMOVE( - JSON_REMOVE( - (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "pricepinning"), + WHERE json_extract( + json_remove( + json_remove( + (SELECT json_extract(value, "$") FROM settings WHERE `key` = "pricepinning"), "$.enabled" ), "$.forexEndpointURL" ), "$.currency" ) IS NOT NULL - AND JSON_EXTRACT( - JSON_REMOVE( - JSON_REMOVE( - (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE `key` = "pricepinning"), + AND json_extract( + json_remove( + json_remove( + (SELECT json_extract(value, "$") FROM settings WHERE `key` = "pricepinning"), "$.enabled" ), "$.forexEndpointURL" diff --git a/stores/sql/sqlite/migrations/main/migration_00018_settings.sql b/stores/sql/sqlite/migrations/main/migration_00018_settings.sql index aef6d775a..91d3b2aa0 100644 --- a/stores/sql/sqlite/migrations/main/migration_00018_settings.sql +++ b/stores/sql/sqlite/migrations/main/migration_00018_settings.sql @@ -9,10 +9,10 @@ FROM ( SELECT "upload" as k, json_patch( - json_object("packing", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE key = "uploadpacking")), + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE key = "uploadpacking")), json_patch( - json_object("redundancy", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE key = "redundancy")), - json_object("defaultContractSet", (SELECT JSON_EXTRACT(value, "$.default") FROM settings WHERE key = "contractset")) + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE key = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE key = "contractset")) ) ) as v WHERE json_extract(v, "$.packing") IS NOT NULL @@ -23,7 +23,7 @@ FROM ( -- s3 wraps the s3authentication setting SELECT "s3" as k, - json_object("authentication", (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE key = "s3authentication")) as v + json_object("authentication", (SELECT json_extract(value, "$") FROM settings WHERE key = "s3authentication")) as v WHERE json_extract(v, "$.authentication") IS NOT NULL UNION ALL @@ -33,7 +33,7 @@ FROM ( "pinned" as k, json_remove( json_remove( - (SELECT JSON_EXTRACT(value, "$") FROM settings WHERE key = "pricepinning"), + (SELECT json_extract(value, "$") FROM settings WHERE key = "pricepinning"), "$.enabled" ), "$.forexEndpointURL" From a2e4ee1a5d1f5df7da2780afd635dbfcc4df26cb Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 11 Sep 2024 09:26:15 +0200 Subject: [PATCH 30/33] db: add missing metrics migration --- internal/sql/migrations.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index c17473542..69a57f413 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -238,6 +238,12 @@ var ( return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00002_idx_wallet_metrics_immature", log) }, }, + { + ID: "00003_unix_ms", + Migrate: func(tx Tx) error { + return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00003_unix_ms", log) + }, + }, } } ) From 1f4d42647b2912e744195436527db2b1a9dd2952 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 11 Sep 2024 14:55:41 +0200 Subject: [PATCH 31/33] stores: remove default settings --- bus/bus.go | 4 ++- bus/routes.go | 33 ++++++++++++++---- cmd/renterd/node.go | 4 +-- internal/bus/pinmanager.go | 14 ++++++-- internal/test/e2e/cluster.go | 6 ++-- stores/settings.go | 34 +++---------------- stores/sql.go | 5 +-- .../main/migration_00018_settings.sql | 25 ++++++++------ stores/sql_test.go | 3 +- 9 files changed, 66 insertions(+), 62 deletions(-) diff --git a/bus/bus.go b/bus/bus.go index 148baae39..56fa946e7 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -309,6 +309,7 @@ type ( type Bus struct { startTime time.Time masterKey utils.MasterKey + network *consensus.Network alerts alerts.Alerter alertMgr AlertManager @@ -338,12 +339,13 @@ type Bus struct { } // New returns a new Bus -func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, store Store, announcementMaxAge time.Duration, explorerURL string, l *zap.Logger) (_ *Bus, err error) { +func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, store Store, announcementMaxAge time.Duration, explorerURL string, network *consensus.Network, l *zap.Logger) (_ *Bus, err error) { l = l.Named("bus") b := &Bus{ startTime: time.Now(), masterKey: masterKey, + network: network, accounts: store, s: s, diff --git a/bus/routes.go b/bus/routes.go index deee1410e..efcfbbfeb 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -16,6 +16,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" rhp3 "go.sia.tech/renterd/internal/rhp/v3" + "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/internal/gouging" rhp2 "go.sia.tech/renterd/internal/rhp/v2" @@ -1341,7 +1342,11 @@ func (b *Bus) packedSlabsHandlerDonePOST(jc jape.Context) { func (b *Bus) settingsGougingHandlerGET(jc jape.Context) { gs, err := b.ss.GougingSettings(jc.Request.Context()) - if jc.Check("failed to get gouging settings", err) == nil { + if errors.Is(err, sql.ErrSettingNotFound) { + b.logger.Warn("gouging settings not found, returning defaults") + jc.Encode(api.DefaultGougingSettings) + return + } else if jc.Check("failed to get gouging settings", err) == nil { jc.Encode(gs) } } @@ -1368,7 +1373,11 @@ func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { ps, err := b.ss.PinnedSettings(jc.Request.Context()) - if jc.Check("failed to get pinned settings", err) == nil { + if errors.Is(err, sql.ErrSettingNotFound) { + b.logger.Warn("pinned settings not found, returning defaults") + jc.Encode(api.DefaultPinnedSettings) + return + } else if jc.Check("failed to get pinned settings", err) == nil { // populate the Autopilots map with the current autopilots aps, err := b.as.Autopilots(jc.Request.Context()) if jc.Check("failed to fetch autopilots", err) != nil { @@ -1413,7 +1422,11 @@ func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { func (b *Bus) settingsUploadHandlerGET(jc jape.Context) { us, err := b.ss.UploadSettings(jc.Request.Context()) - if jc.Check("failed to get upload settings", err) == nil { + if errors.Is(err, sql.ErrSettingNotFound) { + b.logger.Warn("upload settings not found, returning defaults") + jc.Encode(api.DefaultUploadSettings(b.network.Name)) + return + } else if jc.Check("failed to get upload settings", err) == nil { jc.Encode(us) } } @@ -1439,7 +1452,11 @@ func (b *Bus) settingsUploadHandlerPUT(jc jape.Context) { func (b *Bus) settingsS3HandlerGET(jc jape.Context) { s3s, err := b.ss.S3Settings(jc.Request.Context()) - if jc.Check("failed to get S3 settings", err) == nil { + if errors.Is(err, sql.ErrSettingNotFound) { + b.logger.Warn("S3 settings not found, returning defaults") + jc.Encode(api.DefaultS3Settings) + return + } else if jc.Check("failed to get S3 settings", err) == nil { jc.Encode(s3s) } } @@ -1686,12 +1703,16 @@ func (b *Bus) paramsHandlerGougingGET(jc jape.Context) { func (b *Bus) gougingParams(ctx context.Context) (api.GougingParams, error) { gs, err := b.ss.GougingSettings(ctx) - if err != nil { + if errors.Is(err, sql.ErrSettingNotFound) { + gs = api.DefaultGougingSettings + } else if err != nil { return api.GougingParams{}, err } us, err := b.ss.UploadSettings(ctx) - if err != nil { + if errors.Is(err, sql.ErrSettingNotFound) { + us = api.DefaultUploadSettings(b.network.Name) + } else if err != nil { return api.GougingParams{}, err } diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 92c62ae9f..9fad6c05e 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -264,7 +264,7 @@ func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network if err != nil { return nil, nil, err } - sqlStore, err := stores.NewSQLStore(storeCfg, network) + sqlStore, err := stores.NewSQLStore(storeCfg) if err != nil { return nil, nil, err } @@ -386,7 +386,7 @@ func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network // create bus announcementMaxAgeHours := time.Duration(cfg.Bus.AnnouncementMaxAgeHours) * time.Hour - b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, explorerURL, logger) + b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, explorerURL, network, logger) if err != nil { return nil, nil, fmt.Errorf("failed to create bus: %w", err) } diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index 548e6b708..46372c8b9 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -12,6 +12,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" ) @@ -247,7 +248,9 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin // fetch gouging settings gs, err := pm.s.GougingSettings(ctx) - if err != nil { + if errors.Is(err, sql.ErrSettingNotFound) { + gs = api.DefaultGougingSettings + } else if err != nil { return err } @@ -323,9 +326,14 @@ func (pm *pinManager) updatePrices(ctx context.Context, forced bool) error { // fetch pinned settings settings, err := pm.s.PinnedSettings(ctx) - if err != nil { + if errors.Is(err, sql.ErrSettingNotFound) { + settings = api.DefaultPinnedSettings + } else if err != nil { return fmt.Errorf("failed to fetch pinned settings: %w", err) - } else if !settings.Enabled() { + } + + // check if pinning is enabled + if !settings.Enabled() { pm.logger.Debug("no pinned settings, skipping price update") return nil } diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index ec07a465a..4bbbecc80 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -553,8 +553,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, } // create store - network, genesis := testNetwork() - sqlStore, err := stores.NewSQLStore(storeCfg, network) + sqlStore, err := stores.NewSQLStore(storeCfg) if err != nil { return nil, nil, nil, nil, err } @@ -582,6 +581,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, } // create chain manager + network, genesis := testNetwork() store, state, err := chain.NewDBStore(bdb, network, genesis) if err != nil { return nil, nil, nil, nil, err @@ -639,7 +639,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, // create bus announcementMaxAgeHours := time.Duration(cfg.AnnouncementMaxAgeHours) * time.Hour - b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, "", logger) + b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, "", network, logger) if err != nil { return nil, nil, nil, nil, err } diff --git a/stores/settings.go b/stores/settings.go index 5801de1a9..471c013fb 100644 --- a/stores/settings.go +++ b/stores/settings.go @@ -3,7 +3,6 @@ package stores import ( "context" "encoding/json" - "errors" "fmt" "go.sia.tech/renterd/api" @@ -60,10 +59,7 @@ func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{} // fetch setting from cache value, ok := s.settings[key] if ok { - if err := json.Unmarshal([]byte(value), &out); err != nil { - s.logger.Warnf("failed to unmarshal %s setting '%s': %v, using default", key, value, err) - return json.Unmarshal([]byte(s.defaultSetting(key)), &out) - } + _ = json.Unmarshal([]byte(value), &out) // cached values are always valid json return nil } @@ -72,16 +68,13 @@ func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{} if err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { value, err = tx.Setting(ctx, key) return err - }); err != nil && !errors.Is(err, sql.ErrSettingNotFound) { - return fmt.Errorf("failed to fetch setting from db: %w", err) - } else if err != nil { - value = s.defaultSetting(key) + }); err != nil { + return err } // unmarshal setting if err := json.Unmarshal([]byte(value), &out); err != nil { - s.logger.Warnf("failed to unmarshal %s setting '%s': %v, using default", key, value, err) - return json.Unmarshal([]byte(s.defaultSetting(key)), &out) + return fmt.Errorf("failed to unmarshal setting '%s', err: %v", key, err) } // update cache @@ -112,22 +105,3 @@ func (s *SQLStore) updateSetting(ctx context.Context, key string, value any) err s.settings[key] = string(b) return nil } - -func (s *SQLStore) defaultSetting(key string) string { - switch key { - case SettingGouging: - b, _ := json.Marshal(api.DefaultGougingSettings) - return string(b) - case SettingPinned: - b, _ := json.Marshal(api.DefaultPinnedSettings) - return string(b) - case SettingS3: - b, _ := json.Marshal(api.DefaultS3Settings) - return string(b) - case SettingUpload: - b, _ := json.Marshal(api.DefaultUploadSettings(s.network.Name)) - return string(b) - default: - panic("unknown setting") // developer error - } -} diff --git a/stores/sql.go b/stores/sql.go index 424016927..14216ce32 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -8,7 +8,6 @@ import ( "sync" "time" - "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/stores/sql" @@ -43,7 +42,6 @@ type ( dbMetrics sql.MetricsDatabase logger *zap.SugaredLogger - network *consensus.Network walletAddress types.Address // ObjectDB related fields @@ -70,7 +68,7 @@ type ( // NewSQLStore uses a given Dialector to connect to a SQL database. NOTE: Only // pass migrate=true for the first instance of SQLHostDB if you connect via the // same Dialector multiple times. -func NewSQLStore(cfg Config, network *consensus.Network) (*SQLStore, error) { +func NewSQLStore(cfg Config) (*SQLStore, error) { if err := os.MkdirAll(cfg.PartialSlabDir, 0700); err != nil { return nil, fmt.Errorf("failed to create partial slab dir '%s': %v", cfg.PartialSlabDir, err) } @@ -103,7 +101,6 @@ func NewSQLStore(cfg Config, network *consensus.Network) (*SQLStore, error) { settings: make(map[string]string), walletAddress: cfg.WalletAddress, - network: network, slabPruneSigChan: make(chan struct{}, 1), lastPrunedAt: time.Now(), diff --git a/stores/sql/sqlite/migrations/main/migration_00018_settings.sql b/stores/sql/sqlite/migrations/main/migration_00018_settings.sql index 91d3b2aa0..5a8b24739 100644 --- a/stores/sql/sqlite/migrations/main/migration_00018_settings.sql +++ b/stores/sql/sqlite/migrations/main/migration_00018_settings.sql @@ -7,16 +7,17 @@ SELECT DATETIME('now'), k, v FROM ( -- upload is a combination of uploadpacking, redundancy, and contractset SELECT - "upload" as k, - json_patch( - json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE key = "uploadpacking")), - json_patch( - json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE key = "redundancy")), - json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE key = "contractset")) - ) - ) as v - WHERE json_extract(v, "$.packing") IS NOT NULL - AND json_extract(v, "$.redundancy") IS NOT NULL + "upload" as k, + json_patch( + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE key = "uploadpacking")), + json_patch( + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE key = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE key = "contractset")) + ) + ) as v + WHERE + json_extract(v, "$.packing") IS NOT NULL AND + json_extract(v, "$.redundancy") IS NOT NULL UNION ALL @@ -38,7 +39,9 @@ FROM ( ), "$.forexEndpointURL" ) as v - WHERE json_extract(v, "$.currency") IS NOT NULL AND json_extract(v, "$.threshold") IS NOT NULL + WHERE + json_extract(v, "$.currency") IS NOT NULL AND + json_extract(v, "$.threshold") IS NOT NULL ) -- delete old settings diff --git a/stores/sql_test.go b/stores/sql_test.go index 0fd280873..7eb33d07b 100644 --- a/stores/sql_test.go +++ b/stores/sql_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" @@ -180,7 +179,7 @@ func newTestSQLStore(t *testing.T, cfg testSQLStoreConfig) *testSQLStore { LongQueryDuration: 100 * time.Millisecond, LongTxDuration: 100 * time.Millisecond, RetryTransactionIntervals: []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond}, - }, &consensus.Network{}) + }) if err != nil { t.Fatal("failed to create SQLStore", err) } From 62dbcd69d1b6e1eb35c82e17dc3355fc0d6b10e6 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 11 Sep 2024 15:10:53 +0200 Subject: [PATCH 32/33] stores: fix indentation --- .../migrations/main/migration_00019_settings.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stores/sql/sqlite/migrations/main/migration_00019_settings.sql b/stores/sql/sqlite/migrations/main/migration_00019_settings.sql index 5a8b24739..98d4aa36b 100644 --- a/stores/sql/sqlite/migrations/main/migration_00019_settings.sql +++ b/stores/sql/sqlite/migrations/main/migration_00019_settings.sql @@ -30,19 +30,19 @@ FROM ( UNION ALL -- pinning renames pricepinning and removes the 'enabled' and 'forexEndpointURL' fields - SELECT - "pinned" as k, - json_remove( + SELECT + "pinned" as k, + json_remove( json_remove( (SELECT json_extract(value, "$") FROM settings WHERE key = "pricepinning"), "$.enabled" ), "$.forexEndpointURL" ) as v - WHERE + WHERE json_extract(v, "$.currency") IS NOT NULL AND json_extract(v, "$.threshold") IS NOT NULL ) -- delete old settings -DELETE FROM settings WHERE `key` IN ("uploadpacking", "redundancy", "contractset", "s3authentication", "pricepinning"); \ No newline at end of file +DELETE FROM settings WHERE `key` IN ("uploadpacking", "redundancy", "contractset", "s3authentication", "pricepinning"); From 87f0cc2316d7cc215c1496a8675b2c1d3d1e89d9 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 12 Sep 2024 13:51:27 +0200 Subject: [PATCH 33/33] bus: remove network --- bus/bus.go | 4 +--- bus/routes.go | 4 ++-- cmd/renterd/node.go | 2 +- internal/test/e2e/cluster.go | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bus/bus.go b/bus/bus.go index f180a7ee9..a1ff1034d 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -307,7 +307,6 @@ type ( type Bus struct { startTime time.Time masterKey utils.MasterKey - network *consensus.Network alerts alerts.Alerter alertMgr AlertManager @@ -337,13 +336,12 @@ type Bus struct { } // New returns a new Bus -func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, store Store, announcementMaxAge time.Duration, explorerURL string, network *consensus.Network, l *zap.Logger) (_ *Bus, err error) { +func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, store Store, announcementMaxAge time.Duration, explorerURL string, l *zap.Logger) (_ *Bus, err error) { l = l.Named("bus") b := &Bus{ startTime: time.Now(), masterKey: masterKey, - network: network, accounts: store, explorer: ibus.NewExplorer(explorerURL), diff --git a/bus/routes.go b/bus/routes.go index 646540bd8..7f10b03ac 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -1373,7 +1373,7 @@ func (b *Bus) settingsUploadHandlerGET(jc jape.Context) { us, err := b.ss.UploadSettings(jc.Request.Context()) if errors.Is(err, sql.ErrSettingNotFound) { b.logger.Warn("upload settings not found, returning defaults") - jc.Encode(api.DefaultUploadSettings(b.network.Name)) + jc.Encode(api.DefaultUploadSettings(b.cm.TipState().Network.Name)) return } else if jc.Check("failed to get upload settings", err) == nil { jc.Encode(us) @@ -1660,7 +1660,7 @@ func (b *Bus) gougingParams(ctx context.Context) (api.GougingParams, error) { us, err := b.ss.UploadSettings(ctx) if errors.Is(err, sql.ErrSettingNotFound) { - us = api.DefaultUploadSettings(b.network.Name) + us = api.DefaultUploadSettings(b.cm.TipState().Network.Name) } else if err != nil { return api.GougingParams{}, err } diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 9fad6c05e..aa36b9f24 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -386,7 +386,7 @@ func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network // create bus announcementMaxAgeHours := time.Duration(cfg.Bus.AnnouncementMaxAgeHours) * time.Hour - b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, explorerURL, network, logger) + b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, explorerURL, logger) if err != nil { return nil, nil, fmt.Errorf("failed to create bus: %w", err) } diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 951f0f8d8..735085711 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -639,7 +639,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, // create bus announcementMaxAgeHours := time.Duration(cfg.AnnouncementMaxAgeHours) * time.Hour - b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, "", network, logger) + b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, "", logger) if err != nil { return nil, nil, nil, nil, err }