diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 04504c086..a5ccf759f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -66,10 +66,10 @@ body: description: | The configuration of your bus ```bash - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/contractset - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/gouging - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/redundancy - curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/uploadpacking + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/gouging + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/pinned + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/s3 + curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/uploads ``` placeholder: Paste the output of the above commands here validations: diff --git a/README.md b/README.md index a6ff19184..e631c84ec 100644 --- a/README.md +++ b/README.md @@ -554,65 +554,6 @@ formed. } ``` -### Contract Set - -The contract set settings on the bus allow specifying a default contract set. -This contract set will be returned by the `bus` through the upload parameters, -and decides what contracts data is upload or migrated to by default. This -setting does not have a default value, it can be updated using the settings API: - -- `GET /api/bus/setting/contractset` -- `PUT /api/bus/setting/contractset` - -```json -{ - "default": "autopilot" -} -``` - -In most cases the default set should match the set from your autopilot -configuration in order for migrations to work properly. The contract set can be -overridden by passing it as a query string parameter to the worker's upload and -migrate endpoints. - -- `PUT /api/worker/objects/foo?contractset=foo` - -### Redundancy - -The default redundancy on mainnet is 30-10, on testnet it is 6-2. The redundancy -can be updated using the settings API: - -- `GET /api/bus/setting/redundancy` -- `PUT /api/bus/setting/redundancy` - -The redundancy can also be passed through query string parameters on the upload -endpoint in the worker API: - -- `PUT /api/worker/objects/foo?minshards=2&totalshards=5` - -### Gouging - -The default gouging settings are listed below. The gouging settings can be -updated using the settings API: - -- `GET /api/bus/setting/gouging` -- `PUT /api/bus/setting/gouging` - -```json -{ - "hostBlockHeightLeeway": 6, // 6 blocks - "maxContractPrice": "15000000000000000000000000", // 15 SC per contract - "maxDownloadPrice": "3000000000000000000000000000", // 3000 SC per 1 TB - "maxRPCPrice": "1000000000000000000000", // 1mS per RPC - "maxStoragePrice": "631593542824", // 3000 SC per TB per month - "maxUploadPrice": "3000000000000000000000000000", // 3000 SC per 1 TB - "migrationSurchargeMultiplier": 10, // overpay up to 10x for sectors migrations on critical slabs - "minAccountExpiry": 86400000000000, // 1 day - "minMaxEphemeralAccountBalance": "1000000000000000000000000", // 1 SC - "minPriceTableValidity": 300000000000 // 5 minutes -} -``` - ### Blocklist Unfortunately the Sia blockchain is subject to hosts that announced themselves diff --git a/api/events.go b/api/events.go index 7f14ee4c5..85fe857d9 100644 --- a/api/events.go +++ b/api/events.go @@ -19,7 +19,6 @@ const ( EventAdd = "add" EventUpdate = "update" - EventDelete = "delete" EventArchive = "archive" EventRenew = "renew" ) @@ -51,12 +50,6 @@ type ( Timestamp time.Time `json:"timestamp"` } - EventHostUpdate struct { - HostKey types.PublicKey `json:"hostKey"` - NetAddr string `json:"netAddr"` - Timestamp time.Time `json:"timestamp"` - } - EventContractSetUpdate struct { Name string `json:"name"` ToAdd []types.FileContractID `json:"toAdd"` @@ -64,15 +57,18 @@ type ( Timestamp time.Time `json:"timestamp"` } - EventSettingUpdate struct { - Key string `json:"key"` - Update interface{} `json:"update"` - Timestamp time.Time `json:"timestamp"` + EventHostUpdate struct { + HostKey types.PublicKey `json:"hostKey"` + NetAddr string `json:"netAddr"` + Timestamp time.Time `json:"timestamp"` } - EventSettingDelete struct { - Key string `json:"key"` - Timestamp time.Time `json:"timestamp"` + EventSettingUpdate struct { + GougingSettings *GougingSettings `json:"gougingSettings,omitempty"` + PinnedSettings *PinnedSettings `json:"pinnedSettings,omitempty"` + S3Settings *S3Settings `json:"s3Settings,omitempty"` + UploadSettings *UploadSettings `json:"uploadSettings,omitempty"` + Timestamp time.Time `json:"timestamp"` } ) @@ -139,15 +135,6 @@ var ( URL: url, } } - - WebhookSettingDelete = func(url string, headers map[string]string) webhooks.Webhook { - return webhooks.Webhook{ - Event: EventDelete, - Headers: headers, - Module: ModuleSetting, - URL: url, - } - } ) func ParseEventWebhook(event webhooks.Event) (interface{}, error) { @@ -202,19 +189,12 @@ func ParseEventWebhook(event webhooks.Event) (interface{}, error) { return e, nil } case ModuleSetting: - switch event.Event { - case EventUpdate: + if event.Event == EventUpdate { var e EventSettingUpdate if err := json.Unmarshal(bytes, &e); err != nil { return nil, err } return e, nil - case EventDelete: - var e EventSettingDelete - if err := json.Unmarshal(bytes, &e); err != nil { - return nil, err - } - return e, nil } } return nil, fmt.Errorf("%w: module %s event %s", ErrUnknownEvent, event.Module, event.Event) diff --git a/api/setting.go b/api/setting.go index d01acd709..c0b93e46a 100644 --- a/api/setting.go +++ b/api/setting.go @@ -9,15 +9,6 @@ import ( "go.sia.tech/core/types" ) -const ( - SettingContractSet = "contractset" - SettingGouging = "gouging" - SettingPricePinning = "pricepinning" - SettingRedundancy = "redundancy" - SettingS3Authentication = "s3authentication" - SettingUploadPacking = "uploadpacking" -) - const ( S3MinAccessKeyLen = 16 S3MaxAccessKeyLen = 128 @@ -28,15 +19,11 @@ var ( // ErrInvalidRedundancySettings is returned if the redundancy settings are // not valid ErrInvalidRedundancySettings = errors.New("invalid redundancy settings") +) - // ErrSettingNotFound is returned if a requested setting is not present in the - // database. - ErrSettingNotFound = errors.New("setting not found") - +var ( // DefaultGougingSettings define the default gouging settings the bus is - // configured with on startup. These values can be adjusted using the - // settings API. - // + // configured with on startup. DefaultGougingSettings = GougingSettings{ MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC MaxContractPrice: types.Siacoins(15), // 15 SC per contract @@ -50,46 +37,51 @@ var ( MigrationSurchargeMultiplier: 10, // 10x } - // DefaultPricePinSettings define the default price pin settings the bus is + // DefaultPinnedSettings define the default price pin settings the bus is // configured with on startup. These values can be adjusted using the // settings API. - DefaultPricePinSettings = PricePinSettings{ + DefaultPinnedSettings = PinnedSettings{ Currency: "usd", Threshold: 0.05, } - // DefaultUploadPackingSettings define the default upload packing settings - // the bus is configured with on startup. - DefaultUploadPackingSettings = UploadPackingSettings{ - Enabled: true, - SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB - } - - // DefaultRedundancySettings define the default redundancy settings the bus - // is configured with on startup. These values can be adjusted using the - // settings API. - // - // NOTE: default redundancy settings for testnet are different from mainnet. - DefaultRedundancySettings = RedundancySettings{ - MinShards: 10, - TotalShards: 30, - } - - // Same as DefaultRedundancySettings but for running on testnet networks due - // to their reduced number of hosts. + // DefaultRedundancySettingsTestnet defines redundancy settings for the + // testnet, these are lower due to the reduced number of hosts on the + // testnet. DefaultRedundancySettingsTestnet = RedundancySettings{ MinShards: 2, TotalShards: 6, } + + // DefaultS3Settings defines the 3 settings the bus is configured with on + // startup. + DefaultS3Settings = S3Settings{ + Authentication: S3AuthenticationSettings{ + V4Keypairs: map[string]string{}, + }, + } ) -type ( - // ContractSetSetting contains the default contract set used by the worker for - // uploads and migrations. - ContractSetSetting struct { - Default string `json:"default"` +// DefaultUploadSettings define the default upload settings the bus is +// configured with on startup. +func DefaultUploadSettings(network string) UploadSettings { + rs := RedundancySettings{ + MinShards: 10, + TotalShards: 30, + } + if network != "mainnet" { + rs = DefaultRedundancySettingsTestnet } + return UploadSettings{ + Packing: UploadPackingSettings{ + Enabled: true, + SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB + }, + Redundancy: rs, + } +} +type ( // GougingSettings contain some price settings used in price gouging. GougingSettings struct { // MaxRPCPrice is the maximum allowed base price for RPCs @@ -130,11 +122,11 @@ type ( MigrationSurchargeMultiplier uint64 `json:"migrationSurchargeMultiplier"` } - // PricePinSettings holds the configuration for pinning certain settings to a + // PinnedSettings holds the configuration for pinning certain settings to a // specific currency (e.g., USD). It uses the configured explorer to fetch // the current exchange rate, allowing users to set prices in USD instead of // SC. - PricePinSettings struct { + PinnedSettings struct { // Currency is the external three-letter currency code. Currency string `json:"currency"` @@ -150,6 +142,23 @@ type ( GougingSettingsPins GougingSettingsPins `json:"gougingSettingsPins"` } + // UploadSettings contains various settings related to uploads. + UploadSettings struct { + DefaultContractSet string `json:"defaultContractSet"` + Packing UploadPackingSettings `json:"packing"` + Redundancy RedundancySettings `json:"redundancy"` + } + + UploadPackingSettings struct { + Enabled bool `json:"enabled"` + SlabBufferMaxSizeSoft int64 `json:"slabBufferMaxSizeSoft"` + } + + RedundancySettings struct { + MinShards int `json:"minShards"` + TotalShards int `json:"totalShards"` + } + // AutopilotPins contains the available autopilot settings that can be // pinned. AutopilotPins struct { @@ -170,22 +179,15 @@ type ( Value float64 `json:"value"` } - // RedundancySettings contain settings that dictate an object's redundancy. - RedundancySettings struct { - MinShards int `json:"minShards"` - TotalShards int `json:"totalShards"` + // S3Settings contains various settings related to the S3 API. + S3Settings struct { + Authentication S3AuthenticationSettings `json:"authentication"` } // S3AuthenticationSettings contains S3 auth settings. S3AuthenticationSettings struct { V4Keypairs map[string]string `json:"v4Keypairs"` } - - // UploadPackingSettings contains upload packing settings. - UploadPackingSettings struct { - Enabled bool `json:"enabled"` - SlabBufferMaxSizeSoft int64 `json:"slabBufferMaxSizeSoft"` - } ) // IsPinned returns true if the pin is enabled and the value is greater than 0. @@ -194,14 +196,14 @@ func (p Pin) IsPinned() bool { } // Enabled returns true if any pins are enabled. -func (pps PricePinSettings) Enabled() bool { - if pps.GougingSettingsPins.MaxDownload.Pinned || - pps.GougingSettingsPins.MaxStorage.Pinned || - pps.GougingSettingsPins.MaxUpload.Pinned { +func (ps PinnedSettings) Enabled() bool { + if ps.GougingSettingsPins.MaxDownload.Pinned || + ps.GougingSettingsPins.MaxStorage.Pinned || + ps.GougingSettingsPins.MaxUpload.Pinned { return true } - for _, pin := range pps.Autopilots { + for _, pin := range ps.Autopilots { if pin.Allowance.Pinned { return true } @@ -211,14 +213,14 @@ func (pps PricePinSettings) Enabled() bool { } // Validate returns an error if the price pin settings are not considered valid. -func (pps PricePinSettings) Validate() error { - if !pps.Enabled() { +func (ps PinnedSettings) Validate() error { + if !ps.Enabled() { return nil } - if pps.Currency == "" { + if ps.Currency == "" { return fmt.Errorf("price pin settings must have a currency") } - if pps.Threshold <= 0 || pps.Threshold >= 1 { + if ps.Threshold <= 0 || ps.Threshold >= 1 { return fmt.Errorf("price pin settings must have a threshold between 0 and 1") } return nil @@ -246,6 +248,14 @@ func (gs GougingSettings) Validate() error { return nil } +// Validate returns an error if the upload settings are not considered valid. +func (us UploadSettings) Validate() error { + if us.Packing.Enabled && us.Packing.SlabBufferMaxSizeSoft <= 0 { + return errors.New("SlabBufferMaxSizeSoft must be greater than zero when upload packing is enabled") + } + return us.Redundancy.Validate() +} + // Redundancy returns the effective storage redundancy of the // RedundancySettings. func (rs RedundancySettings) Redundancy() float64 { @@ -279,8 +289,8 @@ func (rs RedundancySettings) Validate() error { // Validate returns an error if the authentication settings are not considered // valid. -func (s3as S3AuthenticationSettings) Validate() error { - for accessKeyID, secretAccessKey := range s3as.V4Keypairs { +func (s3s S3Settings) Validate() error { + for accessKeyID, secretAccessKey := range s3s.Authentication.V4Keypairs { if accessKeyID == "" { return fmt.Errorf("AccessKeyID cannot be empty") } else if len(accessKeyID) < S3MinAccessKeyLen || len(accessKeyID) > S3MaxAccessKeyLen { diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 2bf92f6f1..b33cf72b2 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -73,9 +73,8 @@ type Bus interface { SlabsForMigration(ctx context.Context, healthCutoff float64, set string, limit int) ([]api.UnhealthySlab, error) // settings - UpdateSetting(ctx context.Context, key string, value interface{}) error GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) - RedundancySettings(ctx context.Context) (rs api.RedundancySettings, err error) + UploadSettings(ctx context.Context) (us api.UploadSettings, err error) // syncer SyncerPeers(ctx context.Context) (resp []string, err error) @@ -810,10 +809,10 @@ func (ap *Autopilot) buildState(ctx context.Context) (*contractor.MaintenanceSta return nil, fmt.Errorf("could not fetch consensus state, err: %v", err) } - // fetch redundancy settings - rs, err := ap.bus.RedundancySettings(ctx) + // fetch upload settings + us, err := ap.bus.UploadSettings(ctx) if err != nil { - return nil, fmt.Errorf("could not fetch redundancy settings, err: %v", err) + return nil, fmt.Errorf("could not fetch upload settings, err: %v", err) } // fetch gouging settings @@ -863,7 +862,7 @@ func (ap *Autopilot) buildState(ctx context.Context) (*contractor.MaintenanceSta return &contractor.MaintenanceState{ GS: gs, - RS: rs, + RS: us.Redundancy, AP: autopilot, Address: address, diff --git a/bus/bus.go b/bus/bus.go index 79623615c..a1ff1034d 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -5,13 +5,11 @@ package bus import ( "context" - "encoding/json" "errors" "fmt" "math" "net" "net/http" - "strings" "time" "go.sia.tech/core/consensus" @@ -36,7 +34,6 @@ import ( "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" - "golang.org/x/crypto/blake2b" ) const ( @@ -289,10 +286,17 @@ type ( // A SettingStore stores settings. SettingStore interface { - DeleteSetting(ctx context.Context, key string) error - Setting(ctx context.Context, key string) (string, error) - Settings(ctx context.Context) ([]string, error) - UpdateSetting(ctx context.Context, key, value string) error + GougingSettings(ctx context.Context) (api.GougingSettings, error) + UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error + + PinnedSettings(ctx context.Context) (api.PinnedSettings, error) + UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error + + UploadSettings(ctx context.Context) (api.UploadSettings, error) + UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error + + S3Settings(ctx context.Context) (api.S3Settings, error) + UpdateS3Settings(ctx context.Context, s3as api.S3Settings) error } WalletMetricsRecorder interface { @@ -359,11 +363,6 @@ func New(ctx context.Context, masterKey [32]byte, am AlertManager, wm WebhooksMa rhp3: rhp3.New(rhp.NewFallbackDialer(store, net.Dialer{}, l), l), } - // init settings - if err := b.initSettings(ctx); err != nil { - return nil, err - } - // create contract locker b.contractLocker = ibus.NewContractLocker() @@ -474,10 +473,14 @@ func (b *Bus) Handler() http.Handler { "DELETE /sectors/:hk/:root": b.sectorsHostRootHandlerDELETE, - "GET /settings": b.settingsHandlerGET, - "GET /setting/:key": b.settingKeyHandlerGET, - "PUT /setting/:key": b.settingKeyHandlerPUT, - "DELETE /setting/:key": b.settingKeyHandlerDELETE, + "GET /settings/gouging": b.settingsGougingHandlerGET, + "PUT /settings/gouging": b.settingsGougingHandlerPUT, + "GET /settings/pinned": b.settingsPinnedHandlerGET, + "PUT /settings/pinned": b.settingsPinnedHandlerPUT, + "GET /settings/s3": b.settingsS3HandlerGET, + "PUT /settings/s3": b.settingsS3HandlerPUT, + "GET /settings/upload": b.settingsUploadHandlerGET, + "PUT /settings/upload": b.settingsUploadHandlerPUT, "POST /slabs/migration": b.slabsMigrationHandlerPOST, "GET /slabs/partial/:key": b.slabsPartialHandlerGET, @@ -578,9 +581,11 @@ func (b *Bus) broadcastContract(ctx context.Context, fcid types.FileContractID) return types.TransactionID{}, fmt.Errorf("couldn't fetch contract; %w", err) } + // derive the renter key + renterKey := b.masterKey.DeriveContractKey(c.HostKey) + // fetch revision - rk := b.deriveRenterKey(c.HostKey) - rev, err := b.rhp2.SignedRevision(ctx, c.HostIP, c.HostKey, rk, fcid, time.Minute) + rev, err := b.rhp2.SignedRevision(ctx, c.HostIP, c.HostKey, renterKey, fcid, time.Minute) if err != nil { return types.TransactionID{}, fmt.Errorf("couldn't fetch revision; %w", err) } @@ -621,7 +626,7 @@ func (b *Bus) broadcastContract(ctx context.Context, fcid types.FileContractID) func (b *Bus) formContract(ctx context.Context, hostSettings rhpv2.HostSettings, renterAddress types.Address, renterFunds, hostCollateral types.Currency, hostKey types.PublicKey, hostIP string, endHeight uint64) (rhpv2.ContractRevision, error) { // derive the renter key - renterKey := b.deriveRenterKey(hostKey) + renterKey := b.masterKey.DeriveContractKey(hostKey) // prepare the transaction cs := b.cm.TipState() @@ -662,140 +667,11 @@ func (b *Bus) formContract(ctx context.Context, hostSettings rhpv2.HostSettings, return contract, nil } -// initSettings loads the default settings if the setting is not already set and -// ensures the settings are valid -func (b *Bus) initSettings(ctx context.Context) error { - // testnets have different redundancy settings - defaultRedundancySettings := api.DefaultRedundancySettings - if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { - defaultRedundancySettings = api.DefaultRedundancySettingsTestnet - } - - // load default settings if the setting is not already set - for key, value := range map[string]interface{}{ - api.SettingGouging: api.DefaultGougingSettings, - api.SettingPricePinning: api.DefaultPricePinSettings, - api.SettingRedundancy: defaultRedundancySettings, - api.SettingUploadPacking: api.DefaultUploadPackingSettings, - } { - if _, err := b.ss.Setting(ctx, key); errors.Is(err, api.ErrSettingNotFound) { - if bytes, err := json.Marshal(value); err != nil { - panic("failed to marshal default settings") // should never happen - } else if err := b.ss.UpdateSetting(ctx, key, string(bytes)); err != nil { - return err - } - } - } - - // check redundancy settings for validity - var rs api.RedundancySettings - if rss, err := b.ss.Setting(ctx, api.SettingRedundancy); err != nil { - return err - } else if err := json.Unmarshal([]byte(rss), &rs); err != nil { - return err - } else if err := rs.Validate(); err != nil { - b.logger.Warn(fmt.Sprintf("invalid redundancy setting found '%v', overwriting the redundancy settings with the default settings", rss)) - bytes, _ := json.Marshal(defaultRedundancySettings) - if err := b.ss.UpdateSetting(ctx, api.SettingRedundancy, string(bytes)); err != nil { - return err - } - } - - // check gouging settings for validity - var gs api.GougingSettings - if gss, err := b.ss.Setting(ctx, api.SettingGouging); err != nil { - return err - } else if err := json.Unmarshal([]byte(gss), &gs); err != nil { - return err - } else if err := gs.Validate(); err != nil { - // compat: apply default EA gouging settings - gs.MinMaxEphemeralAccountBalance = api.DefaultGougingSettings.MinMaxEphemeralAccountBalance - gs.MinPriceTableValidity = api.DefaultGougingSettings.MinPriceTableValidity - gs.MinAccountExpiry = api.DefaultGougingSettings.MinAccountExpiry - if err := gs.Validate(); err == nil { - b.logger.Info(fmt.Sprintf("updating gouging settings with default EA settings: %+v", gs)) - bytes, _ := json.Marshal(gs) - if err := b.ss.UpdateSetting(ctx, api.SettingGouging, string(bytes)); err != nil { - return err - } - } else { - // compat: apply default host block leeway settings - gs.HostBlockHeightLeeway = api.DefaultGougingSettings.HostBlockHeightLeeway - if err := gs.Validate(); err == nil { - b.logger.Info(fmt.Sprintf("updating gouging settings with default HostBlockHeightLeeway settings: %v", gs)) - bytes, _ := json.Marshal(gs) - if err := b.ss.UpdateSetting(ctx, api.SettingGouging, string(bytes)); err != nil { - return err - } - } else { - b.logger.Warn(fmt.Sprintf("invalid gouging setting found '%v', overwriting the gouging settings with the default settings", gss)) - bytes, _ := json.Marshal(api.DefaultGougingSettings) - if err := b.ss.UpdateSetting(ctx, api.SettingGouging, string(bytes)); err != nil { - return err - } - } - } - } - - // compat: default price pin settings - var pps api.PricePinSettings - if pss, err := b.ss.Setting(ctx, api.SettingPricePinning); err != nil { - return err - } else if err := json.Unmarshal([]byte(pss), &pps); err != nil { - return err - } else if err := pps.Validate(); err != nil { - // overwrite values with defaults - var updates []string - if pps.Currency == "" { - pps.Currency = api.DefaultPricePinSettings.Currency - updates = append(updates, fmt.Sprintf("set PricePinSettings.Currency to %v", pps.Currency)) - } - if pps.Threshold == 0 { - pps.Threshold = api.DefaultPricePinSettings.Threshold - updates = append(updates, fmt.Sprintf("set PricePinSettings.Threshold to %v", pps.Threshold)) - } - - var updated []byte - if err := pps.Validate(); err == nil { - b.logger.Info(fmt.Sprintf("updating price pinning settings with default values: %v", strings.Join(updates, ", "))) - updated, _ = json.Marshal(pps) - } else { - b.logger.Warn(fmt.Sprintf("updated price pinning settings are invalid (%v), they have been overwritten with the default settings", err)) - updated, _ = json.Marshal(api.DefaultPricePinSettings) - } - if err := b.ss.UpdateSetting(ctx, api.SettingPricePinning, string(updated)); err != nil { - return fmt.Errorf("failed to update setting '%v': %w", api.SettingPricePinning, err) - } - } else if pps.Enabled() && !b.explorer.Enabled() { - return fmt.Errorf("price pinning can not be enabled, %w", api.ErrExplorerDisabled) - } - - return nil -} - func (b *Bus) isPassedV2AllowHeight() bool { cs := b.cm.TipState() return cs.Index.Height >= cs.Network.HardforkV2.AllowHeight } -func (b *Bus) deriveRenterKey(hostKey types.PublicKey) types.PrivateKey { - seed := blake2b.Sum256(append(b.deriveSubKey("renterkey"), hostKey[:]...)) - pk := types.NewPrivateKeyFromSeed(seed[:]) - for i := range seed { - seed[i] = 0 - } - return pk -} - -func (b *Bus) deriveSubKey(purpose string) types.PrivateKey { - seed := blake2b.Sum256(append(b.masterKey[:], []byte(purpose)...)) - pk := types.NewPrivateKeyFromSeed(seed[:]) - for i := range seed { - seed[i] = 0 - } - return pk -} - func (b *Bus) prepareRenew(cs consensus.State, revision types.FileContractRevision, hostAddress, renterAddress types.Address, renterFunds, minNewCollateral, maxFundAmount types.Currency, endHeight, expectedStorage uint64) rhp3.PrepareRenewFn { return func(pt rhpv3.HostPriceTable) ([]types.Hash256, []types.Transaction, types.Currency, rhp3.DiscardTxnFn, error) { // create the final revision from the provided revision @@ -844,6 +720,9 @@ func (b *Bus) prepareRenew(cs consensus.State, revision types.FileContractRevisi } func (b *Bus) renewContract(ctx context.Context, cs consensus.State, gp api.GougingParams, c api.ContractMetadata, hs rhpv2.HostSettings, renterFunds, minNewCollateral, maxFundAmount types.Currency, endHeight, expectedNewStorage uint64) (rhpv2.ContractRevision, types.Currency, types.Currency, error) { + // derive the renter key + renterKey := b.masterKey.DeriveContractKey(c.HostKey) + // acquire contract lock indefinitely and defer the release lockID, err := b.contractLocker.Acquire(ctx, lockingPriorityRenew, c.ID, time.Duration(math.MaxInt64)) if err != nil { @@ -863,7 +742,6 @@ func (b *Bus) renewContract(ctx context.Context, cs consensus.State, gp api.Goug // renew contract gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState, gp.TransactionFee, nil, nil) - renterKey := b.deriveRenterKey(c.HostKey) prepareRenew := b.prepareRenew(cs, rev, hs.Address, b.w.Address(), renterFunds, minNewCollateral, maxFundAmount, endHeight, expectedNewStorage) newRevision, txnSet, contractPrice, fundAmount, err := b.rhp3.Renew(ctx, gc, rev, renterKey, c.HostKey, c.SiamuxAddr, prepareRenew, b.w.SignTransaction) if err != nil { diff --git a/bus/client/metrics.go b/bus/client/metrics.go index 10bc2fbca..3923145f0 100644 --- a/bus/client/metrics.go +++ b/bus/client/metrics.go @@ -4,15 +4,14 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" - "io" "net/http" "net/url" "time" "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" ) func (c *Client) ContractMetrics(ctx context.Context, start time.Time, n uint64, interval time.Duration, opts api.ContractMetricsQueryOpts) ([]api.ContractMetric, error) { @@ -130,16 +129,8 @@ func (c *Client) PruneMetrics(ctx context.Context, metric string, cutoff time.Ti panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return errors.New(string(err)) - } - return nil + _, _, err = utils.DoRequest(req, nil) + return err } func (c *Client) recordMetric(ctx context.Context, key string, d interface{}) error { @@ -159,17 +150,8 @@ func (c *Client) recordMetric(ctx context.Context, key string, d interface{}) er panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return errors.New(string(err)) - } - return nil + _, _, err = utils.DoRequest(req, nil) + return err } func (c *Client) metric(ctx context.Context, key string, values url.Values, res interface{}) error { @@ -185,16 +167,6 @@ func (c *Client) metric(ctx context.Context, key string, values url.Values, res panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - - if resp.StatusCode != 200 && resp.StatusCode != 206 { - err, _ := io.ReadAll(resp.Body) - return errors.New(string(err)) - } - return json.NewDecoder(resp.Body).Decode(&res) + _, _, err = utils.DoRequest(req, &res) + return err } diff --git a/bus/client/settings.go b/bus/client/settings.go index 22714cf8b..74d5f34c9 100644 --- a/bus/client/settings.go +++ b/bus/client/settings.go @@ -2,65 +2,50 @@ package client import ( "context" - "fmt" "go.sia.tech/renterd/api" ) -// ContractSetSettings returns the contract set settings. -func (c *Client) ContractSetSettings(ctx context.Context) (gs api.ContractSetSetting, err error) { - err = c.Setting(ctx, api.SettingContractSet, &gs) - return -} - -// DeleteSetting will delete the setting with given key. -func (c *Client) DeleteSetting(ctx context.Context, key string) error { - return c.c.WithContext(ctx).DELETE(fmt.Sprintf("/setting/%s", key)) -} - // GougingSettings returns the gouging settings. func (c *Client) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { - err = c.Setting(ctx, api.SettingGouging, &gs) + err = c.c.WithContext(ctx).GET("/settings/gouging", &gs) return } -// PricePinningSettings returns the contract set settings. -func (c *Client) PricePinningSettings(ctx context.Context) (pps api.PricePinSettings, err error) { - err = c.Setting(ctx, api.SettingPricePinning, &pps) - return +// UpdateGougingSettings updates the given setting. +func (c *Client) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { + return c.c.WithContext(ctx).PUT("/settings/gouging", gs) } -// RedundancySettings returns the redundancy settings. -func (c *Client) RedundancySettings(ctx context.Context) (rs api.RedundancySettings, err error) { - err = c.Setting(ctx, api.SettingRedundancy, &rs) +// PinnedSettings returns the pinned settings. +func (c *Client) PinnedSettings(ctx context.Context) (ps api.PinnedSettings, err error) { + err = c.c.WithContext(ctx).GET("/settings/pinned", &ps) return } -// S3AuthenticationSettings returns the S3 authentication settings. -func (c *Client) S3AuthenticationSettings(ctx context.Context) (as api.S3AuthenticationSettings, err error) { - err = c.Setting(ctx, api.SettingS3Authentication, &as) - return +// UpdatePinnedSettings updates the given setting. +func (c *Client) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { + return c.c.WithContext(ctx).PUT("/settings/pinned", ps) } -// Setting returns the value for the setting with given key. -func (c *Client) Setting(ctx context.Context, key string, value interface{}) (err error) { - err = c.c.WithContext(ctx).GET(fmt.Sprintf("/setting/%s", key), &value) +// S3Settings returns the S3 settings. +func (c *Client) S3Settings(ctx context.Context) (as api.S3Settings, err error) { + err = c.c.WithContext(ctx).GET("/settings/s3", &as) return } -// Settings returns the keys of all settings. -func (c *Client) Settings(ctx context.Context) (settings []string, err error) { - err = c.c.WithContext(ctx).GET("/settings", &settings) - return +// UpdateS3Settings updates the given setting. +func (c *Client) UpdateS3Settings(ctx context.Context, as api.S3Settings) error { + return c.c.WithContext(ctx).PUT("/settings/s3", as) } -// UpdateSetting will update the given setting under the given key. -func (c *Client) UpdateSetting(ctx context.Context, key string, value interface{}) error { - return c.c.WithContext(ctx).PUT(fmt.Sprintf("/setting/%s", key), value) +// UploadSettings returns the upload settings. +func (c *Client) UploadSettings(ctx context.Context) (css api.UploadSettings, err error) { + err = c.c.WithContext(ctx).GET("/settings/upload", &css) + return } -// UploadPackingSettings returns the upload packing settings. -func (c *Client) UploadPackingSettings(ctx context.Context) (ups api.UploadPackingSettings, err error) { - err = c.Setting(ctx, api.SettingUploadPacking, &ups) - return +// UpdateUploadSettings update the given setting. +func (c *Client) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error { + return c.c.WithContext(ctx).PUT("/settings/upload", us) } diff --git a/bus/client/slabs.go b/bus/client/slabs.go index db5c0023a..b0fc8837e 100644 --- a/bus/client/slabs.go +++ b/bus/client/slabs.go @@ -3,7 +3,6 @@ package client import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -12,6 +11,7 @@ import ( "time" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" ) @@ -33,18 +33,8 @@ func (c *Client) AddPartialSlab(ctx context.Context, data []byte, minShards, tot panic(err) } req.SetBasicAuth("", c.c.WithContext(ctx).Password) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, false, err - } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return nil, false, errors.New(string(err)) - } var apsr api.AddPartialSlabResponse - err = json.NewDecoder(resp.Body).Decode(&apsr) + _, _, err = utils.DoRequest(req, &apsr) if err != nil { return nil, false, err } diff --git a/bus/routes.go b/bus/routes.go index c1c5d9d70..7f10b03ac 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -16,6 +16,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" rhp3 "go.sia.tech/renterd/internal/rhp/v3" + "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/internal/gouging" rhp2 "go.sia.tech/renterd/internal/rhp/v2" @@ -33,15 +34,6 @@ import ( "go.uber.org/zap" ) -func (b *Bus) fetchSetting(ctx context.Context, key string, value interface{}) error { - if val, err := b.ss.Setting(ctx, key); err != nil { - return fmt.Errorf("could not get contract set settings: %w", err) - } else if err := json.Unmarshal([]byte(val), &value); err != nil { - b.logger.Panicf("failed to unmarshal %v settings '%s': %v", key, val, err) - } - return nil -} - func (b *Bus) accountsFundHandler(jc jape.Context) { var req api.AccountsFundRequest if jc.Decode(&req) != nil { @@ -800,7 +792,8 @@ func (b *Bus) contractPruneHandlerPOST(jc jape.Context) { } // prune the contract - rev, spending, pruned, remaining, err := b.rhp2.PruneContract(pruneCtx, b.deriveRenterKey(c.HostKey), gc, c.HostIP, c.HostKey, fcid, c.RevisionNumber, func(fcid types.FileContractID, roots []types.Hash256) ([]uint64, error) { + rk := b.masterKey.DeriveContractKey(c.HostKey) + rev, spending, pruned, remaining, err := b.rhp2.PruneContract(pruneCtx, rk, gc, c.HostIP, c.HostKey, fcid, c.RevisionNumber, func(fcid types.FileContractID, roots []types.Hash256) ([]uint64, error) { indices, err := b.ms.PrunableContractRoots(ctx, fcid, roots) if err != nil { return nil, err @@ -1296,6 +1289,146 @@ func (b *Bus) packedSlabsHandlerDonePOST(jc jape.Context) { jc.Check("failed to mark packed slab(s) as uploaded", b.ms.MarkPackedSlabsUploaded(jc.Request.Context(), psrp.Slabs)) } +func (b *Bus) settingsGougingHandlerGET(jc jape.Context) { + gs, err := b.ss.GougingSettings(jc.Request.Context()) + if errors.Is(err, sql.ErrSettingNotFound) { + b.logger.Warn("gouging settings not found, returning defaults") + jc.Encode(api.DefaultGougingSettings) + return + } else if jc.Check("failed to get gouging settings", err) == nil { + jc.Encode(gs) + } +} + +func (b *Bus) settingsGougingHandlerPUT(jc jape.Context) { + var gs api.GougingSettings + if jc.Decode(&gs) != nil { + return + } else if err := gs.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update gouging settings, error: %v", err), http.StatusBadRequest) + return + } else if jc.Check("could not update gouging settings", b.ss.UpdateGougingSettings(jc.Request.Context(), gs)) == nil { + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + GougingSettings: &gs, + Timestamp: time.Now().UTC(), + }, + }) + b.pinMgr.TriggerUpdate() + } +} + +func (b *Bus) settingsPinnedHandlerGET(jc jape.Context) { + ps, err := b.ss.PinnedSettings(jc.Request.Context()) + if errors.Is(err, sql.ErrSettingNotFound) { + b.logger.Warn("pinned settings not found, returning defaults") + jc.Encode(api.DefaultPinnedSettings) + return + } else if jc.Check("failed to get pinned settings", err) == nil { + // populate the Autopilots map with the current autopilots + aps, err := b.as.Autopilots(jc.Request.Context()) + if jc.Check("failed to fetch autopilots", err) != nil { + return + } + if ps.Autopilots == nil { + ps.Autopilots = make(map[string]api.AutopilotPins) + } + for _, ap := range aps { + if _, exists := ps.Autopilots[ap.ID]; !exists { + ps.Autopilots[ap.ID] = api.AutopilotPins{} + } + } + jc.Encode(ps) + } +} + +func (b *Bus) settingsPinnedHandlerPUT(jc jape.Context) { + var ps api.PinnedSettings + if jc.Decode(&ps) != nil { + return + } else if err := ps.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update pinned settings, error: %v", err), http.StatusBadRequest) + return + } else if ps.Enabled() && !b.explorer.Enabled() { + jc.Error(fmt.Errorf("can't enable price pinning, %w", api.ErrExplorerDisabled), http.StatusBadRequest) + return + } + + if jc.Check("could not update pinned settings", b.ss.UpdatePinnedSettings(jc.Request.Context(), ps)) == nil { + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + PinnedSettings: &ps, + Timestamp: time.Now().UTC(), + }, + }) + b.pinMgr.TriggerUpdate() + } +} + +func (b *Bus) settingsUploadHandlerGET(jc jape.Context) { + us, err := b.ss.UploadSettings(jc.Request.Context()) + if errors.Is(err, sql.ErrSettingNotFound) { + b.logger.Warn("upload settings not found, returning defaults") + jc.Encode(api.DefaultUploadSettings(b.cm.TipState().Network.Name)) + return + } else if jc.Check("failed to get upload settings", err) == nil { + jc.Encode(us) + } +} + +func (b *Bus) settingsUploadHandlerPUT(jc jape.Context) { + var us api.UploadSettings + if jc.Decode(&us) != nil { + return + } else if err := us.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update upload settings, error: %v", err), http.StatusBadRequest) + return + } else if jc.Check("could not update upload settings", b.ss.UpdateUploadSettings(jc.Request.Context(), us)) == nil { + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + UploadSettings: &us, + Timestamp: time.Now().UTC(), + }, + }) + } +} + +func (b *Bus) settingsS3HandlerGET(jc jape.Context) { + s3s, err := b.ss.S3Settings(jc.Request.Context()) + if errors.Is(err, sql.ErrSettingNotFound) { + b.logger.Warn("S3 settings not found, returning defaults") + jc.Encode(api.DefaultS3Settings) + return + } else if jc.Check("failed to get S3 settings", err) == nil { + jc.Encode(s3s) + } +} + +func (b *Bus) settingsS3HandlerPUT(jc jape.Context) { + var s3s api.S3Settings + if jc.Decode(&s3s) != nil { + return + } else if err := s3s.Validate(); err != nil { + jc.Error(fmt.Errorf("couldn't update S3 settings, error: %v", err), http.StatusBadRequest) + return + } else if jc.Check("could not update S3 settings", b.ss.UpdateS3Settings(jc.Request.Context(), s3s)) == nil { + b.broadcastAction(webhooks.Event{ + Module: api.ModuleSetting, + Event: api.EventUpdate, + Payload: api.EventSettingUpdate{ + S3Settings: &s3s, + Timestamp: time.Now().UTC(), + }, + }) + } +} + func (b *Bus) sectorsHostRootHandlerDELETE(jc jape.Context) { var hk types.PublicKey var root types.Hash256 @@ -1429,165 +1562,17 @@ func (b *Bus) slabsPartialHandlerPOST(jc jape.Context) { if jc.Check("failed to add partial slab", err) != nil { return } - var pus api.UploadPackingSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingUploadPacking, &pus); err != nil && !errors.Is(err, api.ErrSettingNotFound) { + us, err := b.ss.UploadSettings(jc.Request.Context()) + if err != nil { jc.Error(fmt.Errorf("could not get upload packing settings: %w", err), http.StatusInternalServerError) return } jc.Encode(api.AddPartialSlabResponse{ Slabs: slabs, - SlabBufferMaxSizeSoftReached: bufferSize >= pus.SlabBufferMaxSizeSoft, + SlabBufferMaxSizeSoftReached: bufferSize >= us.Packing.SlabBufferMaxSizeSoft, }) } -func (b *Bus) settingsHandlerGET(jc jape.Context) { - if settings, err := b.ss.Settings(jc.Request.Context()); jc.Check("couldn't load settings", err) == nil { - jc.Encode(settings) - } -} - -func (b *Bus) settingKeyHandlerGET(jc jape.Context) { - jc.Custom(nil, (any)(nil)) - - key := jc.PathParam("key") - if key == "" { - jc.Error(errors.New("path parameter 'key' can not be empty"), http.StatusBadRequest) - return - } - - setting, err := b.ss.Setting(jc.Request.Context(), jc.PathParam("key")) - if errors.Is(err, api.ErrSettingNotFound) { - jc.Error(err, http.StatusNotFound) - return - } else if err != nil { - jc.Error(err, http.StatusInternalServerError) - return - } - resp := []byte(setting) - - // populate autopilots of price pinning settings with defaults for better DX - if key == api.SettingPricePinning { - var pps api.PricePinSettings - err = json.Unmarshal([]byte(setting), &pps) - if jc.Check("failed to unmarshal price pinning settings", err) != nil { - return - } else if pps.Autopilots == nil { - pps.Autopilots = make(map[string]api.AutopilotPins) - } - // populate the Autopilots map with the current autopilots - aps, err := b.as.Autopilots(jc.Request.Context()) - if jc.Check("failed to fetch autopilots", err) != nil { - return - } - for _, ap := range aps { - if _, exists := pps.Autopilots[ap.ID]; !exists { - pps.Autopilots[ap.ID] = api.AutopilotPins{} - } - } - // encode the settings back - resp, err = json.Marshal(pps) - if jc.Check("failed to marshal price pinning settings", err) != nil { - return - } - } - jc.ResponseWriter.Header().Set("Content-Type", "application/json") - jc.ResponseWriter.Write(resp) -} - -func (b *Bus) settingKeyHandlerPUT(jc jape.Context) { - key := jc.PathParam("key") - if key == "" { - jc.Error(errors.New("path parameter 'key' can not be empty"), http.StatusBadRequest) - return - } - - var value interface{} - if jc.Decode(&value) != nil { - return - } - - data, err := json.Marshal(value) - if err != nil { - jc.Error(fmt.Errorf("couldn't marshal the given value, error: %v", err), http.StatusBadRequest) - return - } - - switch key { - case api.SettingGouging: - var gs api.GougingSettings - if err := json.Unmarshal(data, &gs); err != nil { - jc.Error(fmt.Errorf("couldn't update gouging settings, invalid request body, %t", value), http.StatusBadRequest) - return - } else if err := gs.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update gouging settings, error: %v", err), http.StatusBadRequest) - return - } - b.pinMgr.TriggerUpdate() - case api.SettingRedundancy: - var rs api.RedundancySettings - if err := json.Unmarshal(data, &rs); err != nil { - jc.Error(fmt.Errorf("couldn't update redundancy settings, invalid request body"), http.StatusBadRequest) - return - } else if err := rs.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update redundancy settings, error: %v", err), http.StatusBadRequest) - return - } - case api.SettingS3Authentication: - var s3as api.S3AuthenticationSettings - if err := json.Unmarshal(data, &s3as); err != nil { - jc.Error(fmt.Errorf("couldn't update s3 authentication settings, invalid request body"), http.StatusBadRequest) - return - } else if err := s3as.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update s3 authentication settings, error: %v", err), http.StatusBadRequest) - return - } - case api.SettingPricePinning: - var pps api.PricePinSettings - if err := json.Unmarshal(data, &pps); err != nil { - jc.Error(fmt.Errorf("couldn't update price pinning settings, invalid request body"), http.StatusBadRequest) - return - } else if err := pps.Validate(); err != nil { - jc.Error(fmt.Errorf("couldn't update price pinning settings, invalid settings, error: %v", err), http.StatusBadRequest) - return - } else if pps.Enabled() && !b.explorer.Enabled() { - jc.Error(fmt.Errorf("pinning can not be enabled, %w", api.ErrExplorerDisabled), http.StatusBadRequest) - return - } - b.pinMgr.TriggerUpdate() - } - - if jc.Check("could not update setting", b.ss.UpdateSetting(jc.Request.Context(), key, string(data))) == nil { - b.broadcastAction(webhooks.Event{ - Module: api.ModuleSetting, - Event: api.EventUpdate, - Payload: api.EventSettingUpdate{ - Key: key, - Update: value, - Timestamp: time.Now().UTC(), - }, - }) - } -} - -func (b *Bus) settingKeyHandlerDELETE(jc jape.Context) { - key := jc.PathParam("key") - if key == "" { - jc.Error(errors.New("path parameter 'key' can not be empty"), http.StatusBadRequest) - return - } - - if jc.Check("could not delete setting", b.ss.DeleteSetting(jc.Request.Context(), key)) == nil { - b.broadcastAction(webhooks.Event{ - Module: api.ModuleSetting, - Event: api.EventDelete, - Payload: api.EventSettingDelete{ - Key: key, - Timestamp: time.Now().UTC(), - }, - }) - } -} - func (b *Bus) contractIDAncestorsHandler(jc jape.Context) { var fcid types.FileContractID if jc.DecodeParam("id", &fcid) != nil { @@ -1622,22 +1607,12 @@ func (b *Bus) paramsHandlerUploadGET(jc jape.Context) { return } - var contractSet string - var css api.ContractSetSetting - if err := b.fetchSetting(jc.Request.Context(), api.SettingContractSet, &css); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - jc.Error(fmt.Errorf("could not get contract set settings: %w", err), http.StatusInternalServerError) - return - } else if err == nil { - contractSet = css.Default - } - var uploadPacking bool - var pus api.UploadPackingSettings - if err := b.fetchSetting(jc.Request.Context(), api.SettingUploadPacking, &pus); err != nil && !errors.Is(err, api.ErrSettingNotFound) { - jc.Error(fmt.Errorf("could not get upload packing settings: %w", err), http.StatusInternalServerError) - return - } else if err == nil { - uploadPacking = pus.Enabled + var contractSet string + us, err := b.ss.UploadSettings(jc.Request.Context()) + if jc.Check("could not get upload settings", err) == nil { + contractSet = us.DefaultContractSet + uploadPacking = us.Packing.Enabled } jc.Encode(api.UploadParams{ @@ -1676,18 +1651,18 @@ func (b *Bus) paramsHandlerGougingGET(jc jape.Context) { } func (b *Bus) gougingParams(ctx context.Context) (api.GougingParams, error) { - var gs api.GougingSettings - if gss, err := b.ss.Setting(ctx, api.SettingGouging); err != nil { + gs, err := b.ss.GougingSettings(ctx) + if errors.Is(err, sql.ErrSettingNotFound) { + gs = api.DefaultGougingSettings + } else if err != nil { return api.GougingParams{}, err - } else if err := json.Unmarshal([]byte(gss), &gs); err != nil { - b.logger.Panicf("failed to unmarshal gouging settings '%s': %v", gss, err) } - var rs api.RedundancySettings - if rss, err := b.ss.Setting(ctx, api.SettingRedundancy); err != nil { + us, err := b.ss.UploadSettings(ctx) + if errors.Is(err, sql.ErrSettingNotFound) { + us = api.DefaultUploadSettings(b.cm.TipState().Network.Name) + } else if err != nil { return api.GougingParams{}, err - } else if err := json.Unmarshal([]byte(rss), &rs); err != nil { - b.logger.Panicf("failed to unmarshal redundancy settings '%s': %v", rss, err) } cs, err := b.consensusState(ctx) @@ -1698,7 +1673,7 @@ func (b *Bus) gougingParams(ctx context.Context) (api.GougingParams, error) { return api.GougingParams{ ConsensusState: cs, GougingSettings: gs, - RedundancySettings: rs, + RedundancySettings: us.Redundancy, TransactionFee: b.cm.RecommendedFee(), }, nil } diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 2cc84523e..aa36b9f24 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -378,11 +378,13 @@ func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network // to ensure contracts formed by the bus can be renewed by the autopilot masterKey := blake2b.Sum256(append([]byte("worker"), pk...)) - // create bus + // get explorer URL var explorerURL string if !cfg.Explorer.Disable { explorerURL = cfg.Explorer.URL } + + // create bus announcementMaxAgeHours := time.Duration(cfg.Bus.AnnouncementMaxAgeHours) * time.Hour b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, explorerURL, logger) if err != nil { diff --git a/go.mod b/go.mod index 77870cda4..908c20ad3 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/gotd/contrib v0.20.0 github.com/klauspost/reedsolomon v1.12.3 - github.com/mattn/go-sqlite3 v1.14.22 + github.com/mattn/go-sqlite3 v1.14.23 github.com/minio/minio-go/v7 v7.0.76 github.com/montanaflynn/stats v0.7.1 github.com/shopspring/decimal v1.4.0 @@ -19,11 +19,11 @@ require ( go.sia.tech/hostd v1.1.3-0.20240903081107-6e044db95238 go.sia.tech/jape v0.12.1 go.sia.tech/mux v1.2.0 - go.sia.tech/web/renterd v0.60.1 + go.sia.tech/web/renterd v0.61.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.26.0 - golang.org/x/sys v0.24.0 - golang.org/x/term v0.23.0 + golang.org/x/crypto v0.27.0 + golang.org/x/sys v0.25.0 + golang.org/x/term v0.24.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/frand v1.4.2 ) @@ -50,7 +50,7 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.28.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.22.0 // indirect nhooyr.io/websocket v1.8.17 // indirect diff --git a/go.sum b/go.sum index f393bfbd4..f32b39629 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= +github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.76 h1:9nxHH2XDai61cT/EFhyIw/wW4vJfpPNvl7lSFpRt+Ng= @@ -86,8 +86,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/renterd v0.60.1 h1:KJ/DgYKES29HoRd4/XY/G9CzTrHpMANCRCffIYc6Sxg= -go.sia.tech/web/renterd v0.60.1/go.mod h1:SWwKoAJvLxiHjTXsNPKX3RLiQzJb/vxwcpku3F78MO8= +go.sia.tech/web/renterd v0.61.0 h1:DmSGkpbaqodKvP4Mn79lLeZF2xqcWFQRrT2xPuLf8Uo= +go.sia.tech/web/renterd v0.61.0/go.mod h1:VWfvYtmdJGfrqSoNRO3NoOjUij+RB/xNO4M0HqIf1+M= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -95,8 +95,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -108,13 +108,13 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/internal/bus/explorer.go b/internal/bus/explorer.go index 5b31910c6..a4f7374d6 100644 --- a/internal/bus/explorer.go +++ b/internal/bus/explorer.go @@ -2,13 +2,11 @@ package bus import ( "context" - "encoding/json" - "errors" "fmt" - "io" "net/http" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" ) type ( @@ -48,25 +46,6 @@ func (e *Explorer) SiacoinExchangeRate(ctx context.Context, currency string) (ra } req.Header.Set("Accept", "application/json") - // create http client - resp, err := http.DefaultClient.Do(req) - if err != nil { - return 0, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - // check status code - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - var errorMessage string - if err := json.NewDecoder(io.LimitReader(resp.Body, 1024)).Decode(&errorMessage); err != nil { - return 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - return 0, errors.New(errorMessage) - } - - // decode exchange rate - if err := json.NewDecoder(resp.Body).Decode(&rate); err != nil { - return 0, fmt.Errorf("failed to decode response: %w", err) - } + _, _, err = utils.DoRequest(req, &rate) return } diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index bd560091b..0238b057b 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -2,7 +2,6 @@ package bus import ( "context" - "encoding/json" "errors" "fmt" "sync" @@ -13,6 +12,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" ) @@ -28,9 +28,13 @@ type ( Store interface { Autopilot(ctx context.Context, id string) (api.Autopilot, error) - Setting(ctx context.Context, key string) (string, error) UpdateAutopilot(ctx context.Context, ap api.Autopilot) error - UpdateSetting(ctx context.Context, key, value string) error + + GougingSettings(ctx context.Context) (api.GougingSettings, error) + UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error + + PinnedSettings(ctx context.Context) (api.PinnedSettings, error) + UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error } ) @@ -44,7 +48,7 @@ type ( updateInterval time.Duration rateWindow time.Duration - triggerChan chan struct{} + triggerChan chan bool closedChan chan struct{} wg sync.WaitGroup @@ -71,7 +75,7 @@ func NewPinManager(alerts alerts.Alerter, broadcaster webhooks.Broadcaster, e Ex updateInterval: updateInterval, rateWindow: rateWindow, - triggerChan: make(chan struct{}, 1), + triggerChan: make(chan bool, 1), closedChan: make(chan struct{}), } @@ -106,7 +110,7 @@ func (pm *pinManager) Shutdown(ctx context.Context) error { func (pm *pinManager) TriggerUpdate() { select { - case pm.triggerChan <- struct{}{}: + case pm.triggerChan <- true: default: } } @@ -119,16 +123,6 @@ func (pm *pinManager) averageRate() decimal.Decimal { return decimal.NewFromFloat(median) } -func (pm *pinManager) pinnedSettings(ctx context.Context) (api.PricePinSettings, error) { - var ps api.PricePinSettings - if pss, err := pm.s.Setting(ctx, api.SettingPricePinning); err != nil { - return api.PricePinSettings{}, err - } else if err := json.Unmarshal([]byte(pss), &ps); err != nil { - pm.logger.Panicf("failed to unmarshal pinned settings '%s': %v", pss, err) - } - return ps, nil -} - func (pm *pinManager) rateExceedsThreshold(threshold float64) bool { pm.mu.Lock() defer pm.mu.Unlock() @@ -181,8 +175,7 @@ func (pm *pinManager) run() { select { case <-pm.closedChan: return - case <-pm.triggerChan: - forced = true + case forced = <-pm.triggerChan: case <-t.C: } } @@ -254,11 +247,10 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin var updated bool // fetch gouging settings - var gs api.GougingSettings - if gss, err := pm.s.Setting(ctx, api.SettingGouging); err != nil { - return err - } else if err := json.Unmarshal([]byte(gss), &gs); err != nil { - pm.logger.Panicf("failed to unmarshal gouging settings '%s': %v", gss, err) + gs, err := pm.s.GougingSettings(ctx) + if errors.Is(err, sql.ErrSettingNotFound) { + gs = api.DefaultGougingSettings + } else if err != nil { return err } @@ -305,15 +297,14 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin } // validate settings - err := gs.Validate() + err = gs.Validate() if err != nil { pm.logger.Warnw("failed to update gouging setting, new settings make the setting invalid", zap.Error(err)) return err } // update settings - bytes, _ := json.Marshal(gs) - err = pm.s.UpdateSetting(ctx, api.SettingGouging, string(bytes)) + err = pm.s.UpdateGougingSettings(ctx, gs) // broadcast event if err == nil { @@ -321,9 +312,8 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin Module: api.ModuleSetting, Event: api.EventUpdate, Payload: api.EventSettingUpdate{ - Key: api.SettingGouging, - Update: string(bytes), - Timestamp: time.Now().UTC(), + GougingSettings: &gs, + Timestamp: time.Now().UTC(), }, }) } @@ -335,13 +325,15 @@ func (pm *pinManager) updatePrices(ctx context.Context, forced bool) error { pm.logger.Debugw("updating prices", zap.Bool("forced", forced)) // fetch pinned settings - settings, err := pm.pinnedSettings(ctx) - if errors.Is(err, api.ErrSettingNotFound) { - pm.logger.Debug("price pinning not configured, skipping price update") - return nil + settings, err := pm.s.PinnedSettings(ctx) + if errors.Is(err, sql.ErrSettingNotFound) { + settings = api.DefaultPinnedSettings } else if err != nil { return fmt.Errorf("failed to fetch pinned settings: %w", err) - } else if !settings.Enabled() { + } + + // check if pinning is enabled + if !settings.Enabled() { pm.logger.Debug("no pinned settings, skipping price update") return nil } diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index 6a4ae55dd..d7800ed71 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -107,22 +107,18 @@ func (e *mockExplorer) setUnreachable(unreachable bool) { type mockPinStore struct { mu sync.Mutex - settings map[string]string + gs api.GougingSettings + ps api.PinnedSettings autopilots map[string]api.Autopilot } func newTestStore() *mockPinStore { s := &mockPinStore{ autopilots: make(map[string]api.Autopilot), - settings: make(map[string]string), + gs: api.DefaultGougingSettings, + ps: api.DefaultPinnedSettings, } - // add default price pin - and gouging settings - b, _ := json.Marshal(api.DefaultPricePinSettings) - s.settings[api.SettingPricePinning] = string(b) - b, _ = json.Marshal(api.DefaultGougingSettings) - s.settings[api.SettingGouging] = string(b) - // add default autopilot s.autopilots[testAutopilotID] = api.Autopilot{ ID: testAutopilotID, @@ -136,37 +132,38 @@ func newTestStore() *mockPinStore { return s } -func (ms *mockPinStore) gougingSettings() api.GougingSettings { - val, err := ms.Setting(context.Background(), api.SettingGouging) - if err != nil { - panic(err) - } - var gs api.GougingSettings - if err := json.Unmarshal([]byte(val), &gs); err != nil { - panic(err) - } - return gs +func (ms *mockPinStore) GougingSettings(ctx context.Context) (api.GougingSettings, error) { + ms.mu.Lock() + defer ms.mu.Unlock() + return ms.gs, nil } -func (ms *mockPinStore) updatPinnedSettings(pps api.PricePinSettings) { - b, _ := json.Marshal(pps) - err := ms.UpdateSetting(context.Background(), api.SettingPricePinning, string(b)) - if err != nil { - panic(err) - } - time.Sleep(10 * testUpdateInterval) +func (ms *mockPinStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { + ms.mu.Lock() + defer ms.mu.Unlock() + ms.gs = gs + return nil } -func (ms *mockPinStore) Setting(ctx context.Context, key string) (string, error) { +func (ms *mockPinStore) PinnedSettings(ctx context.Context) (api.PinnedSettings, error) { ms.mu.Lock() defer ms.mu.Unlock() - return ms.settings[key], nil + return ms.ps, nil } -func (ms *mockPinStore) UpdateSetting(ctx context.Context, key, value string) error { +func (ms *mockPinStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { ms.mu.Lock() defer ms.mu.Unlock() - ms.settings[key] = value + + b, err := json.Marshal(ps) + if err != nil { + return err + } + var cloned api.PinnedSettings + if err := json.Unmarshal(b, &cloned); err != nil { + return err + } + ms.ps = cloned return nil } @@ -198,67 +195,63 @@ func TestPinManager(t *testing.T) { } }() - // define a small helper to fetch the price manager's rates - rates := func() []float64 { + // waitForUpdate waits for the price manager to update + waitForUpdate := func() { t.Helper() - pm.mu.Lock() - defer pm.mu.Unlock() - return pm.rates - } - - // assert price manager is disabled by default - if cnt := len(rates()); cnt != 0 { - t.Fatalf("expected no rates, got %d", cnt) + pm.triggerChan <- false + time.Sleep(testUpdateInterval) } // enable price pinning - pps := api.DefaultPricePinSettings - pps.Currency = "usd" - pps.Threshold = 0.5 - s.updatPinnedSettings(pps) + ps := api.DefaultPinnedSettings + ps.Currency = "usd" + ps.Threshold = 0.5 + s.UpdatePinnedSettings(context.Background(), ps) - // update exchange rate and fetch current gouging settings - gs := s.gougingSettings() + // fetch current gouging settings + gs, _ := s.GougingSettings(context.Background()) // configure all pins but disable them for now - pps.GougingSettingsPins.MaxDownload = api.Pin{Value: 3, Pinned: false} - pps.GougingSettingsPins.MaxStorage = api.Pin{Value: 3, Pinned: false} - pps.GougingSettingsPins.MaxUpload = api.Pin{Value: 3, Pinned: false} - s.updatPinnedSettings(pps) + ps.GougingSettingsPins.MaxDownload = api.Pin{Value: 3, Pinned: false} + ps.GougingSettingsPins.MaxStorage = api.Pin{Value: 3, Pinned: false} + ps.GougingSettingsPins.MaxUpload = api.Pin{Value: 3, Pinned: false} + s.UpdatePinnedSettings(context.Background(), ps) // assert gouging settings are unchanged - if gss := s.gougingSettings(); !reflect.DeepEqual(gs, gss) { + if gss, _ := s.GougingSettings(context.Background()); !reflect.DeepEqual(gs, gss) { t.Fatalf("expected gouging settings to be the same, got %v", gss) } // enable the max download pin - pps.GougingSettingsPins.MaxDownload.Pinned = true - s.updatPinnedSettings(pps) + ps.GougingSettingsPins.MaxDownload.Pinned = true + s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() - // adjust the rate - e.setRate(1.5) - time.Sleep(2 * testUpdateInterval) - - // at threshold of .5 the prices should not be updated - if gss := s.gougingSettings(); !reflect.DeepEqual(gs, gss) { - t.Fatalf("expected gouging settings to be the same, got %v", gss) + // assert prices are not updated + if gss, _ := s.GougingSettings(context.Background()); !reflect.DeepEqual(gs, gss) { + t.Fatalf("expected gouging settings to be the same, got %v expected %v", gss, gs) } - // lower the threshold, gouging settings should be updated - pps.Threshold = 0.05 - s.updatPinnedSettings(pps) - if gss := s.gougingSettings(); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) { + // adjust and lower the threshold + e.setRate(1.5) + ps.Threshold = 0.05 + s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() + + // assert prices are updated + if gss, _ := s.GougingSettings(context.Background()); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) { t.Fatalf("expected gouging settings to be updated, got %v = %v", gss.MaxDownloadPrice, gs.MaxDownloadPrice) } // enable the rest of the pins - pps.GougingSettingsPins.MaxDownload.Pinned = true - pps.GougingSettingsPins.MaxStorage.Pinned = true - pps.GougingSettingsPins.MaxUpload.Pinned = true - s.updatPinnedSettings(pps) + ps.GougingSettingsPins.MaxDownload.Pinned = true + ps.GougingSettingsPins.MaxStorage.Pinned = true + ps.GougingSettingsPins.MaxUpload.Pinned = true + s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() // assert they're all updated - if gss := s.gougingSettings(); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) || + if gss, _ := s.GougingSettings(context.Background()); gss.MaxDownloadPrice.Equals(gs.MaxDownloadPrice) || gss.MaxStoragePrice.Equals(gs.MaxStoragePrice) || gss.MaxUploadPrice.Equals(gs.MaxUploadPrice) { t.Fatalf("expected gouging settings to be updated, got %v = %v", gss, gs) @@ -277,8 +270,9 @@ func TestPinManager(t *testing.T) { Value: 2, }, } - pps.Autopilots = map[string]api.AutopilotPins{testAutopilotID: pins} - s.updatPinnedSettings(pps) + ps.Autopilots = map[string]api.AutopilotPins{testAutopilotID: pins} + s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() // assert autopilot was not updated if app, _ := s.Autopilot(context.Background(), testAutopilotID); !app.Config.Contracts.Allowance.Equals(ap.Config.Contracts.Allowance) { @@ -287,8 +281,9 @@ func TestPinManager(t *testing.T) { // enable the pin pins.Allowance.Pinned = true - pps.Autopilots[testAutopilotID] = pins - s.updatPinnedSettings(pps) + ps.Autopilots[testAutopilotID] = pins + s.UpdatePinnedSettings(context.Background(), ps) + waitForUpdate() // assert autopilot was updated if app, _ := s.Autopilot(context.Background(), testAutopilotID); app.Config.Contracts.Allowance.Equals(ap.Config.Contracts.Allowance) { @@ -297,9 +292,9 @@ func TestPinManager(t *testing.T) { // make explorer return an error e.setUnreachable(true) + waitForUpdate() // assert alert was registered - s.updatPinnedSettings(pps) res, _ := a.Alerts(context.Background(), alerts.AlertsOpts{}) if len(res.Alerts) == 0 { t.Fatalf("expected 1 alert, got %d", len(a.alerts)) @@ -307,9 +302,9 @@ func TestPinManager(t *testing.T) { // make explorer return a valid response e.setUnreachable(false) + waitForUpdate() // assert alert was dismissed - s.updatPinnedSettings(pps) res, _ = a.Alerts(context.Background(), alerts.AlertsOpts{}) if len(res.Alerts) != 0 { t.Fatalf("expected 0 alerts, got %d", len(a.alerts)) diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index 5cee30e7f..7c591c19a 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -28,6 +28,7 @@ type ( MainMigrator interface { Migrator MakeDirsForPath(ctx context.Context, tx Tx, path string) (int64, error) + UpdateSetting(ctx context.Context, tx Tx, key, value string) error } ) @@ -223,6 +224,12 @@ var ( return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00018_gouging_units", log) }, }, + { + ID: "00019_settings", + Migrate: func(tx Tx) error { + return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00019_settings", log) + }, + }, } } MetricsMigrations = func(ctx context.Context, migrationsFs embed.FS, log *zap.SugaredLogger) []Migration { @@ -244,6 +251,12 @@ var ( return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00002_idx_wallet_metrics_immature", log) }, }, + { + ID: "00003_unix_ms", + Migrate: func(tx Tx) error { + return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00003_unix_ms", log) + }, + }, } } ) diff --git a/internal/test/config.go b/internal/test/config.go index 64dc98c7f..b33c2b0d6 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -33,10 +33,7 @@ var ( }, } - ContractSet = "testset" - ContractSetSettings = api.ContractSetSetting{ - Default: ContractSet, - } + ContractSet = "testset" GougingSettings = api.GougingSettings{ MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC @@ -52,13 +49,18 @@ var ( MinMaxEphemeralAccountBalance: types.Siacoins(1), // 1SC } - PricePinSettings = api.DefaultPricePinSettings + PricePinSettings = api.DefaultPinnedSettings RedundancySettings = api.RedundancySettings{ MinShards: 2, TotalShards: 3, } + UploadSettings = api.UploadSettings{ + DefaultContractSet: ContractSet, + Redundancy: RedundancySettings, + } + S3AccessKeyID = "TESTINGYNHUWCPKOPSYQ" S3SecretAccessKey = "Rh30BNyj+qNI4ftYRteoZbHJ3X4Ln71QtZkRXzJ9" S3Credentials = credentials.NewStaticV4(S3AccessKeyID, S3SecretAccessKey, "") diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 2156c3244..735085711 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -93,11 +93,6 @@ type dbConfig struct { RetryTxIntervals []time.Duration } -type explorerConfig struct { - URL string - Disable bool -} - func (tc *TestCluster) Accounts() []api.Account { tc.tt.Helper() accounts, err := tc.Worker.Accounts(context.Background()) @@ -269,7 +264,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if opts.walletKey != nil { wk = *opts.walletKey } - busCfg, workerCfg, apCfg, dbCfg, explorerCfg := testBusCfg(), testWorkerCfg(), testApCfg(), testDBCfg(), testExplorerCfg() + busCfg, workerCfg, apCfg, dbCfg := testBusCfg(), testWorkerCfg(), testApCfg(), testDBCfg() if opts.busCfg != nil { busCfg = *opts.busCfg } @@ -369,7 +364,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { // Create bus. busDir := filepath.Join(dir, "bus") - b, bShutdownFn, cm, bs, err := newTestBus(ctx, busDir, busCfg, dbCfg, explorerCfg, wk, logger) + b, bShutdownFn, cm, bs, err := newTestBus(ctx, busDir, busCfg, dbCfg, wk, logger) tt.OK(err) busAuth := jape.BasicAuth(busPassword) @@ -491,18 +486,25 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { })) } - // Update the bus settings. - tt.OK(busClient.UpdateSetting(ctx, api.SettingGouging, test.GougingSettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingContractSet, test.ContractSetSettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingPricePinning, test.PricePinSettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingRedundancy, test.RedundancySettings)) - tt.OK(busClient.UpdateSetting(ctx, api.SettingS3Authentication, api.S3AuthenticationSettings{ - V4Keypairs: map[string]string{test.S3AccessKeyID: test.S3SecretAccessKey}, - })) - tt.OK(busClient.UpdateSetting(ctx, api.SettingUploadPacking, api.UploadPackingSettings{ + // Build upload settings. + us := test.UploadSettings + us.Packing = api.UploadPackingSettings{ Enabled: enableUploadPacking, - SlabBufferMaxSizeSoft: api.DefaultUploadPackingSettings.SlabBufferMaxSizeSoft, - })) + SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB, + } + + // Build S3 settings. + s3 := api.S3Settings{ + Authentication: api.S3AuthenticationSettings{ + V4Keypairs: map[string]string{test.S3AccessKeyID: test.S3SecretAccessKey}, + }, + } + + // Update the bus settings. + tt.OK(busClient.UpdateGougingSettings(ctx, test.GougingSettings)) + tt.OK(busClient.UpdatePinnedSettings(ctx, test.PricePinSettings)) + tt.OK(busClient.UpdateUploadSettings(ctx, us)) + tt.OK(busClient.UpdateS3Settings(ctx, s3)) // Fund the bus. if funding { @@ -542,14 +544,15 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { return cluster } -func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, cfgExplorer explorerConfig, pk types.PrivateKey, logger *zap.Logger) (*bus.Bus, func(ctx context.Context) error, *chain.Manager, bus.Store, error) { - // create store +func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, pk types.PrivateKey, logger *zap.Logger) (*bus.Bus, func(ctx context.Context) error, *chain.Manager, bus.Store, error) { + // create store config alertsMgr := alerts.NewManager() storeCfg, err := buildStoreConfig(alertsMgr, dir, cfg.SlabBufferCompletionThreshold, cfgDb, pk, logger) if err != nil { return nil, nil, nil, nil, err } + // create store sqlStore, err := stores.NewSQLStore(storeCfg) if err != nil { return nil, nil, nil, nil, err @@ -636,11 +639,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, // create bus announcementMaxAgeHours := time.Duration(cfg.AnnouncementMaxAgeHours) * time.Hour - var explorerURL string - if cfgExplorer.URL != "" { - explorerURL = cfgExplorer.URL - } - b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, explorerURL, logger) + b, err := bus.New(ctx, masterKey, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, "", logger) if err != nil { return nil, nil, nil, nil, err } @@ -945,6 +944,20 @@ func (c *TestCluster) AddHostsBlocking(n int) []*Host { return hosts } +// MineTransactions tries to mine the transactions in the transaction pool until +// it is empty. +func (c *TestCluster) MineTransactions(ctx context.Context) error { + return test.Retry(100, 100*time.Millisecond, func() error { + txns, err := c.Bus.TransactionPool(ctx) + if err != nil { + return err + } else if len(txns) > 0 { + c.MineBlocks(1) + } + return nil + }) +} + // Shutdown shuts down a TestCluster. func (c *TestCluster) Shutdown() { c.tt.Helper() @@ -1093,12 +1106,6 @@ func testDBCfg() dbConfig { } } -func testExplorerCfg() explorerConfig { - return explorerConfig{ - Disable: true, - } -} - func testWorkerCfg() config.Worker { return config.Worker{ AccountsRefillInterval: time.Second, diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index d41edffac..2f26752a5 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -178,25 +178,25 @@ func TestNewTestCluster(t *testing.T) { tt := cluster.tt // Upload packing should be disabled by default. - ups, err := b.UploadPackingSettings(context.Background()) + us, err := b.UploadSettings(context.Background()) tt.OK(err) - if ups.Enabled { - t.Fatalf("expected upload packing to be disabled by default, got %v", ups.Enabled) + if us.Packing.Enabled { + t.Fatalf("expected upload packing to be disabled by default, got %v", us.Packing.Enabled) } - // PricePinningSettings should have default values - pps, err := b.PricePinningSettings(context.Background()) + // PinnedSettings should have default values + ps, err := b.PinnedSettings(context.Background()) tt.OK(err) - if pps.Currency == "" { + if ps.Currency == "" { t.Fatal("expected default value for Currency") - } else if pps.Threshold == 0 { + } else if ps.Threshold == 0 { t.Fatal("expected default value for Threshold") } // Autopilot shouldn't have its prices pinned - if len(pps.Autopilots) != 1 { - t.Fatalf("expected 1 autopilot, got %v", len(pps.Autopilots)) - } else if pin, exists := pps.Autopilots[api.DefaultAutopilotID]; !exists { + if len(ps.Autopilots) != 1 { + t.Fatalf("expected 1 autopilot, got %v", len(ps.Autopilots)) + } else if pin, exists := ps.Autopilots[api.DefaultAutopilotID]; !exists { t.Fatalf("expected autopilot %v to exist", api.DefaultAutopilotID) } else if pin.Allowance != (api.Pin{}) { t.Fatalf("expected autopilot %v to have no pinned allowance, got %v", api.DefaultAutopilotID, pin.Allowance) @@ -1314,6 +1314,11 @@ func TestEphemeralAccountSync(t *testing.T) { } acc := accounts[0] + // stop autopilot and mine transactions, this prevents an NDF where we + // double spend outputs after restarting the bus + cluster.ShutdownAutopilot(context.Background()) + tt.OK(cluster.MineTransactions(context.Background())) + // stop the cluster host := cluster.hosts[0] cluster.hosts = nil // exclude hosts from shutdown diff --git a/internal/test/e2e/events_test.go b/internal/test/e2e/events_test.go index 515bce5a4..c86fb1d10 100644 --- a/internal/test/e2e/events_test.go +++ b/internal/test/e2e/events_test.go @@ -26,7 +26,6 @@ func TestEvents(t *testing.T) { api.WebhookContractRenew, api.WebhookContractSetUpdate, api.WebhookHostUpdate, - api.WebhookSettingDelete, api.WebhookSettingUpdate, } @@ -125,10 +124,7 @@ func TestEvents(t *testing.T) { // update settings gs := gp.GougingSettings gs.HostBlockHeightLeeway = 100 - tt.OK(b.UpdateSetting(context.Background(), api.SettingGouging, gs)) - - // delete setting - tt.OK(b.DeleteSetting(context.Background(), api.SettingRedundancy)) + tt.OK(b.UpdateGougingSettings(context.Background(), gs)) // update host setting h := cluster.hosts[0] @@ -173,17 +169,7 @@ func TestEvents(t *testing.T) { t.Fatalf("unexpected event %+v", e) } case api.EventSettingUpdate: - if e.Key != api.SettingGouging || e.Timestamp.IsZero() { - t.Fatalf("unexpected event %+v", e) - } - var update api.GougingSettings - bytes, _ := json.Marshal(e.Update) - tt.OK(json.Unmarshal(bytes, &update)) - if update.HostBlockHeightLeeway != 100 { - t.Fatalf("unexpected update %+v", update) - } - case api.EventSettingDelete: - if e.Key != api.SettingRedundancy || e.Timestamp.IsZero() { + if e.GougingSettings == nil || e.GougingSettings.HostBlockHeightLeeway != 100 || e.Timestamp.IsZero() { t.Fatalf("unexpected event %+v", e) } } diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go index 851362489..be24b7186 100644 --- a/internal/test/e2e/gouging_test.go +++ b/internal/test/e2e/gouging_test.go @@ -69,7 +69,7 @@ func TestGouging(t *testing.T) { // update the gouging settings to limit the max storage price to 100H gs := test.GougingSettings gs.MaxStoragePrice = types.NewCurrency64(100) - if err := b.UpdateSetting(context.Background(), api.SettingGouging, gs); err != nil { + if err := b.UpdateGougingSettings(context.Background(), gs); err != nil { t.Fatal(err) } @@ -117,7 +117,7 @@ func TestGouging(t *testing.T) { } // set optimised settings - tt.OK(b.UpdateSetting(context.Background(), api.SettingGouging, resp.Recommendation.GougingSettings)) + tt.OK(b.UpdateGougingSettings(context.Background(), resp.Recommendation.GougingSettings)) // evaluate optimised settings resp, err = cluster.Autopilot.EvaluateConfig(context.Background(), test.AutopilotConfig, resp.Recommendation.GougingSettings, test.RedundancySettings) diff --git a/internal/test/e2e/s3_test.go b/internal/test/e2e/s3_test.go index 743a8e64c..08e1fd8fa 100644 --- a/internal/test/e2e/s3_test.go +++ b/internal/test/e2e/s3_test.go @@ -845,9 +845,11 @@ func TestS3SettingsValidate(t *testing.T) { }, } for i, test := range tests { - err := cluster.Bus.UpdateSetting(context.Background(), api.SettingS3Authentication, api.S3AuthenticationSettings{ - V4Keypairs: map[string]string{ - test.id: test.key, + err := cluster.Bus.UpdateS3Settings(context.Background(), api.S3Settings{ + Authentication: api.S3AuthenticationSettings{ + V4Keypairs: map[string]string{ + test.id: test.key, + }, }, }) if err != nil && !test.shouldFail { diff --git a/internal/utils/web.go b/internal/utils/web.go index 6f0caa571..471270deb 100644 --- a/internal/utils/web.go +++ b/internal/utils/web.go @@ -1,8 +1,10 @@ package utils import ( + "encoding/json" "errors" "fmt" + "io" "net" "net/http" _ "net/http/pprof" @@ -80,3 +82,21 @@ func OpenBrowser(url string) error { return fmt.Errorf("unsupported platform %q", runtime.GOOS) } } + +func DoRequest(req *http.Request, resp interface{}) (http.Header, int, error) { + r, err := http.DefaultClient.Do(req) + if err != nil { + return nil, 0, err + } + defer r.Body.Close() + defer io.Copy(io.Discard, r.Body) + + if r.StatusCode < 200 || r.StatusCode >= 300 { + lr := io.LimitReader(r.Body, 1<<20) // 1MiB + errMsg, _ := io.ReadAll(lr) + return http.Header{}, 0, fmt.Errorf("HTTP error: %s (status: %d)", string(errMsg), r.StatusCode) + } else if resp != nil { + return http.Header{}, 0, json.NewDecoder(r.Body).Decode(resp) + } + return r.Header, r.StatusCode, nil +} diff --git a/internal/worker/cache.go b/internal/worker/cache.go index dfc749d2a..1f5d28d22 100644 --- a/internal/worker/cache.go +++ b/internal/worker/cache.go @@ -181,11 +181,8 @@ func (c *cache) HandleEvent(event webhooks.Event) (err error) { log = log.With("hk", e.HostKey, "ts", e.Timestamp) c.handleHostUpdate(e) case api.EventSettingUpdate: - log = log.With("key", e.Key, "ts", e.Timestamp) - err = c.handleSettingUpdate(e) - case api.EventSettingDelete: - log = log.With("key", e.Key, "ts", e.Timestamp) - c.handleSettingDelete(e) + log = log.With("gouging", e.GougingSettings != nil, "pinned", e.PinnedSettings != nil, "upload", e.UploadSettings != nil, "ts", e.Timestamp) + c.handleSettingUpdate(e) default: log.Info("unhandled event", e) return @@ -310,52 +307,22 @@ func (c *cache) handleHostUpdate(e api.EventHostUpdate) { c.cache.Set(cacheKeyDownloadContracts, contracts) } -func (c *cache) handleSettingDelete(e api.EventSettingDelete) { - if e.Key == api.SettingGouging || e.Key == api.SettingRedundancy { - c.cache.Invalidate(cacheKeyGougingParams) - } -} - -func (c *cache) handleSettingUpdate(e api.EventSettingUpdate) (err error) { +func (c *cache) handleSettingUpdate(e api.EventSettingUpdate) { // return early if the cache doesn't have gouging params to update value, found, _ := c.cache.Get(cacheKeyGougingParams) if !found { - return nil + return } - gp := value.(api.GougingParams) - // marshal the updated value - data, err := json.Marshal(e.Update) - if err != nil { - return fmt.Errorf("couldn't marshal the given value, error: %v", err) + // update the cache + gp := value.(api.GougingParams) + if e.GougingSettings != nil { + gp.GougingSettings = *e.GougingSettings } - - // unmarshal into the appropriated setting and update the cache - switch e.Key { - case api.SettingGouging: - var gs api.GougingSettings - if err := json.Unmarshal(data, &gs); err != nil { - return fmt.Errorf("couldn't update gouging settings, invalid request body, %t", e.Update) - } else if err := gs.Validate(); err != nil { - return fmt.Errorf("couldn't update gouging settings, error: %v", err) - } - - gp.GougingSettings = gs - c.cache.Set(cacheKeyGougingParams, gp) - case api.SettingRedundancy: - var rs api.RedundancySettings - if err := json.Unmarshal(data, &rs); err != nil { - return fmt.Errorf("couldn't update redundancy settings, invalid request body, %t", e.Update) - } else if err := rs.Validate(); err != nil { - return fmt.Errorf("couldn't update redundancy settings, error: %v", err) - } - - gp.RedundancySettings = rs - c.cache.Set(cacheKeyGougingParams, gp) - default: + if e.UploadSettings != nil { + gp.RedundancySettings = e.UploadSettings.Redundancy } - - return nil + c.cache.Set(cacheKeyGougingParams, gp) } func contractsEqual(x, y []api.ContractMetadata) bool { diff --git a/internal/worker/cache_test.go b/internal/worker/cache_test.go index 9bc8d682d..0fa3c10d8 100644 --- a/internal/worker/cache_test.go +++ b/internal/worker/cache_test.go @@ -170,7 +170,6 @@ func TestWorkerCache(t *testing.T) { {Module: api.ModuleContract, Event: api.EventRenew, Payload: nil}, {Module: api.ModuleHost, Event: api.EventUpdate, Payload: nil}, {Module: api.ModuleSetting, Event: api.EventUpdate, Payload: nil}, - {Module: api.ModuleSetting, Event: api.EventDelete, Payload: nil}, } { if err := c.HandleEvent(event); err != nil { t.Fatal(err) diff --git a/internal/worker/events_test.go b/internal/worker/events_test.go index cab65c62d..95a74da91 100644 --- a/internal/worker/events_test.go +++ b/internal/worker/events_test.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "net/http/httptest" "sync" @@ -16,6 +15,7 @@ import ( "go.sia.tech/jape" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" "go.uber.org/zap/zaptest/observer" @@ -202,18 +202,7 @@ func sendEvent(url string, event webhooks.Event) error { if err != nil { return err } - defer io.ReadAll(req.Body) // always drain body - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - errStr, err := io.ReadAll(req.Body) - if err != nil { - return fmt.Errorf("failed to read response body: %w", err) - } - return fmt.Errorf("Webhook returned unexpected status %v: %v", resp.StatusCode, string(errStr)) - } - return nil + _, _, err = utils.DoRequest(req, nil) + return err } diff --git a/stores/settings.go b/stores/settings.go new file mode 100644 index 000000000..471c013fb --- /dev/null +++ b/stores/settings.go @@ -0,0 +1,107 @@ +package stores + +import ( + "context" + "encoding/json" + "fmt" + + "go.sia.tech/renterd/api" + sql "go.sia.tech/renterd/stores/sql" +) + +const ( + SettingGouging = "gouging" + SettingPinned = "pinned" + SettingS3 = "s3" + SettingUpload = "upload" +) + +func (s *SQLStore) GougingSettings(ctx context.Context) (gs api.GougingSettings, err error) { + err = s.fetchSetting(ctx, SettingGouging, &gs) + return +} + +func (s *SQLStore) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error { + return s.updateSetting(ctx, SettingGouging, gs) +} + +func (s *SQLStore) PinnedSettings(ctx context.Context) (ps api.PinnedSettings, err error) { + err = s.fetchSetting(ctx, SettingPinned, &ps) + return +} + +func (s *SQLStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedSettings) error { + return s.updateSetting(ctx, SettingPinned, ps) +} + +func (s *SQLStore) UploadSettings(ctx context.Context) (us api.UploadSettings, err error) { + err = s.fetchSetting(ctx, SettingUpload, &us) + return +} + +func (s *SQLStore) UpdateUploadSettings(ctx context.Context, us api.UploadSettings) error { + return s.updateSetting(ctx, SettingUpload, us) +} + +func (s *SQLStore) S3Settings(ctx context.Context) (ss api.S3Settings, err error) { + err = s.fetchSetting(ctx, SettingS3, &ss) + return +} + +func (s *SQLStore) UpdateS3Settings(ctx context.Context, ss api.S3Settings) error { + return s.updateSetting(ctx, SettingS3, ss) +} + +func (s *SQLStore) fetchSetting(ctx context.Context, key string, out interface{}) error { + s.settingsMu.Lock() + defer s.settingsMu.Unlock() + + // fetch setting from cache + value, ok := s.settings[key] + if ok { + _ = json.Unmarshal([]byte(value), &out) // cached values are always valid json + return nil + } + + // fetch setting from database + var err error + if err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + value, err = tx.Setting(ctx, key) + return err + }); err != nil { + return err + } + + // unmarshal setting + if err := json.Unmarshal([]byte(value), &out); err != nil { + return fmt.Errorf("failed to unmarshal setting '%s', err: %v", key, err) + } + + // update cache + s.settings[key] = value + + return nil +} + +func (s *SQLStore) updateSetting(ctx context.Context, key string, value any) error { + s.settingsMu.Lock() + defer s.settingsMu.Unlock() + + // marshal the value + b, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("couldn't marshal the given value, error: %v", err) + } + + // update db first + err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + return tx.UpdateSetting(ctx, key, string(b)) + }) + if err != nil { + return err + } + + // update cache second + s.settings[key] = string(b) + return nil +} diff --git a/stores/settingsdb.go b/stores/settingsdb.go deleted file mode 100644 index 7a895108c..000000000 --- a/stores/settingsdb.go +++ /dev/null @@ -1,75 +0,0 @@ -package stores - -import ( - "context" - "fmt" - - sql "go.sia.tech/renterd/stores/sql" -) - -// DeleteSetting implements the bus.SettingStore interface. -func (s *SQLStore) DeleteSetting(ctx context.Context, key string) error { - s.settingsMu.Lock() - defer s.settingsMu.Unlock() - - // delete from database first - if err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - return tx.DeleteSettings(ctx, key) - }); err != nil { - return err - } - - // delete from cache - delete(s.settings, key) - return nil -} - -// Setting implements the bus.SettingStore interface. -func (s *SQLStore) Setting(ctx context.Context, key string) (string, error) { - // Check cache first. - s.settingsMu.Lock() - defer s.settingsMu.Unlock() - value, ok := s.settings[key] - if ok { - return value, nil - } - - // Check database. - var err error - err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - value, err = tx.Setting(ctx, key) - return err - }) - if err != nil { - return "", fmt.Errorf("failed to fetch setting from db: %w", err) - } - s.settings[key] = value - return value, nil -} - -// Settings implements the bus.SettingStore interface. -func (s *SQLStore) Settings(ctx context.Context) (settings []string, err error) { - err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - settings, err = tx.Settings(ctx) - return err - }) - return -} - -// UpdateSetting implements the bus.SettingStore interface. -func (s *SQLStore) UpdateSetting(ctx context.Context, key, value string) error { - // update db first - s.settingsMu.Lock() - defer s.settingsMu.Unlock() - - err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - return tx.UpdateSetting(ctx, key, value) - }) - if err != nil { - return err - } - - // update cache second - s.settings[key] = value - return nil -} diff --git a/stores/settingsdb_test.go b/stores/settingsdb_test.go deleted file mode 100644 index cf2582579..000000000 --- a/stores/settingsdb_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package stores - -import ( - "context" - "errors" - "testing" - - "go.sia.tech/renterd/api" -) - -// TestSQLSettingStore tests the bus.SettingStore methods on the SQLSettingStore. -func TestSQLSettingStore(t *testing.T) { - ss := newTestSQLStore(t, defaultTestSQLStoreConfig) - defer ss.Close() - - // assert there are no settings - ctx := context.Background() - if keys, err := ss.Settings(ctx); err != nil { - t.Fatal(err) - } else if len(keys) != 0 { - t.Fatalf("unexpected number of settings, %v != 0", len(keys)) - } - - // add a setting - if err := ss.UpdateSetting(ctx, "foo", "bar"); err != nil { - t.Fatal(err) - } - - // assert it's returned - if keys, err := ss.Settings(ctx); err != nil { - t.Fatal(err) - } else if len(keys) != 1 { - t.Fatalf("unexpected number of settings, %v != 1", len(keys)) - } else if keys[0] != "foo" { - t.Fatalf("unexpected key, %s != 'foo'", keys[0]) - } - - // assert we can query the setting by key - if value, err := ss.Setting(ctx, "foo"); err != nil { - t.Fatal(err) - } else if value != "bar" { - t.Fatalf("unexpected value, %s != 'bar'", value) - } - - // assert we can update the setting - if err := ss.UpdateSetting(ctx, "foo", "barbaz"); err != nil { - t.Fatal(err) - } else if value, err := ss.Setting(ctx, "foo"); err != nil { - t.Fatal(err) - } else if value != "barbaz" { - t.Fatalf("unexpected value, %s != 'barbaz'", value) - } - - // delete the setting - if err := ss.DeleteSetting(ctx, "foo"); err != nil { - t.Fatal(err) - } else if _, err := ss.Setting(ctx, "foo"); !errors.Is(err, api.ErrSettingNotFound) { - t.Fatal("should fail with api.ErrSettingNotFound", err) - } else if keys, err := ss.Settings(ctx); err != nil { - t.Fatal(err) - } else if len(keys) != 0 { - t.Fatalf("unexpected number of settings, %v != 0", len(keys)) - } -} diff --git a/stores/sql.go b/stores/sql.go index 50533768d..14216ce32 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -31,6 +31,10 @@ type ( LongTxDuration time.Duration } + Explorer interface { + Enabled() bool + } + // SQLStore is a helper type for interacting with a SQL-based backend. SQLStore struct { alerts alerts.Alerter diff --git a/stores/sql/database.go b/stores/sql/database.go index 0727a1930..5e6eff041 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -150,8 +150,8 @@ type ( // prefix and returns 'true' if any object was deleted. DeleteObjects(ctx context.Context, bucket, prefix string, limit int64) (bool, error) - // DeleteSettings deletes the settings with the given key. - DeleteSettings(ctx context.Context, key string) error + // DeleteSetting deletes the setting with the given key. + DeleteSetting(ctx context.Context, key string) error // DeleteWebhook deletes the webhook with the matching module, event and // URL of the provided webhook. If the webhook doesn't exist, @@ -317,9 +317,6 @@ type ( // Setting returns the setting with the given key from the database. Setting(ctx context.Context, key string) (string, error) - // Settings returns all available settings from the database. - Settings(ctx context.Context) ([]string, error) - // Slab returns the slab with the given ID or api.ErrSlabNotFound. Slab(ctx context.Context, key object.EncryptionKey) (object.Slab, error) diff --git a/stores/sql/main.go b/stores/sql/main.go index 9a94d6c47..2b156907f 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -28,7 +28,10 @@ import ( "lukechampine.com/frand" ) -var ErrNegativeOffset = errors.New("offset can not be negative") +var ( + ErrNegativeOffset = errors.New("offset can not be negative") + ErrSettingNotFound = errors.New("setting not found") +) // helper types type ( @@ -548,7 +551,7 @@ func DeleteMetadata(ctx context.Context, tx sql.Tx, objID int64) error { return err } -func DeleteSettings(ctx context.Context, tx sql.Tx, key string) error { +func DeleteSetting(ctx context.Context, tx sql.Tx, key string) error { if _, err := tx.Exec(ctx, "DELETE FROM settings WHERE `key` = ?", key); err != nil { return fmt.Errorf("failed to delete setting '%s': %w", key, err) } @@ -1965,30 +1968,13 @@ func Setting(ctx context.Context, tx sql.Tx, key string) (string, error) { var value string err := tx.QueryRow(ctx, "SELECT value FROM settings WHERE `key` = ?", key).Scan((*BusSetting)(&value)) if errors.Is(err, dsql.ErrNoRows) { - return "", api.ErrSettingNotFound + return "", ErrSettingNotFound } else if err != nil { return "", fmt.Errorf("failed to fetch setting '%s': %w", key, err) } return value, nil } -func Settings(ctx context.Context, tx sql.Tx) ([]string, error) { - rows, err := tx.Query(ctx, "SELECT `key` FROM settings") - if err != nil { - return nil, fmt.Errorf("failed to query settings: %w", err) - } - defer rows.Close() - var settings []string - for rows.Next() { - var setting string - if err := rows.Scan(&setting); err != nil { - return nil, fmt.Errorf("failed to scan setting key") - } - settings = append(settings, setting) - } - return settings, nil -} - func Slab(ctx context.Context, tx sql.Tx, key object.EncryptionKey) (object.Slab, error) { // fetch slab var slabID int64 diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index 3da866d5b..111acd187 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -87,6 +87,11 @@ func (b *MainDatabase) Transaction(ctx context.Context, fn func(tx ssql.Database }) } +func (b *MainDatabase) UpdateSetting(ctx context.Context, tx sql.Tx, key, value string) error { + mtx := b.wrapTxn(tx) + return mtx.UpdateSetting(ctx, key, value) +} + func (b *MainDatabase) Version(ctx context.Context) (string, string, error) { return version(ctx, b.db) } @@ -344,8 +349,8 @@ func (tx *MainDatabaseTx) InsertMultipartUpload(ctx context.Context, bucket, key return ssql.InsertMultipartUpload(ctx, tx, bucket, key, ec, mimeType, metadata) } -func (tx *MainDatabaseTx) DeleteSettings(ctx context.Context, key string) error { - return ssql.DeleteSettings(ctx, tx, key) +func (tx *MainDatabaseTx) DeleteSetting(ctx context.Context, key string) error { + return ssql.DeleteSetting(ctx, tx, key) } func (tx *MainDatabaseTx) DeleteWebhook(ctx context.Context, wh webhooks.Webhook) error { @@ -824,10 +829,6 @@ func (tx *MainDatabaseTx) Setting(ctx context.Context, key string) (string, erro return ssql.Setting(ctx, tx, key) } -func (tx *MainDatabaseTx) Settings(ctx context.Context) ([]string, error) { - return ssql.Settings(ctx, tx) -} - func (tx *MainDatabaseTx) Slab(ctx context.Context, key object.EncryptionKey) (object.Slab, error) { return ssql.Slab(ctx, tx, key) } diff --git a/stores/sql/mysql/migrations/main/migration_00019_settings.sql b/stores/sql/mysql/migrations/main/migration_00019_settings.sql new file mode 100644 index 000000000..27f512d7f --- /dev/null +++ b/stores/sql/mysql/migrations/main/migration_00019_settings.sql @@ -0,0 +1,83 @@ +-- avoid duplicate key errors +DELETE FROM settings WHERE `key` IN ("s3", "upload", "pinned"); + +-- migrate settings +INSERT INTO settings (created_at, `key`, value) +SELECT NOW(), k, v +FROM ( + -- upload is a combination of uploadpacking, redundancy, and contractset + SELECT + "upload" as k, + json_merge_patch( + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "uploadpacking")), + json_merge_patch( + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE `key` = "contractset")) + ) + ) as v + WHERE json_extract( + json_merge_patch( + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "uploadpacking")), + json_merge_patch( + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE `key` = "contractset")) + ) + ), "$.packing" + ) IS NOT NULL + AND json_extract( + json_merge_patch( + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "uploadpacking")), + json_merge_patch( + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE `key` = "contractset")) + ) + ), "$.redundancy" + ) IS NOT NULL + + UNION ALL + + -- s3 wraps the s3authentication setting + SELECT + "s3" as k, + json_object("authentication", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "s3authentication")) as v + WHERE json_extract( + json_object("authentication", (SELECT json_extract(value, "$") FROM settings WHERE `key` = "s3authentication")), + "$.authentication" + ) IS NOT NULL + + UNION ALL + + -- pinning renames pricepinning and removes the 'enabled' and 'forexEndpointURL' fields + SELECT + "pinned" as k, + json_remove( + json_remove( + (SELECT json_extract(value, "$") FROM settings WHERE `key` = "pricepinning"), + "$.enabled" + ), + "$.forexEndpointURL" + ) as v + WHERE json_extract( + json_remove( + json_remove( + (SELECT json_extract(value, "$") FROM settings WHERE `key` = "pricepinning"), + "$.enabled" + ), + "$.forexEndpointURL" + ), + "$.currency" + ) IS NOT NULL + AND json_extract( + json_remove( + json_remove( + (SELECT json_extract(value, "$") FROM settings WHERE `key` = "pricepinning"), + "$.enabled" + ), + "$.forexEndpointURL" + ), + "$.threshold" + ) IS NOT NULL +) as migration; + +-- delete old settings +DELETE FROM settings WHERE `key` IN ("uploadpacking", "redundancy", "contractset", "s3authentication", "pricepinning"); diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 18565e41b..5a78517fc 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -86,6 +86,11 @@ func (b *MainDatabase) Transaction(ctx context.Context, fn func(tx ssql.Database }) } +func (b *MainDatabase) UpdateSetting(ctx context.Context, tx sql.Tx, key, value string) error { + mtx := b.wrapTxn(tx) + return mtx.UpdateSetting(ctx, key, value) +} + func (b *MainDatabase) Version(ctx context.Context) (string, string, error) { return version(ctx, b.db) } @@ -336,8 +341,8 @@ func (tx *MainDatabaseTx) DeleteHostSector(ctx context.Context, hk types.PublicK return ssql.DeleteHostSector(ctx, tx, hk, root) } -func (tx *MainDatabaseTx) DeleteSettings(ctx context.Context, key string) error { - return ssql.DeleteSettings(ctx, tx, key) +func (tx *MainDatabaseTx) DeleteSetting(ctx context.Context, key string) error { + return ssql.DeleteSetting(ctx, tx, key) } func (tx *MainDatabaseTx) DeleteWebhook(ctx context.Context, wh webhooks.Webhook) error { @@ -892,10 +897,6 @@ func (tx *MainDatabaseTx) Setting(ctx context.Context, key string) (string, erro return ssql.Setting(ctx, tx, key) } -func (tx *MainDatabaseTx) Settings(ctx context.Context) ([]string, error) { - return ssql.Settings(ctx, tx) -} - func (tx *MainDatabaseTx) Slab(ctx context.Context, key object.EncryptionKey) (object.Slab, error) { return ssql.Slab(ctx, tx, key) } diff --git a/stores/sql/sqlite/migrations/main/migration_00019_settings.sql b/stores/sql/sqlite/migrations/main/migration_00019_settings.sql new file mode 100644 index 000000000..98d4aa36b --- /dev/null +++ b/stores/sql/sqlite/migrations/main/migration_00019_settings.sql @@ -0,0 +1,48 @@ +-- avoid duplicate key errors +DELETE FROM settings WHERE `key` IN ("s3", "upload", "pinned"); + +-- migrate settings +INSERT INTO settings (created_at, `key`, value) +SELECT DATETIME('now'), k, v +FROM ( + -- upload is a combination of uploadpacking, redundancy, and contractset + SELECT + "upload" as k, + json_patch( + json_object("packing", (SELECT json_extract(value, "$") FROM settings WHERE key = "uploadpacking")), + json_patch( + json_object("redundancy", (SELECT json_extract(value, "$") FROM settings WHERE key = "redundancy")), + json_object("defaultContractSet", (SELECT json_extract(value, "$.default") FROM settings WHERE key = "contractset")) + ) + ) as v + WHERE + json_extract(v, "$.packing") IS NOT NULL AND + json_extract(v, "$.redundancy") IS NOT NULL + + UNION ALL + + -- s3 wraps the s3authentication setting + SELECT + "s3" as k, + json_object("authentication", (SELECT json_extract(value, "$") FROM settings WHERE key = "s3authentication")) as v + WHERE json_extract(v, "$.authentication") IS NOT NULL + + UNION ALL + + -- pinning renames pricepinning and removes the 'enabled' and 'forexEndpointURL' fields + SELECT + "pinned" as k, + json_remove( + json_remove( + (SELECT json_extract(value, "$") FROM settings WHERE key = "pricepinning"), + "$.enabled" + ), + "$.forexEndpointURL" + ) as v + WHERE + json_extract(v, "$.currency") IS NOT NULL AND + json_extract(v, "$.threshold") IS NOT NULL +) + +-- delete old settings +DELETE FROM settings WHERE `key` IN ("uploadpacking", "redundancy", "contractset", "s3authentication", "pricepinning"); diff --git a/webhooks/webhooks.go b/webhooks/webhooks.go index 0f1eb636f..ce643835c 100644 --- a/webhooks/webhooks.go +++ b/webhooks/webhooks.go @@ -7,11 +7,11 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "sync" "time" + "go.sia.tech/renterd/internal/utils" "go.uber.org/zap" ) @@ -268,18 +268,7 @@ func sendEvent(ctx context.Context, url string, headers map[string]string, actio for k, v := range headers { req.Header.Set(k, v) } - defer io.ReadAll(req.Body) // always drain body - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - errStr, err := io.ReadAll(req.Body) - if err != nil { - return fmt.Errorf("failed to read response body: %w", err) - } - return fmt.Errorf("Webhook returned unexpected status %v: %v", resp.StatusCode, string(errStr)) - } - return nil + _, _, err = utils.DoRequest(req, nil) + return err } diff --git a/worker/client/client.go b/worker/client/client.go index 95a997277..ca5aee3c8 100644 --- a/worker/client/client.go +++ b/worker/client/client.go @@ -14,6 +14,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/jape" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/webhooks" ) @@ -107,21 +108,14 @@ func (c *Client) HeadObject(ctx context.Context, bucket, key string, opts api.He req.SetBasicAuth("", c.c.WithContext(ctx).Password) opts.ApplyHeaders(req.Header) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - if resp.StatusCode != 200 && resp.StatusCode != 206 { - _ = resp.Body.Close() - switch resp.StatusCode { - case http.StatusNotFound: - return nil, api.ErrObjectNotFound - default: - return nil, errors.New(http.StatusText(resp.StatusCode)) - } + headers, statusCode, err := utils.DoRequest(req, nil) + if err != nil && statusCode == http.StatusNotFound { + return nil, api.ErrObjectNotFound + } else if err != nil { + return nil, errors.New(http.StatusText(statusCode)) } - head, err := parseObjectResponseHeaders(resp.Header) + head, err := parseObjectResponseHeaders(headers) if err != nil { return nil, err } @@ -209,17 +203,11 @@ func (c *Client) UploadMultipartUploadPart(ctx context.Context, r io.Reader, buc } else if req.ContentLength, err = sizeFromSeeker(r); err != nil { return nil, fmt.Errorf("failed to get content length from seeker: %w", err) } - resp, err := http.DefaultClient.Do(req) + header, _, err := utils.DoRequest(req, nil) if err != nil { return nil, err } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return nil, errors.New(string(err)) - } - return &api.UploadMultipartUploadPartResponse{ETag: resp.Header.Get("ETag")}, nil + return &api.UploadMultipartUploadPartResponse{ETag: header.Get("ETag")}, nil } // UploadObject uploads the data in r, creating an object at the given path. @@ -246,17 +234,11 @@ func (c *Client) UploadObject(ctx context.Context, r io.Reader, bucket, key stri } else if req.ContentLength, err = sizeFromSeeker(r); err != nil { return nil, fmt.Errorf("failed to get content length from seeker: %w", err) } - resp, err := http.DefaultClient.Do(req) + header, _, err := utils.DoRequest(req, nil) if err != nil { return nil, err } - defer io.Copy(io.Discard, resp.Body) - defer resp.Body.Close() - if resp.StatusCode != 200 { - err, _ := io.ReadAll(resp.Body) - return nil, errors.New(string(err)) - } - return &api.UploadObjectResponse{ETag: resp.Header.Get("ETag")}, nil + return &api.UploadObjectResponse{ETag: header.Get("ETag")}, nil } // UploadStats returns the upload stats. diff --git a/worker/mocks_test.go b/worker/mocks_test.go index 20657940f..eefb70eea 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -652,8 +652,8 @@ func (*s3Mock) MultipartUploadParts(ctx context.Context, bucket, object string, return api.MultipartListPartsResponse{}, nil } -func (*s3Mock) S3AuthenticationSettings(context.Context) (as api.S3AuthenticationSettings, err error) { - return api.S3AuthenticationSettings{}, nil +func (*s3Mock) S3Settings(context.Context) (as api.S3Settings, err error) { + return api.S3Settings{}, nil } func (*s3Mock) UpdateSetting(context.Context, string, interface{}) error { diff --git a/worker/s3/authentication.go b/worker/s3/authentication.go index 58ebad677..066e27e53 100644 --- a/worker/s3/authentication.go +++ b/worker/s3/authentication.go @@ -117,11 +117,11 @@ func (b *authenticatedBackend) permsFromCtx(ctx context.Context, bucket string) } func (b *authenticatedBackend) reloadV4Keys(ctx context.Context) error { - as, err := b.backend.b.S3AuthenticationSettings(ctx) + s3, err := b.backend.b.S3Settings(ctx) if err != nil { return err } - signature.ReloadKeys(as.V4Keypairs) + signature.ReloadKeys(s3.Authentication.V4Keypairs) return nil } diff --git a/worker/s3/s3.go b/worker/s3/s3.go index 347e3c5cc..10b074306 100644 --- a/worker/s3/s3.go +++ b/worker/s3/s3.go @@ -41,8 +41,7 @@ type Bus interface { MultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker string, maxUploads int) (resp api.MultipartListUploadsResponse, _ error) MultipartUploadParts(ctx context.Context, bucket, object string, uploadID string, marker int, limit int64) (resp api.MultipartListPartsResponse, _ error) - S3AuthenticationSettings(ctx context.Context) (as api.S3AuthenticationSettings, err error) - UpdateSetting(ctx context.Context, key string, value interface{}) error + S3Settings(ctx context.Context) (as api.S3Settings, err error) UploadParams(ctx context.Context) (api.UploadParams, error) }