Merge branch 'api-breakers' of github.com:SiaFoundation/renterd into …

…pj/contract-soft-delete

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -66,10 +66,10 @@ body:
       description: |
         The configuration of your bus
         ```bash
-        curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/contractset
-        curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/gouging
-        curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/redundancy
-        curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/setting/uploadpacking
+        curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/gouging
+        curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/pinned
+        curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/s3
+        curl -u ":[YOUR_PASSWORD]" http://localhost:9980/api/bus/settings/uploads
         ```
       placeholder: Paste the output of the above commands here
     validations:

README.md

@@ -54,10 +54,6 @@ overview of all settings configurable through the CLI.
 | `Log.Database.Level`                 | Logger level for database queries (info\|warn\|error). Defaults to 'warn' | `warn`                            | `--log.database.level`          | `RENTERD_LOG_DATABASE_LEVEL`, `RENTERD_LOG_LEVEL` | `log.database.level`           |
 | `Log.Database.IgnoreRecordNotFoundError` | Enable ignoring 'not found' errors resulting from database queries. Defaults to 'true' | `true`                            | `--log.database.ignoreRecordNotFoundError` | `RENTERD_LOG_DATABASE_IGNORE_RECORD_NOT_FOUND_ERROR` | `log.database.ignoreRecordNotFoundError` |
 | `Log.Database.SlowThreshold`         | Threshold for slow queries in logger. Defaults to 100ms | `100ms`                          | `--log.database.slowThreshold`  | `RENTERD_LOG_DATABASE_SLOW_THRESHOLD`         | `log.database.slowThreshold`        |
-| `Log.Database.Level (DEPRECATED)`    | Logger level                                        | `warn`                            | `--db.logger.logLevel`           | `RENTERD_DB_LOGGER_LOG_LEVEL`                  | `log.database.level`               |
-| `Log.Database.IgnoreRecordNotFoundError (DEPRECATED)` | Ignores 'not found' errors in logger    | `true`                            | `--db.logger.ignoreNotFoundError`| `RENTERD_DB_LOGGER_IGNORE_NOT_FOUND_ERROR`     | `log.ignoreRecordNotFoundError` |
-| `Log.Database.SlowThreshold (DEPRECATED)`    | Threshold for slow queries in logger    | `100ms`                          | `--db.logger.slowThreshold`      | `RENTERD_DB_LOGGER_SLOW_THRESHOLD`             | `log.slowThreshold`       |
-| `Log.Path (DEPRECATED)`              | Path to directory for logs                          | -                                 | `--log-path`                     | `RENTERD_LOG_PATH`                             | `log.path`                         |
 | `Database.MySQL.URI`                 | Database URI for the bus                             | -                                 | `--db.uri`                      | `RENTERD_DB_URI`                              | `database.mysql.uri`                |
 | `Database.MySQL.User`                | Database username for the bus                        | `renterd`                         | `--db.user`                     | `RENTERD_DB_USER`                             | `database.mysql.user`               |
 | `Database.MySQL.Password`            | Database password for the bus                        | -                                 | -                               | `RENTERD_DB_PASSWORD`                         | `database.mysql.password`           |
@@ -70,7 +66,6 @@ overview of all settings configurable through the CLI.
 | `Bus.GatewayAddr`                    | Address for Sia peer connections                     | `:9981`                          | `--bus.gatewayAddr`             | `RENTERD_BUS_GATEWAY_ADDR`                     | `bus.gatewayAddr`                   |
 | `Bus.RemoteAddr`                     | Remote address for the bus                           | -                                 | -                               | `RENTERD_BUS_REMOTE_ADDR`                      | `bus.remoteAddr`                    |
 | `Bus.RemotePassword`                 | Remote password for the bus                          | -                                 | -                               | `RENTERD_BUS_API_PASSWORD`                     | `bus.remotePassword`                |
-| `Bus.PersistInterval`                | Interval for persisting consensus updates            | `1m`                              | `--bus.persistInterval`         | -                                              | `bus.persistInterval`               |
 | `Bus.UsedUTXOExpiry`                 | Expiry for used UTXOs in transactions                | `24h`                             | `--bus.usedUTXOExpiry`          | -                                              | `bus.usedUtxoExpiry`                |
 | `Bus.SlabBufferCompletionThreshold`  | Threshold for slab buffer upload                     | `4096`                            | `--bus.slabBufferCompletionThreshold` | `RENTERD_BUS_SLAB_BUFFER_COMPLETION_THRESHOLD` | `bus.slabBufferCompletionThreshold` |
 | `Worker.AllowPrivateIPs`             | Allows hosts with private IPs                        | -                                 | `--worker.allowPrivateIPs`       | -                                              | `worker.allowPrivateIPs`            |
@@ -102,7 +97,8 @@ overview of all settings configurable through the CLI.
 | `S3.Enabled`                         | Enables/disables S3 API                              | `true`                            | `--s3.enabled`                     | `RENTERD_S3_ENABLED`                           | `s3.enabled`                        |
 | `S3.HostBucketBases`       | Enables bucket rewriting in the router for the provided bases  | -                                 | `--s3.hostBucketBases`           | `RENTERD_S3_HOST_BUCKET_BASES`               | `s3.hostBucketBases`              |
 | `S3.HostBucketEnabled`               | Enables bucket rewriting in the router               | -                                 | `--s3.hostBucketEnabled`           | `RENTERD_S3_HOST_BUCKET_ENABLED`               | `s3.hostBucketEnabled`              |
-| `S3.KeypairsV4 (DEPRECATED)`                      | V4 keypairs for S3                                   | -                                 | -                                  | -            | `s3.keypairsV4`                     |
+| `Explorer.Disable`                    | Disables explorer service                            | `false`                           | `--explorer.disable`               | `RENTERD_EXPLORER_DISABLE`                      | `explorer.disable`                  |
+| `Explorer.URL`                        | URL of service to retrieve data about the Sia network | `https://api.siascan.com`         | `--explorer.url`                   | `RENTERD_EXPLORER_URL`                          | `explorer.url`                      |
 
 ### Single-Node Setup
 
@@ -558,65 +554,6 @@ formed.
 }
 ```
 
-### Contract Set
-
-The contract set settings on the bus allow specifying a default contract set.
-This contract set will be returned by the `bus` through the upload parameters,
-and decides what contracts data is upload or migrated to by default. This
-setting does not have a default value, it can be updated using the settings API:
-
-- `GET /api/bus/setting/contractset`
-- `PUT /api/bus/setting/contractset`
-
-```json
-{
-        "default": "autopilot"
-}
-```
-
-In most cases the default set should match the set from your autopilot
-configuration in order for migrations to work properly. The contract set can be
-overridden by passing it as a query string parameter to the worker's upload and
-migrate endpoints.
-
-- `PUT /api/worker/objects/foo?contractset=foo`
-
-### Redundancy
-
-The default redundancy on mainnet is 30-10, on testnet it is 6-2. The redundancy
-can be updated using the settings API:
-
-- `GET /api/bus/setting/redundancy`
-- `PUT /api/bus/setting/redundancy`
-
-The redundancy can also be passed through query string parameters on the upload
-endpoint in the worker API:
-
-- `PUT /api/worker/objects/foo?minshards=2&totalshards=5`
-
-### Gouging
-
-The default gouging settings are listed below. The gouging settings can be
-updated using the settings API:
-
-- `GET /api/bus/setting/gouging`
-- `PUT /api/bus/setting/gouging`
-
-```json
-{
-	"hostBlockHeightLeeway": 6,                                   // 6 blocks
-	"maxContractPrice": "15000000000000000000000000",             // 15 SC per contract
-	"maxDownloadPrice": "3000000000000000000000000000",           // 3000 SC per 1 TB
-	"maxRPCPrice": "1000000000000000000000",                      // 1mS per RPC
-	"maxStoragePrice": "631593542824",                            // 3000 SC per TB per month
-	"maxUploadPrice": "3000000000000000000000000000",             // 3000 SC per 1 TB
-	"migrationSurchargeMultiplier": 10,                           // overpay up to 10x for sectors migrations on critical slabs
-	"minAccountExpiry": 86400000000000,                           // 1 day
-	"minMaxEphemeralAccountBalance": "1000000000000000000000000", // 1 SC
-	"minPriceTableValidity": 300000000000                         // 5 minutes
-}
-```
-
 ### Blocklist
 
 Unfortunately the Sia blockchain is subject to hosts that announced themselves

api/autopilot.go

@@ -86,6 +86,7 @@ type (
 	// AutopilotStateResponse is the response type for the /autopilot/state
 	// endpoint.
 	AutopilotStateResponse struct {
+		ID                 string      `json:"id"`
 		Configured         bool        `json:"configured"`
 		Migrating          bool        `json:"migrating"`
 		MigratingLastStart TimeRFC3339 `json:"migratingLastStart"`

api/bus.go

@@ -10,6 +10,7 @@ import (
 var (
 	ErrMarkerNotFound        = errors.New("marker not found")
 	ErrMaxFundAmountExceeded = errors.New("renewal exceeds max fund amount")
+	ErrExplorerDisabled      = errors.New("explorer is disabled")
 )
 
 type (
@@ -65,6 +66,13 @@ type (
 		StartTime TimeRFC3339 `json:"startTime"`
 		Network   string      `json:"network"`
 		BuildState
+		Explorer ExplorerState `json:"explorer"`
+	}
+
+	// ExplorerState contains static information about explorer data sources.
+	ExplorerState struct {
+		Enabled bool   `json:"enabled"`
+		URL     string `json:"url,omitempty"`
 	}
 
 	ContractSetUpdateRequest struct {

api/events.go

@@ -19,7 +19,6 @@ const (
 
 	EventAdd     = "add"
 	EventUpdate  = "update"
-	EventDelete  = "delete"
 	EventArchive = "archive"
 	EventRenew   = "renew"
 )
@@ -51,28 +50,25 @@ type (
 		Timestamp time.Time        `json:"timestamp"`
 	}
 
-	EventHostUpdate struct {
-		HostKey   types.PublicKey `json:"hostKey"`
-		NetAddr   string          `json:"netAddr"`
-		Timestamp time.Time       `json:"timestamp"`
-	}
-
 	EventContractSetUpdate struct {
 		Name      string                 `json:"name"`
 		ToAdd     []types.FileContractID `json:"toAdd"`
 		ToRemove  []types.FileContractID `json:"toRemove"`
 		Timestamp time.Time              `json:"timestamp"`
 	}
 
-	EventSettingUpdate struct {
-		Key       string      `json:"key"`
-		Update    interface{} `json:"update"`
-		Timestamp time.Time   `json:"timestamp"`
+	EventHostUpdate struct {
+		HostKey   types.PublicKey `json:"hostKey"`
+		NetAddr   string          `json:"netAddr"`
+		Timestamp time.Time       `json:"timestamp"`
 	}
 
-	EventSettingDelete struct {
-		Key       string    `json:"key"`
-		Timestamp time.Time `json:"timestamp"`
+	EventSettingUpdate struct {
+		GougingSettings *GougingSettings `json:"gougingSettings,omitempty"`
+		PinnedSettings  *PinnedSettings  `json:"pinnedSettings,omitempty"`
+		S3Settings      *S3Settings      `json:"s3Settings,omitempty"`
+		UploadSettings  *UploadSettings  `json:"uploadSettings,omitempty"`
+		Timestamp       time.Time        `json:"timestamp"`
 	}
 )
 
@@ -139,15 +135,6 @@ var (
 			URL:     url,
 		}
 	}
-
-	WebhookSettingDelete = func(url string, headers map[string]string) webhooks.Webhook {
-		return webhooks.Webhook{
-			Event:   EventDelete,
-			Headers: headers,
-			Module:  ModuleSetting,
-			URL:     url,
-		}
-	}
 )
 
 func ParseEventWebhook(event webhooks.Event) (interface{}, error) {
@@ -202,19 +189,12 @@ func ParseEventWebhook(event webhooks.Event) (interface{}, error) {
 			return e, nil
 		}
 	case ModuleSetting:
-		switch event.Event {
-		case EventUpdate:
+		if event.Event == EventUpdate {
 			var e EventSettingUpdate
 			if err := json.Unmarshal(bytes, &e); err != nil {
 				return nil, err
 			}
 			return e, nil
-		case EventDelete:
-			var e EventSettingDelete
-			if err := json.Unmarshal(bytes, &e); err != nil {
-				return nil, err
-			}
-			return e, nil
 		}
 	}
 	return nil, fmt.Errorf("%w: module %s event %s", ErrUnknownEvent, event.Module, event.Event)

api/host.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
@@ -61,9 +62,8 @@ type (
 		MaxConsecutiveScanFailures uint64    `json:"maxConsecutiveScanFailures"`
 	}
 
-	// SearchHostsRequest is the request type for the /api/bus/search/hosts
-	// endpoint.
-	SearchHostsRequest struct {
+	// HostsRequest is the request type for the /api/bus/hosts endpoint.
+	HostsRequest struct {
 		Offset          int               `json:"offset"`
 		Limit           int               `json:"limit"`
 		AutopilotID     string            `json:"autopilotID"`
@@ -72,22 +72,6 @@ type (
 		AddressContains string            `json:"addressContains"`
 		KeyIn           []types.PublicKey `json:"keyIn"`
 	}
-
-	// HostResponse is the response type for the GET
-	// /api/autopilot/host/:hostkey endpoint.
-	HostResponse struct {
-		Host   Host        `json:"host"`
-		Checks *HostChecks `json:"checks,omitempty"`
-	}
-
-	HostChecks struct {
-		Gouging          bool                 `json:"gouging"`
-		GougingBreakdown HostGougingBreakdown `json:"gougingBreakdown"`
-		Score            float64              `json:"score"`
-		ScoreBreakdown   HostScoreBreakdown   `json:"scoreBreakdown"`
-		Usable           bool                 `json:"usable"`
-		UnusableReasons  []string             `json:"unusableReasons,omitempty"`
-	}
 )
 
 type (
@@ -108,17 +92,13 @@ type (
 
 // Option types.
 type (
-	GetHostsOptions struct {
-		Offset int
-		Limit  int
-	}
 	HostsForScanningOptions struct {
 		MaxLastScan TimeRFC3339
 		Limit       int
 		Offset      int
 	}
 
-	SearchHostOptions struct {
+	HostOptions struct {
 		AutopilotID     string
 		AddressContains string
 		FilterMode      string
@@ -129,15 +109,6 @@ type (
 	}
 )
 
-func (opts GetHostsOptions) Apply(values url.Values) {
-	if opts.Offset != 0 {
-		values.Set("offset", fmt.Sprint(opts.Offset))
-	}
-	if opts.Limit != 0 {
-		values.Set("limit", fmt.Sprint(opts.Limit))
-	}
-}
-
 func (opts HostsForScanningOptions) Apply(values url.Values) {
 	if opts.Offset != 0 {
 		values.Set("offset", fmt.Sprint(opts.Offset))
@@ -208,9 +179,9 @@ type (
 	}
 
 	HostCheck struct {
-		Gouging   HostGougingBreakdown   `json:"gouging"`
-		Score     HostScoreBreakdown     `json:"score"`
-		Usability HostUsabilityBreakdown `json:"usability"`
+		GougingBreakdown   HostGougingBreakdown   `json:"gougingBreakdown"`
+		ScoreBreakdown     HostScoreBreakdown     `json:"scoreBreakdown"`
+		UsabilityBreakdown HostUsabilityBreakdown `json:"usabilityBreakdown"`
 	}
 
 	HostGougingBreakdown struct {
@@ -243,6 +214,19 @@ type (
 	}
 )
 
+func (hc HostCheck) MarshalJSON() ([]byte, error) {
+	type check HostCheck
+	return json.Marshal(struct {
+		check
+		Score  float64 `json:"score"`
+		Usable bool    `json:"usable"`
+	}{
+		check:  check(hc),
+		Score:  hc.ScoreBreakdown.Score(),
+		Usable: hc.UsabilityBreakdown.IsUsable(),
+	})
+}
+
 // IsAnnounced returns whether the host has been announced.
 func (h Host) IsAnnounced() bool {
 	return !h.LastAnnouncement.IsZero()

api/host_test.go

@@ -0,0 +1,27 @@
+package api
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+func TestMarshalHostScoreBreakdownJSON(t *testing.T) {
+	hc := HostCheck{
+		ScoreBreakdown: HostScoreBreakdown{
+			Age:              1.1,
+			Collateral:       1.1,
+			Interactions:     1.1,
+			StorageRemaining: 1.1,
+			Uptime:           1.1,
+			Version:          1.1,
+			Prices:           1.1,
+		},
+	}
+	b, err := json.MarshalIndent(hc, " ", " ")
+	if err != nil {
+		t.Fatal(err)
+	} else if !strings.Contains(string(b), "\"score\": 1.9487171000000014") {
+		t.Fatal("expected a score field")
+	}
+}