Merge pull request #430 from SiaFoundation/nate/prevent-panic

Prevent price pin failure from crashing node

Author: n8maninger

cmd/hostd/main.go

@@ -396,7 +396,7 @@ func main() {
 		defer cancel()
 
 		if _, err := ex.SiacoinExchangeRate(ctx, "usd"); err != nil {
-			log.Fatal("failed to get exchange rate", zap.Error(err))
+			log.Error("failed to get exchange rate. explorer features may not work correctly", zap.Error(err))
 		}
 	}
 

cmd/hostd/node.go

@@ -187,6 +187,7 @@ func newNode(ctx context.Context, walletKey types.PrivateKey, ex *explorer.Explo
 	var pm *pin.Manager
 	if !cfg.Explorer.Disable {
 		pm, err = pin.NewManager(
+			pin.WithAlerts(am),
 			pin.WithStore(db),
 			pin.WithSettings(sr),
 			pin.WithExchangeRateRetriever(ex),

host/settings/pin/options.go

@@ -16,6 +16,13 @@ func WithLogger(log *zap.Logger) Option {
 	}
 }
 
+// WithAlerts sets the alerts manager for the pinner to register alerts with.
+func WithAlerts(a Alerts) Option {
+	return func(m *Manager) {
+		m.alerts = a
+	}
+}
+
 // WithFrequency sets the frequency at which the manager updates the host's
 // settings based on the current exchange rate.
 func WithFrequency(frequency time.Duration) Option {

host/settings/pin/pin.go

@@ -9,10 +9,14 @@ import (
 
 	"github.com/shopspring/decimal"
 	"go.sia.tech/core/types"
+	"go.sia.tech/hostd/alerts"
 	"go.sia.tech/hostd/host/settings"
 	"go.uber.org/zap"
+	"lukechampine.com/frand"
 )
 
+var pinAlertID = frand.Entropy256()
+
 type (
 	// A Pin is a pinned price in an external currency.
 	Pin struct {
@@ -44,6 +48,12 @@ type (
 		MaxCollateral Pin `json:"maxCollateral"`
 	}
 
+	// Alerts registers global alerts.
+	Alerts interface {
+		Register(alerts.Alert)
+		Dismiss(...types.Hash256)
+	}
+
 	// A SettingsManager updates and retrieves the host's settings.
 	SettingsManager interface {
 		Settings() settings.Settings
@@ -67,6 +77,7 @@ type (
 	Manager struct {
 		log      *zap.Logger
 		store    Store
+		alerts   Alerts
 		explorer ExchangeRateRetriever
 		sm       SettingsManager
 
@@ -99,6 +110,26 @@ func averageRate(rates []decimal.Decimal) decimal.Decimal {
 	return sum.Div(decimal.NewFromInt(int64(len(rates))))
 }
 
+func (m *Manager) registerPinFailureAlert(err error) {
+	if m.alerts != nil && err != nil {
+		m.alerts.Register(alerts.Alert{
+			ID:        pinAlertID,
+			Severity:  alerts.SeverityError,
+			Message:   "failed to update prices",
+			Timestamp: time.Now(),
+			Data: map[string]interface{}{
+				"error": err.Error(),
+			},
+		})
+	}
+}
+
+func (m *Manager) dismissPinFailureAlert() {
+	if m.alerts != nil {
+		m.alerts.Dismiss(pinAlertID)
+	}
+}
+
 func (m *Manager) updatePrices(ctx context.Context, force bool) error {
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
@@ -227,6 +258,7 @@ func (m *Manager) Run(ctx context.Context) error {
 
 	// update prices immediately
 	if err := m.updatePrices(ctx, true); err != nil {
+		m.registerPinFailureAlert(err)
 		m.log.Error("failed to update prices", zap.Error(err))
 	}
 
@@ -237,6 +269,9 @@ func (m *Manager) Run(ctx context.Context) error {
 		case <-t.C:
 			if err := m.updatePrices(ctx, false); err != nil {
 				m.log.Error("failed to update prices", zap.Error(err))
+				m.registerPinFailureAlert(err)
+			} else {
+				m.dismissPinFailureAlert()
 			}
 		}
 	}