Cache DNS Responses (#1385)

Use Mercari's DNS caching library to cache resolved IPs.

#1375

api/host.go

@@ -148,18 +148,19 @@ func (opts HostsForScanningOptions) Apply(values url.Values) {
 
 type (
 	Host struct {
-		KnownSince       time.Time            `json:"knownSince"`
-		LastAnnouncement time.Time            `json:"lastAnnouncement"`
-		PublicKey        types.PublicKey      `json:"publicKey"`
-		NetAddress       string               `json:"netAddress"`
-		PriceTable       HostPriceTable       `json:"priceTable"`
-		Settings         rhpv2.HostSettings   `json:"settings"`
-		Interactions     HostInteractions     `json:"interactions"`
-		Scanned          bool                 `json:"scanned"`
-		Blocked          bool                 `json:"blocked"`
-		Checks           map[string]HostCheck `json:"checks"`
-		StoredData       uint64               `json:"storedData"`
-		Subnets          []string             `json:"subnets"`
+		KnownSince        time.Time            `json:"knownSince"`
+		LastAnnouncement  time.Time            `json:"lastAnnouncement"`
+		PublicKey         types.PublicKey      `json:"publicKey"`
+		NetAddress        string               `json:"netAddress"`
+		PriceTable        HostPriceTable       `json:"priceTable"`
+		Settings          rhpv2.HostSettings   `json:"settings"`
+		Interactions      HostInteractions     `json:"interactions"`
+		Scanned           bool                 `json:"scanned"`
+		Blocked           bool                 `json:"blocked"`
+		Checks            map[string]HostCheck `json:"checks"`
+		StoredData        uint64               `json:"storedData"`
+		ResolvedAddresses []string             `json:"resolvedAddresses"`
+		Subnets           []string             `json:"subnets"`
 	}
 
 	HostAddress struct {
@@ -181,12 +182,13 @@ type (
 	}
 
 	HostScan struct {
-		HostKey    types.PublicKey `json:"hostKey"`
-		PriceTable rhpv3.HostPriceTable
-		Settings   rhpv2.HostSettings
-		Subnets    []string
-		Success    bool
-		Timestamp  time.Time
+		HostKey           types.PublicKey      `json:"hostKey"`
+		PriceTable        rhpv3.HostPriceTable `json:"priceTable"`
+		Settings          rhpv2.HostSettings   `json:"settings"`
+		ResolvedAddresses []string             `json:"resolvedAddresses"`
+		Subnets           []string             `json:"subnets"`
+		Success           bool                 `json:"success"`
+		Timestamp         time.Time            `json:"timestamp"`
 	}
 
 	HostPriceTable struct {
@@ -196,9 +198,9 @@ type (
 
 	HostPriceTableUpdate struct {
 		HostKey    types.PublicKey `json:"hostKey"`
-		Success    bool
-		Timestamp  time.Time
-		PriceTable HostPriceTable
+		Success    bool            `json:"success"`
+		Timestamp  time.Time       `json:"timestamp"`
+		PriceTable HostPriceTable  `json:"priceTable"`
 	}
 
 	HostCheck struct {

autopilot/contractor/contractor.go

@@ -980,7 +980,7 @@ func performContractChecks(ctx *mCtx, alerter alerts.Alerter, bus Bus, w Worker,
 		}
 
 		// extend logger
-		logger = logger.With("subnets", host.Subnets).
+		logger = logger.With("addresses", host.ResolvedAddresses).
 			With("blocked", host.Blocked)
 
 		// check if host is blocked
@@ -1202,18 +1202,18 @@ func performContractFormations(ctx *mCtx, bus Bus, w Worker, cr contractReviser,
 		}
 		gc := ctx.GougingChecker(cs)
 
-		// prepare a gouging checker
+		// prepare a logger
 		logger := logger.With("hostKey", candidate.host.PublicKey).
 			With("remainingBudget", remainingFunds).
-			With("subnets", candidate.host.Subnets)
+			With("addresses", candidate.host.ResolvedAddresses)
 
 		// perform gouging checks on the fly to ensure the host is not gouging its prices
 		if breakdown := gc.Check(nil, &candidate.host.PriceTable.HostPriceTable); breakdown.Gouging() {
 			logger.With("reasons", breakdown.String()).Info("candidate is price gouging")
 			continue
 		}
 
-		// check if we already have a contract with a host on that subnet
+		// check if we already have a contract with a host on that address
 		if ctx.ShouldFilterRedundantIPs() && ipFilter.HasRedundantIP(candidate.host) {
 			logger.Info("host has redundant IP")
 			continue

autopilot/contractor/hostset.go

@@ -1,9 +1,12 @@
 package contractor
 
 import (
+	"context"
 	"errors"
+	"time"
 
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/utils"
 	"go.uber.org/zap"
 )
 
@@ -20,18 +23,31 @@ type (
 )
 
 func (hs *hostSet) HasRedundantIP(host api.Host) bool {
+	// compat code for hosts that have been scanned before ResolvedAddresses
+	// were introduced
+	if len(host.ResolvedAddresses) == 0 {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		host.ResolvedAddresses, _, _ = utils.ResolveHostIP(ctx, host.NetAddress)
+	}
+
+	subnets, err := utils.AddressesToSubnets(host.ResolvedAddresses)
+	if err != nil {
+		hs.logger.Errorf("failed to parse host %v subnets: %v", host.PublicKey, err)
+		return true
+	}
 	// validate host subnets
-	if len(host.Subnets) == 0 {
+	if len(subnets) == 0 {
 		hs.logger.Errorf("host %v has no subnet, treating its IP %v as redundant", host.PublicKey, host.NetAddress)
 		return true
-	} else if len(host.Subnets) > 2 {
+	} else if len(subnets) > 2 {
 		hs.logger.Errorf("host %v has more than 2 subnets, treating its IP %v as redundant", host.PublicKey, errHostTooManySubnets)
 		return true
 	}
 
 	// check if we know about this subnet
 	var knownHost string
-	for _, subnet := range host.Subnets {
+	for _, subnet := range subnets {
 		if knownHost = hs.subnetToHostKey[subnet]; knownHost != "" {
 			break
 		}
@@ -45,7 +61,12 @@ func (hs *hostSet) HasRedundantIP(host api.Host) bool {
 }
 
 func (hs *hostSet) Add(host api.Host) {
-	for _, subnet := range host.Subnets {
+	subnets, err := utils.AddressesToSubnets(host.ResolvedAddresses)
+	if err != nil {
+		hs.logger.Errorf("failed to parse host %v subnets: %v", host.PublicKey, err)
+		return
+	}
+	for _, subnet := range subnets {
 		hs.subnetToHostKey[subnet] = host.PublicKey.String()
 	}
 }

autopilot/contractor/hostset_test.go

@@ -0,0 +1,77 @@
+package contractor
+
+import (
+	"testing"
+
+	"go.sia.tech/core/types"
+	"go.sia.tech/renterd/api"
+	"go.uber.org/zap"
+)
+
+func TestHostSet(t *testing.T) {
+	hs := &hostSet{
+		subnetToHostKey: make(map[string]string),
+		logger:          zap.NewNop().Sugar(),
+	}
+
+	// Host with no subnets
+	host1 := api.Host{
+		PublicKey:         types.GeneratePrivateKey().PublicKey(),
+		ResolvedAddresses: []string{},
+	}
+	if !hs.HasRedundantIP(host1) {
+		t.Fatalf("Expected host with no subnets to be considered redundant")
+	}
+
+	// Host with more than 2 subnets
+	host2 := api.Host{
+		PublicKey:         types.GeneratePrivateKey().PublicKey(),
+		ResolvedAddresses: []string{"192.168.1.1", "10.0.0.1", "172.16.0.1"},
+	}
+	if !hs.HasRedundantIP(host2) {
+		t.Fatalf("Expected host with more than 2 subnets to be considered redundant")
+	}
+
+	// New host with unique subnet
+	host3 := api.Host{
+		PublicKey:         types.GeneratePrivateKey().PublicKey(),
+		ResolvedAddresses: []string{"192.168.2.1"},
+	}
+	if hs.HasRedundantIP(host3) {
+		t.Fatal("Expected new host with unique subnet to not be considered redundant")
+	}
+	hs.Add(host3)
+
+	// New host with same subnet but different public key
+	host4 := api.Host{
+		PublicKey:         types.GeneratePrivateKey().PublicKey(),
+		ResolvedAddresses: []string{"192.168.2.1"},
+	}
+	if !hs.HasRedundantIP(host4) {
+		t.Fatal("Expected host with same subnet but different public key to be considered redundant")
+	}
+
+	// Same host from before
+	if hs.HasRedundantIP(host3) {
+		t.Fatal("Expected same host to not be considered redundant")
+	}
+
+	// Host with two valid subnets
+	host5 := api.Host{
+		PublicKey:         types.GeneratePrivateKey().PublicKey(),
+		ResolvedAddresses: []string{"192.168.3.1", "10.0.0.1"},
+	}
+	if hs.HasRedundantIP(host5) {
+		t.Fatal("Expected host with two valid subnets to not be considered redundant")
+	}
+	hs.Add(host5)
+
+	// New host with one overlapping subnet
+	host6 := api.Host{
+		PublicKey:         types.GeneratePrivateKey().PublicKey(),
+		ResolvedAddresses: []string{"10.0.0.1", "172.16.0.1"},
+	}
+	if !hs.HasRedundantIP(host6) {
+		t.Fatal("Expected host with one overlapping subnet to be considered redundant")
+	}
+}

internal/rhp/v2/rhp.go

@@ -68,15 +68,21 @@ var (
 )
 
 type (
+	Dialer interface {
+		Dial(ctx context.Context, hk types.PublicKey, address string) (net.Conn, error)
+	}
+
 	PrepareFormFn func(ctx context.Context, renterAddress types.Address, renterKey types.PublicKey, renterFunds, hostCollateral types.Currency, hostKey types.PublicKey, hostSettings rhpv2.HostSettings, endHeight uint64) (txns []types.Transaction, discard func(types.Transaction), err error)
 )
 
 type Client struct {
+	dialer Dialer
 	logger *zap.SugaredLogger
 }
 
-func New(logger *zap.Logger) *Client {
+func New(dialer Dialer, logger *zap.Logger) *Client {
 	return &Client{
+		dialer: dialer,
 		logger: logger.Sugar().Named("rhp2"),
 	}
 }
@@ -569,8 +575,8 @@ func (w *Client) withRevisionV2(renterKey types.PrivateKey, gougingChecker gougi
 	return fn(t, rev, settings)
 }
 
-func (w *Client) withTransportV2(ctx context.Context, hostKey types.PublicKey, hostIP string, fn func(*rhpv2.Transport) error) (err error) {
-	conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", hostIP)
+func (c *Client) withTransportV2(ctx context.Context, hostKey types.PublicKey, hostIP string, fn func(*rhpv2.Transport) error) (err error) {
+	conn, err := c.dialer.Dial(ctx, hostKey, hostIP)
 	if err != nil {
 		return err
 	}

internal/sql/migrations.go

@@ -193,6 +193,12 @@ var (
 					return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00013_coreutils_wallet", log)
 				},
 			},
+			{
+				ID: "00014_hosts_resolvedaddresses",
+				Migrate: func(tx Tx) error {
+					return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00014_hosts_resolvedaddresses", log)
+				},
+			},
 		}
 	}
 	MetricsMigrations = func(ctx context.Context, migrationsFs embed.FS, log *zap.SugaredLogger) []Migration {

internal/test/host.go

@@ -35,11 +35,12 @@ func NewHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSet
 			SuccessfulInteractions: 2,
 			FailedInteractions:     0,
 		},
-		PublicKey:  hk,
-		PriceTable: api.HostPriceTable{HostPriceTable: pt, Expiry: time.Now().Add(time.Minute)},
-		Settings:   settings,
-		Scanned:    true,
-		Subnets:    []string{"38.135.51.0/24"},
+		PublicKey:         hk,
+		PriceTable:        api.HostPriceTable{HostPriceTable: pt, Expiry: time.Now().Add(time.Minute)},
+		Settings:          settings,
+		Scanned:           true,
+		ResolvedAddresses: []string{"38.135.51.1"},
+		Subnets:           []string{"38.135.51.0/24"},
 	}
 }
 

internal/utils/net.go

@@ -36,7 +36,34 @@ func init() {
 	}
 }
 
-func ResolveHostIP(ctx context.Context, hostIP string) (subnets []string, private bool, _ error) {
+func AddressesToSubnets(resolvedAddresses []string) ([]string, error) {
+	var subnets []string
+	for _, addr := range resolvedAddresses {
+		parsed := net.ParseIP(addr)
+		if parsed == nil {
+			return nil, fmt.Errorf("failed to parse address: %s", addr)
+		}
+
+		// figure out the IP range
+		ipRange := ipv6FilterRange
+		if parsed.To4() != nil {
+			ipRange = ipv4FilterRange
+		}
+
+		// parse the subnet
+		cidr := fmt.Sprintf("%s/%d", parsed.String(), ipRange)
+		_, ipnet, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse cidr: %w", err)
+		}
+
+		subnets = append(subnets, ipnet.String())
+	}
+
+	return subnets, nil
+}
+
+func ResolveHostIP(ctx context.Context, hostIP string) (ips []string, private bool, _ error) {
 	// resolve host address
 	host, _, err := net.SplitHostPort(hostIP)
 	if err != nil {
@@ -52,30 +79,17 @@ func ResolveHostIP(ctx context.Context, hostIP string) (subnets []string, privat
 		return nil, false, fmt.Errorf("%w: %+v", ErrHostTooManyAddresses, addrs)
 	}
 
-	// parse out subnets
+	// get ips
 	for _, address := range addrs {
 		private = private || isPrivateIP(address.IP)
 
-		// figure out the IP range
-		ipRange := ipv6FilterRange
-		if address.IP.To4() != nil {
-			ipRange = ipv4FilterRange
-		}
-
-		// parse the subnet
-		cidr := fmt.Sprintf("%s/%d", address.String(), ipRange)
-		_, ipnet, err := net.ParseCIDR(cidr)
-		if err != nil {
-			continue
-		}
-
 		// add it
-		subnets = append(subnets, ipnet.String())
+		ips = append(ips, address.IP.String())
 	}
 
-	// sort the subnets
-	sort.Slice(subnets, func(i, j int) bool {
-		return subnets[i] < subnets[j]
+	// sort the ips
+	sort.Slice(ips, func(i, j int) bool {
+		return ips[i] < ips[j]
 	})
 	return
 }