diff --git a/api/host.go b/api/host.go index 4ad1f87a1..0dfe6da81 100644 --- a/api/host.go +++ b/api/host.go @@ -159,6 +159,7 @@ type ( Blocked bool `json:"blocked"` Checks map[string]HostCheck `json:"checks"` StoredData uint64 `json:"storedData"` + Subnets []string `json:"subnets"` } HostAddress struct { @@ -181,10 +182,11 @@ type ( HostScan struct { HostKey types.PublicKey `json:"hostKey"` + PriceTable rhpv3.HostPriceTable + Settings rhpv2.HostSettings + Subnets []string Success bool Timestamp time.Time - Settings rhpv2.HostSettings - PriceTable rhpv3.HostPriceTable } HostPriceTable struct { diff --git a/api/worker.go b/api/worker.go index ca2ae30f3..ae6024b84 100644 --- a/api/worker.go +++ b/api/worker.go @@ -27,6 +27,10 @@ var ( // be scanned since it is on a private network. ErrHostOnPrivateNetwork = errors.New("host is on a private network") + // ErrHostTooManyAddresses is returned by the worker API when a host has + // more than two addresses of the same type. + ErrHostTooManyAddresses = errors.New("host has more than two addresses, or two of the same type") + // ErrMultiRangeNotSupported is returned by the worker API when a request // tries to download multiple ranges at once. ErrMultiRangeNotSupported = errors.New("multipart ranges are not supported") diff --git a/autopilot/contractor/contract_spending.go b/autopilot/contractor/contract_spending.go index f3e900ac8..82e08831c 100644 --- a/autopilot/contractor/contract_spending.go +++ b/autopilot/contractor/contract_spending.go @@ -13,13 +13,11 @@ func (c *Contractor) currentPeriodSpending(contracts []api.Contract, currentPeri // filter contracts in the current period var filtered []api.ContractMetadata - c.mu.Lock() for _, contract := range contracts { if contract.WindowStart <= currentPeriod { filtered = append(filtered, contract.ContractMetadata) } } - c.mu.Unlock() // calculate the money allocated var totalAllocated types.Currency diff --git a/autopilot/contractor/contractor.go b/autopilot/contractor/contractor.go index 749edddbc..ca0c5c9df 100644 --- a/autopilot/contractor/contractor.go +++ b/autopilot/contractor/contractor.go @@ -7,7 +7,6 @@ import ( "math" "sort" "strings" - "sync" "time" "github.com/montanaflynn/stats" @@ -110,11 +109,10 @@ type Worker interface { type ( Contractor struct { - alerter alerts.Alerter - bus Bus - churn *accumulatedChurn - resolver *ipResolver - logger *zap.SugaredLogger + alerter alerts.Alerter + bus Bus + churn *accumulatedChurn + logger *zap.SugaredLogger revisionBroadcastInterval time.Duration revisionLastBroadcast map[types.FileContractID]time.Time @@ -122,8 +120,6 @@ type ( firstRefreshFailure map[types.FileContractID]time.Time - mu sync.Mutex - shutdownCtx context.Context shutdownCtxCancel context.CancelFunc } @@ -134,9 +130,8 @@ type ( } contractInfo struct { + host api.Host contract api.Contract - settings rhpv2.HostSettings - priceTable rhpv3.HostPriceTable usable bool recoverable bool } @@ -184,8 +179,6 @@ func New(bus Bus, alerter alerts.Alerter, logger *zap.SugaredLogger, revisionSub firstRefreshFailure: make(map[types.FileContractID]time.Time), - resolver: newIPResolver(ctx, resolverLookupTimeout, logger.Named("resolver")), - shutdownCtx: ctx, shutdownCtxCancel: cancel, } @@ -246,13 +239,15 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro if err != nil && !strings.Contains(err.Error(), api.ErrContractSetNotFound.Error()) { return false, err } + hasContractInSet := make(map[types.PublicKey]types.FileContractID) isInCurrentSet := make(map[types.FileContractID]struct{}) for _, c := range currentSet { + hasContractInSet[c.HostKey] = c.ID isInCurrentSet[c.ID] = struct{}{} } c.logger.Infof("contract set '%s' holds %d contracts", ctx.ContractSet(), len(currentSet)) - // fetch all contracts from the worker. + // fetch all contracts from the worker start := time.Now() resp, err := w.Contracts(ctx, timeoutHostRevision) if err != nil { @@ -295,6 +290,19 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro return false, err } + // resolve host IPs on the fly for hosts that have a contract in the set but + // no subnet information, this was added to minimize churn immediately after + // adding 'subnets' to the host table + for _, h := range hosts { + if fcid, ok := hasContractInSet[h.PublicKey]; ok && len(h.Subnets) == 0 { + h.Subnets, _, err = utils.ResolveHostIP(ctx, h.NetAddress) + if err != nil { + c.logger.Warnw("failed to resolve host IP for a host with a contract in the set", "hk", h.PublicKey, "fcid", fcid, "err", err) + continue + } + } + } + // check if any used hosts have lost data to warn the user var toDismiss []types.Hash256 for _, h := range hosts { @@ -416,6 +424,7 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro // check if we need to form contracts and add them to the contract set var formed []api.ContractMetadata if uint64(len(updatedSet)) < threshold && !ctx.state.SkipContractFormations { + // form contracts formed, err = c.runContractFormations(ctx, w, candidates, usedHosts, unusableHosts, ctx.WantedContracts()-uint64(len(updatedSet)), &remaining) if err != nil { c.logger.Errorf("failed to form contracts, err: %v", err) // continue @@ -460,7 +469,7 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro return c.computeContractSetChanged(mCtx, currentSet, updatedSet, formed, refreshed, renewed, toStopUsing, contractData), nil } -func (c *Contractor) computeContractSetChanged(ctx *mCtx, oldSet, newSet []api.ContractMetadata, formed []api.ContractMetadata, refreshed, renewed []renewal, toStopUsing map[types.FileContractID]string, contractData map[types.FileContractID]uint64) bool { +func (c *Contractor) computeContractSetChanged(ctx *mCtx, oldSet, newSet, formed []api.ContractMetadata, refreshed, renewed []renewal, toStopUsing map[types.FileContractID]string, contractData map[types.FileContractID]uint64) bool { name := ctx.ContractSet() // build set lookups @@ -700,7 +709,7 @@ LOOP: if contract.Revision == nil { if !inSet || remainingKeepLeeway == 0 { toStopUsing[fcid] = errContractNoRevision.Error() - } else if !ctx.AllowRedundantIPs() && ipFilter.IsRedundantIP(contract.HostIP, contract.HostKey) { + } else if ctx.ShouldFilterRedundantIPs() && ipFilter.HasRedundantIP(host) { toStopUsing[fcid] = fmt.Sprintf("%v; %v", api.ErrUsabilityHostRedundantIP, errContractNoRevision) hostChecks[contract.HostKey].Usability.RedundantIP = true } else { @@ -711,7 +720,7 @@ LOOP: } // decide whether the contract is still good - ci := contractInfo{contract: contract, priceTable: host.PriceTable.HostPriceTable, settings: host.Settings} + ci := contractInfo{contract: contract, host: host} usable, recoverable, refresh, renew, reasons := c.isUsableContract(ctx.AutopilotConfig(), ctx.state.RS, ci, inSet, bh, ipFilter) ci.usable = usable ci.recoverable = recoverable @@ -768,9 +777,6 @@ func (c *Contractor) runContractFormations(ctx *mCtx, w Worker, candidates score default: } - // convenience variables - shouldFilter := !ctx.AllowRedundantIPs() - c.logger.Infow( "run contract formations", "usedHosts", len(usedHosts), @@ -786,6 +792,14 @@ func (c *Contractor) runContractFormations(ctx *mCtx, w Worker, candidates score ) }() + // build a new host filter + filter := c.newIPFilter() + for _, h := range candidates { + if _, used := usedHosts[h.host.PublicKey]; used { + _ = filter.HasRedundantIP(h.host) + } + } + // select candidates wanted := int(addLeeway(missing, leewayPctCandidateHosts)) selected := candidates.randSelectByScore(wanted) @@ -793,8 +807,10 @@ func (c *Contractor) runContractFormations(ctx *mCtx, w Worker, candidates score // print warning if we couldn't find enough hosts were found c.logger.Infof("looking for %d candidate hosts", wanted) if len(selected) < wanted { - msg := "no candidate hosts found" - if len(selected) > 0 { + var msg string + if len(selected) == 0 { + msg = "no candidate hosts found" + } else { msg = fmt.Sprintf("only found %d candidate host(s) out of the %d we wanted", len(selected), wanted) } if len(candidates) >= wanted { @@ -814,16 +830,6 @@ func (c *Contractor) runContractFormations(ctx *mCtx, w Worker, candidates score // prepare a gouging checker gc := ctx.GougingChecker(cs) - // prepare an IP filter that contains all used hosts - ipFilter := c.newIPFilter() - if shouldFilter { - for _, h := range candidates { - if _, used := usedHosts[h.host.PublicKey]; used { - _ = ipFilter.IsRedundantIP(h.host.NetAddress, h.host.PublicKey) - } - } - } - // calculate min/max contract funds minInitialContractFunds, maxInitialContractFunds := initialContractFundingMinMax(ctx.AutopilotConfig()) @@ -863,12 +869,16 @@ LOOP: } // check if we already have a contract with a host on that subnet - if shouldFilter && ipFilter.IsRedundantIP(host.NetAddress, host.PublicKey) { + if ctx.ShouldFilterRedundantIPs() && filter.HasRedundantIP(host) { continue } + // form the contract formedContract, proceed, err := c.formContract(ctx, w, host, minInitialContractFunds, maxInitialContractFunds, budget) - if err == nil { + if err != nil { + // remove the host from the filter + filter.Remove(host) + } else { // add contract to contract set formed = append(formed, formedContract) missing-- @@ -1081,7 +1091,7 @@ func (c *Contractor) refreshFundingEstimate(cfg api.AutopilotConfig, ci contract // check for a sane minimum that is equal to the initial contract funding // but without an upper cap. minInitialContractFunds, _ := initialContractFundingMinMax(cfg) - minimum := c.initialContractFunding(ci.settings, txnFeeEstimate, minInitialContractFunds, types.ZeroCurrency) + minimum := c.initialContractFunding(ci.host.Settings, txnFeeEstimate, minInitialContractFunds, types.ZeroCurrency) refreshAmountCapped := refreshAmount if refreshAmountCapped.Cmp(minimum) < 0 { refreshAmountCapped = minimum @@ -1223,11 +1233,12 @@ func (c *Contractor) renewContract(ctx *mCtx, w Worker, ci contractInfo, budget if ci.contract.Revision == nil { return api.ContractMetadata{}, true, errors.New("can't renew contract without a revision") } - log := c.logger.With("to_renew", ci.contract.ID, "hk", ci.contract.HostKey, "hostVersion", ci.settings.Version, "hostRelease", ci.settings.Release) + log := c.logger.With("to_renew", ci.contract.ID, "hk", ci.contract.HostKey, "hostVersion", ci.host.Settings.Version, "hostRelease", ci.host.Settings.Release) // convenience variables contract := ci.contract - settings := ci.settings + settings := ci.host.Settings + pt := ci.host.PriceTable.HostPriceTable fcid := contract.ID rev := contract.Revision hk := contract.HostKey @@ -1257,7 +1268,7 @@ func (c *Contractor) renewContract(ctx *mCtx, w Worker, ci contractInfo, budget } // calculate the expected new storage - expectedNewStorage := renterFundsToExpectedStorage(renterFunds, endHeight-cs.BlockHeight, ci.priceTable) + expectedNewStorage := renterFundsToExpectedStorage(renterFunds, endHeight-cs.BlockHeight, pt) // renew the contract resp, err := w.RHPRenew(ctx, fcid, endHeight, hk, contract.SiamuxAddr, settings.Address, ctx.state.Address, renterFunds, types.ZeroCurrency, *budget, expectedNewStorage, settings.WindowSize) @@ -1300,11 +1311,12 @@ func (c *Contractor) refreshContract(ctx *mCtx, w Worker, ci contractInfo, budge if ci.contract.Revision == nil { return api.ContractMetadata{}, true, errors.New("can't refresh contract without a revision") } - log := c.logger.With("to_renew", ci.contract.ID, "hk", ci.contract.HostKey, "hostVersion", ci.settings.Version, "hostRelease", ci.settings.Release) + log := c.logger.With("to_renew", ci.contract.ID, "hk", ci.contract.HostKey, "hostVersion", ci.host.Settings.Version, "hostRelease", ci.host.Settings.Release) // convenience variables contract := ci.contract - settings := ci.settings + settings := ci.host.Settings + pt := ci.host.PriceTable.HostPriceTable fcid := contract.ID rev := contract.Revision hk := contract.HostKey @@ -1317,7 +1329,7 @@ func (c *Contractor) refreshContract(ctx *mCtx, w Worker, ci contractInfo, budge // calculate the renter funds var renterFunds types.Currency - if isOutOfFunds(ctx.AutopilotConfig(), ci.priceTable, ci.contract) { + if isOutOfFunds(ctx.AutopilotConfig(), pt, ci.contract) { renterFunds = c.refreshFundingEstimate(ctx.AutopilotConfig(), ci, ctx.state.Fee) } else { renterFunds = rev.ValidRenterPayout() // don't increase funds @@ -1329,11 +1341,11 @@ func (c *Contractor) refreshContract(ctx *mCtx, w Worker, ci contractInfo, budge return api.ContractMetadata{}, false, fmt.Errorf("insufficient budget: %s < %s", budget.String(), renterFunds.String()) } - expectedNewStorage := renterFundsToExpectedStorage(renterFunds, contract.EndHeight()-cs.BlockHeight, ci.priceTable) + expectedNewStorage := renterFundsToExpectedStorage(renterFunds, contract.EndHeight()-cs.BlockHeight, pt) unallocatedCollateral := contract.RemainingCollateral() // a refresh should always result in a contract that has enough collateral - minNewCollateral := minRemainingCollateral(ctx.AutopilotConfig(), ctx.state.RS, renterFunds, settings, ci.priceTable).Mul64(2) + minNewCollateral := minRemainingCollateral(ctx.AutopilotConfig(), ctx.state.RS, renterFunds, settings, pt).Mul64(2) // maxFundAmount is the remaining funds of the contract to refresh plus the // budget since the previous contract was in the same period diff --git a/autopilot/contractor/hostfilter.go b/autopilot/contractor/hostfilter.go index 42ee045b3..9298ab009 100644 --- a/autopilot/contractor/hostfilter.go +++ b/autopilot/contractor/hostfilter.go @@ -103,7 +103,7 @@ func (u *unusableHostsBreakdown) keysAndValues() []interface{} { // - refresh -> should be refreshed // - renew -> should be renewed func (c *Contractor) isUsableContract(cfg api.AutopilotConfig, rs api.RedundancySettings, ci contractInfo, inSet bool, bh uint64, f *ipFilter) (usable, recoverable, refresh, renew bool, reasons []string) { - contract, s, pt := ci.contract, ci.settings, ci.priceTable + contract, s, pt := ci.contract, ci.host.Settings, ci.host.PriceTable.HostPriceTable usable = true if bh > contract.EndHeight() { @@ -144,7 +144,7 @@ func (c *Contractor) isUsableContract(cfg api.AutopilotConfig, rs api.Redundancy // IP check should be last since it modifies the filter shouldFilter := !cfg.Hosts.AllowRedundantIPs && (usable || recoverable) - if shouldFilter && f.IsRedundantIP(contract.HostIP, contract.HostKey) { + if shouldFilter && f.HasRedundantIP(ci.host) { reasons = append(reasons, api.ErrUsabilityHostRedundantIP.Error()) usable = false recoverable = false // do not use in the contract set, but keep it around for downloads diff --git a/autopilot/contractor/ipfilter.go b/autopilot/contractor/ipfilter.go index 3b754fa0a..b29668372 100644 --- a/autopilot/contractor/ipfilter.go +++ b/autopilot/contractor/ipfilter.go @@ -1,190 +1,66 @@ package contractor import ( - "context" "errors" - "fmt" - "net" - "time" - "go.sia.tech/core/types" - "go.sia.tech/renterd/internal/utils" + "go.sia.tech/renterd/api" "go.uber.org/zap" ) -const ( - // number of unique bits the host IP must have to prevent it from being filtered - ipv4FilterRange = 24 - ipv6FilterRange = 32 - - // ipCacheEntryValidity defines the amount of time the IP filter uses a - // cached entry when it encounters an error while trying to resolve a host's - // IP address - ipCacheEntryValidity = 24 * time.Hour - - // resolverLookupTimeout is the timeout we apply when resolving a host's IP address - resolverLookupTimeout = 10 * time.Second -) - var ( - ErrIOTimeout = errors.New("i/o timeout") - errServerMisbehaving = errors.New("server misbehaving") - errTooManyAddresses = errors.New("host has more than two addresses, or two of the same type") - errUnparsableAddress = errors.New("host address could not be parsed to a subnet") + errHostTooManySubnets = errors.New("host has more than two subnets") ) type ( ipFilter struct { subnetToHostKey map[string]string - resolver *ipResolver - logger *zap.SugaredLogger + logger *zap.SugaredLogger } ) func (c *Contractor) newIPFilter() *ipFilter { - c.resolver.pruneCache() return &ipFilter{ + logger: c.logger, subnetToHostKey: make(map[string]string), - - resolver: c.resolver, - logger: c.logger, } } -func (f *ipFilter) IsRedundantIP(hostIP string, hostKey types.PublicKey) bool { - // perform lookup - subnets, err := f.resolver.lookup(hostIP) - if err != nil { - if !utils.IsErr(err, utils.ErrNoSuchHost) { - f.logger.Errorf("failed to check for redundant IP, treating host %v with IP %v as redundant, err: %v", hostKey, hostIP, err) - } +func (f *ipFilter) HasRedundantIP(host api.Host) bool { + // validate host subnets + if len(host.Subnets) == 0 { + f.logger.Errorf("host %v has no subnet, treating its IP %v as redundant", host.PublicKey, host.NetAddress) return true - } - - // return early if we couldn't resolve to a subnet - if len(subnets) == 0 { - f.logger.Errorf("failed to resolve IP to a subnet, treating host %v with IP %v as redundant, err: %v", hostKey, hostIP, errUnparsableAddress) + } else if len(host.Subnets) > 2 { + f.logger.Errorf("host %v has more than 2 subnets, treating its IP %v as redundant", host.PublicKey, errHostTooManySubnets) return true } - // check if we know about this subnet, if not register all the subnets - host, found := f.subnetToHostKey[subnets[0]] - if !found { - for _, subnet := range subnets { - f.subnetToHostKey[subnet] = hostKey.String() + // check if we know about this subnet + var knownHost string + for _, subnet := range host.Subnets { + if knownHost = f.subnetToHostKey[subnet]; knownHost != "" { + break } - return false - } - - // otherwise compare host keys - sameHost := host == hostKey.String() - return !sameHost -} - -type ( - resolver interface { - LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) } - ipResolver struct { - resolver resolver - cache map[string]ipCacheEntry - timeout time.Duration - shutdownCtx context.Context - logger *zap.SugaredLogger + // if we know about the subnet, the host is redundant if it's not the same + if knownHost != "" { + return host.PublicKey.String() != knownHost } - ipCacheEntry struct { - created time.Time - subnets []string + // otherwise register all the host'ssubnets + for _, subnet := range host.Subnets { + f.subnetToHostKey[subnet] = host.PublicKey.String() } -) -func newIPResolver(ctx context.Context, timeout time.Duration, logger *zap.SugaredLogger) *ipResolver { - if timeout == 0 { - panic("timeout must be greater than zero") // developer error - } - return &ipResolver{ - resolver: &net.Resolver{}, - cache: make(map[string]ipCacheEntry), - timeout: resolverLookupTimeout, - shutdownCtx: ctx, - logger: logger, - } + return false } -func (r *ipResolver) pruneCache() { - for hostIP, entry := range r.cache { - if time.Since(entry.created) > ipCacheEntryValidity { - delete(r.cache, hostIP) +func (f *ipFilter) Remove(h api.Host) { + for k, v := range f.subnetToHostKey { + if v == h.PublicKey.String() { + delete(f.subnetToHostKey, k) } } } - -func (r *ipResolver) lookup(hostIP string) ([]string, error) { - // split off host - host, _, err := net.SplitHostPort(hostIP) - if err != nil { - return nil, err - } - - // make sure we don't hang - ctx, cancel := context.WithTimeout(r.shutdownCtx, r.timeout) - defer cancel() - - // lookup IP addresses - addrs, err := r.resolver.LookupIPAddr(ctx, host) - if err != nil { - // check the cache if it's an i/o timeout or server misbehaving error - if utils.IsErr(err, ErrIOTimeout) || utils.IsErr(err, errServerMisbehaving) { - if entry, found := r.cache[hostIP]; found && time.Since(entry.created) < ipCacheEntryValidity { - r.logger.Infof("using cached IP addresses for %v, err: %v", hostIP, err) - return entry.subnets, nil - } - } - return nil, err - } - - // filter out hosts associated with more than two addresses or two of the same type - if len(addrs) > 2 || (len(addrs) == 2) && (len(addrs[0].IP) == len(addrs[1].IP)) { - return nil, errTooManyAddresses - } - - // parse out subnets - subnets := parseSubnets(addrs) - - // add to cache - if len(subnets) > 0 { - r.cache[hostIP] = ipCacheEntry{ - created: time.Now(), - subnets: subnets, - } - } - - return subnets, nil -} - -func parseSubnets(addresses []net.IPAddr) []string { - subnets := make([]string, 0, len(addresses)) - - for _, address := range addresses { - // figure out the IP range - ipRange := ipv6FilterRange - if address.IP.To4() != nil { - ipRange = ipv4FilterRange - } - - // parse the subnet - cidr := fmt.Sprintf("%s/%d", address.String(), ipRange) - _, ipnet, err := net.ParseCIDR(cidr) - if err != nil { - continue - } - - // add it - subnets = append(subnets, ipnet.String()) - } - - return subnets -} diff --git a/autopilot/contractor/ipfilter_test.go b/autopilot/contractor/ipfilter_test.go deleted file mode 100644 index 63be78753..000000000 --- a/autopilot/contractor/ipfilter_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package contractor - -import ( - "context" - "errors" - "net" - "testing" - "time" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/internal/utils" - "go.uber.org/zap" -) - -var ( - ipv4Localhost = net.IP{127, 0, 0, 1} - ipv6Localhost = net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} -) - -type testResolver struct { - addr map[string][]net.IPAddr - err error -} - -func (r *testResolver) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) { - // return error if set - if err := r.err; err != nil { - r.err = nil - return nil, err - } - // return IP addr if set - if addrs, ok := r.addr[host]; ok { - return addrs, nil - } - return nil, nil -} - -func (r *testResolver) setNextErr(err error) { r.err = err } -func (r *testResolver) setAddr(host string, addrs []net.IPAddr) { r.addr[host] = addrs } - -func newTestResolver() *testResolver { - return &testResolver{addr: make(map[string][]net.IPAddr)} -} - -func newTestIPResolver(r resolver) *ipResolver { - ipr := newIPResolver(context.Background(), time.Minute, zap.NewNop().Sugar()) - ipr.resolver = r - return ipr -} - -func newTestIPFilter(r resolver) *ipFilter { - return &ipFilter{ - subnetToHostKey: make(map[string]string), - resolver: newTestIPResolver(r), - logger: zap.NewNop().Sugar(), - } -} - -func TestIPResolver(t *testing.T) { - r := newTestResolver() - ipr := newTestIPResolver(r) - - // test lookup error - r.setNextErr(errors.New("unknown error")) - if _, err := ipr.lookup("example.com:1234"); !utils.IsErr(err, errors.New("unknown error")) { - t.Fatal("unexpected error", err) - } - - // test IO timeout - no cache entry - r.setNextErr(ErrIOTimeout) - if _, err := ipr.lookup("example.com:1234"); !utils.IsErr(err, ErrIOTimeout) { - t.Fatal("unexpected error", err) - } - - // test IO timeout - expired cache entry - ipr.cache["example.com:1234"] = ipCacheEntry{subnets: []string{"a"}} - r.setNextErr(ErrIOTimeout) - if _, err := ipr.lookup("example.com:1234"); !utils.IsErr(err, ErrIOTimeout) { - t.Fatal("unexpected error", err) - } - - // test IO timeout - live cache entry - ipr.cache["example.com:1234"] = ipCacheEntry{created: time.Now(), subnets: []string{"a"}} - r.setNextErr(ErrIOTimeout) - if subnets, err := ipr.lookup("example.com:1234"); err != nil { - t.Fatal("unexpected error", err) - } else if len(subnets) != 1 || subnets[0] != "a" { - t.Fatal("unexpected subnets", subnets) - } - - // test too many addresses - more than two - r.setAddr("example.com", []net.IPAddr{{}, {}, {}}) - if _, err := ipr.lookup("example.com:1234"); !utils.IsErr(err, errTooManyAddresses) { - t.Fatal("unexpected error", err) - } - - // test too many addresses - two of the same type - r.setAddr("example.com", []net.IPAddr{{IP: net.IPv4(1, 2, 3, 4)}, {IP: net.IPv4(1, 2, 3, 4)}}) - if _, err := ipr.lookup("example.com:1234"); !utils.IsErr(err, errTooManyAddresses) { - t.Fatal("unexpected error", err) - } - - // test invalid addresses - r.setAddr("example.com", []net.IPAddr{{IP: ipv4Localhost}, {IP: net.IP{127, 0, 0, 2}}}) - if _, err := ipr.lookup("example.com:1234"); !utils.IsErr(err, errTooManyAddresses) { - t.Fatal("unexpected error", err) - } - - // test valid addresses - r.setAddr("example.com", []net.IPAddr{{IP: ipv4Localhost}, {IP: ipv6Localhost}}) - if subnets, err := ipr.lookup("example.com:1234"); err != nil { - t.Fatal("unexpected error", err) - } else if len(subnets) != 2 || subnets[0] != "127.0.0.0/24" || subnets[1] != "::/32" { - t.Fatal("unexpected subnets", subnets) - } -} - -func TestIPFilter(t *testing.T) { - r := newTestResolver() - r.setAddr("host1.com", []net.IPAddr{{IP: net.IP{192, 168, 0, 1}}}) - r.setAddr("host2.com", []net.IPAddr{{IP: net.IP{192, 168, 1, 1}}}) - r.setAddr("host3.com", []net.IPAddr{{IP: net.IP{192, 168, 2, 1}}}) - ipf := newTestIPFilter(r) - - // add 3 hosts - unique IPs - r1 := ipf.IsRedundantIP("host1.com:1234", types.PublicKey{1}) - r2 := ipf.IsRedundantIP("host2.com:1234", types.PublicKey{2}) - r3 := ipf.IsRedundantIP("host3.com:1234", types.PublicKey{3}) - if r1 || r2 || r3 { - t.Fatal("unexpected result", r1, r2, r3) - } - - // try add 4th host - redundant IP - r.setAddr("host4.com", []net.IPAddr{{IP: net.IP{192, 168, 0, 12}}}) - if redundant := ipf.IsRedundantIP("host4.com:1234", types.PublicKey{4}); !redundant { - t.Fatal("unexpected result", redundant) - } - - // add 4th host - unique IP - 2 subnets - r.setAddr("host4.com", []net.IPAddr{{IP: net.IP{192, 168, 3, 1}}, {IP: net.ParseIP("2001:0db8:85a3::8a2e:0370:7334")}}) - if redundant := ipf.IsRedundantIP("host4.com:1234", types.PublicKey{4}); redundant { - t.Fatal("unexpected result", redundant) - } - - // try add 5th host - redundant IP based on the IPv6 subnet from host4 - r.setAddr("host5.com", []net.IPAddr{{IP: net.ParseIP("2001:0db8:85b3::8a2e:0370:7335")}}) - if redundant := ipf.IsRedundantIP("host5.com:1234", types.PublicKey{5}); !redundant { - t.Fatal("unexpected result", redundant) - } - - // add 5th host - unique IP - r.setAddr("host5.com", []net.IPAddr{{IP: net.ParseIP("2001:0db9:85b3::8a2e:0370:7335")}}) - if redundant := ipf.IsRedundantIP("host5.com:1234", types.PublicKey{5}); redundant { - t.Fatal("unexpected result", redundant) - } -} diff --git a/autopilot/contractor/state.go b/autopilot/contractor/state.go index 2bf549da1..9f06fe168 100644 --- a/autopilot/contractor/state.go +++ b/autopilot/contractor/state.go @@ -40,26 +40,6 @@ func (ctx *mCtx) ApID() string { return ctx.state.AP.ID } -func (ctx *mCtx) Deadline() (deadline time.Time, ok bool) { - return ctx.ctx.Deadline() -} - -func (ctx *mCtx) Done() <-chan struct{} { - return ctx.ctx.Done() -} - -func (ctx *mCtx) Err() error { - return ctx.ctx.Err() -} - -func (ctx *mCtx) Value(key interface{}) interface{} { - return ctx.ctx.Value(key) -} - -func (ctx *mCtx) AllowRedundantIPs() bool { - return ctx.state.AP.Config.Hosts.AllowRedundantIPs -} - func (ctx *mCtx) Allowance() types.Currency { return ctx.state.Allowance() } @@ -76,16 +56,24 @@ func (ctx *mCtx) ContractSet() string { return ctx.state.AP.Config.Contracts.Set } +func (ctx *mCtx) Deadline() (deadline time.Time, ok bool) { + return ctx.ctx.Deadline() +} + +func (ctx *mCtx) Done() <-chan struct{} { + return ctx.ctx.Done() +} + func (ctx *mCtx) EndHeight() uint64 { return ctx.state.AP.EndHeight() } -func (ctx *mCtx) GougingChecker(cs api.ConsensusState) worker.GougingChecker { - return worker.NewGougingChecker(ctx.state.GS, cs, ctx.state.Fee, ctx.Period(), ctx.RenewWindow()) +func (ctx *mCtx) Err() error { + return ctx.ctx.Err() } -func (ctx *mCtx) WantedContracts() uint64 { - return ctx.state.AP.Config.Contracts.Amount +func (ctx *mCtx) GougingChecker(cs api.ConsensusState) worker.GougingChecker { + return worker.NewGougingChecker(ctx.state.GS, cs, ctx.state.Fee, ctx.Period(), ctx.RenewWindow()) } func (ctx *mCtx) Period() uint64 { @@ -96,6 +84,18 @@ func (ctx *mCtx) RenewWindow() uint64 { return ctx.state.AP.Config.Contracts.RenewWindow } +func (ctx *mCtx) ShouldFilterRedundantIPs() bool { + return !ctx.state.AP.Config.Hosts.AllowRedundantIPs +} + +func (ctx *mCtx) Value(key interface{}) interface{} { + return ctx.ctx.Value(key) +} + +func (ctx *mCtx) WantedContracts() uint64 { + return ctx.state.AP.Config.Contracts.Amount +} + func (state *MaintenanceState) Allowance() types.Currency { return state.AP.Config.Contracts.Allowance } diff --git a/autopilot/scanner.go b/autopilot/scanner.go index 475511c7b..fa317fafa 100644 --- a/autopilot/scanner.go +++ b/autopilot/scanner.go @@ -11,7 +11,6 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/autopilot/contractor" "go.sia.tech/renterd/internal/utils" "go.uber.org/zap" ) @@ -309,7 +308,7 @@ func (s *scanner) launchScanWorkers(ctx context.Context, w scanWorker, reqs chan scan, err := w.RHPScan(ctx, req.hostKey, req.hostIP, s.currentTimeout()) if err != nil { break // abort - } else if !utils.IsErr(errors.New(scan.ScanError), contractor.ErrIOTimeout) && scan.Ping > 0 { + } else if !utils.IsErr(errors.New(scan.ScanError), utils.ErrIOTimeout) && scan.Ping > 0 { s.tracker.addDataPoint(time.Duration(scan.Ping)) } diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index 29ce16d13..d29e88fe3 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -175,6 +175,12 @@ var ( return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00010_webhook_headers", log) }, }, + { + ID: "00011_host_subnets", + Migrate: func(tx Tx) error { + return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00011_host_subnets", log) + }, + }, } } MetricsMigrations = func(ctx context.Context, migrationsFs embed.FS, log *zap.SugaredLogger) []Migration { diff --git a/internal/test/host.go b/internal/test/host.go index c412e6813..e95d1d3d1 100644 --- a/internal/test/host.go +++ b/internal/test/host.go @@ -39,6 +39,7 @@ func NewHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSet PriceTable: api.HostPriceTable{HostPriceTable: pt, Expiry: time.Now().Add(time.Minute)}, Settings: settings, Scanned: true, + Subnets: []string{"38.135.51.0/24"}, } } diff --git a/internal/utils/errors.go b/internal/utils/errors.go index 6c248b61d..67696b984 100644 --- a/internal/utils/errors.go +++ b/internal/utils/errors.go @@ -12,6 +12,7 @@ var ( ErrConnectionRefused = errors.New("connection refused") ErrConnectionTimedOut = errors.New("connection timed out") ErrConnectionResetByPeer = errors.New("connection reset by peer") + ErrIOTimeout = errors.New("i/o timeout") ) // IsErr can be used to compare an error to a target and also works when used on diff --git a/internal/utils/net.go b/internal/utils/net.go new file mode 100644 index 000000000..a4aabd252 --- /dev/null +++ b/internal/utils/net.go @@ -0,0 +1,91 @@ +package utils + +import ( + "context" + "fmt" + "net" + "sort" + + "go.sia.tech/renterd/api" +) + +const ( + ipv4FilterRange = 24 + ipv6FilterRange = 32 +) + +var ( + privateSubnets []*net.IPNet +) + +func init() { + for _, subnet := range []string{ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "100.64.0.0/10", + } { + _, subnet, err := net.ParseCIDR(subnet) + if err != nil { + panic(fmt.Sprintf("failed to parse subnet: %v", err)) + } + privateSubnets = append(privateSubnets, subnet) + } +} + +func ResolveHostIP(ctx context.Context, hostIP string) (subnets []string, private bool, _ error) { + // resolve host address + host, _, err := net.SplitHostPort(hostIP) + if err != nil { + return nil, false, err + } + addrs, err := (&net.Resolver{}).LookupIPAddr(ctx, host) + if err != nil { + return nil, false, err + } + + // filter out hosts associated with more than two addresses or two of the same type + if len(addrs) > 2 || (len(addrs) == 2) && (len(addrs[0].IP) == len(addrs[1].IP)) { + return nil, false, api.ErrHostTooManyAddresses + } + + // parse out subnets + for _, address := range addrs { + private = private || isPrivateIP(address.IP) + + // figure out the IP range + ipRange := ipv6FilterRange + if address.IP.To4() != nil { + ipRange = ipv4FilterRange + } + + // parse the subnet + cidr := fmt.Sprintf("%s/%d", address.String(), ipRange) + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + continue + } + + // add it + subnets = append(subnets, ipnet.String()) + } + + // sort the subnets + sort.Slice(subnets, func(i, j int) bool { + return subnets[i] < subnets[j] + }) + return +} + +func isPrivateIP(addr net.IP) bool { + if addr.IsLoopback() || addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() { + return true + } + + for _, block := range privateSubnets { + if block.Contains(addr) { + return true + } + } + return false +} diff --git a/stores/hostdb.go b/stores/hostdb.go index 58e9a567b..9831db7a4 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -68,6 +68,7 @@ type ( LastAnnouncement time.Time NetAddress string `gorm:"index"` + Subnets string Allowlist []dbAllowlistEntry `gorm:"many2many:host_allowlist_entry_hosts;constraint:OnDelete:CASCADE"` Blocklist []dbBlocklistEntry `gorm:"many2many:host_blocklist_entry_hosts;constraint:OnDelete:CASCADE"` diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 59cfa4264..d6195d9c9 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -485,6 +485,11 @@ func TestRecordScan(t *testing.T) { t.Fatal("mismatch") } + // The host shouldn't have any subnets. + if len(host.Subnets) != 0 { + t.Fatal("unexpected", host.Subnets, len(host.Subnets)) + } + // Fetch the host directly to get the creation time. h, err := hostByPubKey(ss.db, hk) if err != nil { @@ -496,11 +501,12 @@ func TestRecordScan(t *testing.T) { // Record a scan. firstScanTime := time.Now().UTC() + subnets := []string{"212.1.96.0/24", "38.135.51.0/24"} settings := rhpv2.HostSettings{NetAddress: "host.com"} pt := rhpv3.HostPriceTable{ HostBlockHeight: 123, } - if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, firstScanTime, settings, pt, true)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, firstScanTime, settings, pt, true, subnets)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -518,6 +524,11 @@ func TestRecordScan(t *testing.T) { t.Fatal(err) } + // The host should have the subnets. + if !reflect.DeepEqual(host.Subnets, subnets) { + t.Fatal("mismatch") + } + // We expect no uptime or downtime from only a single scan. uptime := time.Duration(0) downtime := time.Duration(0) @@ -541,10 +552,11 @@ func TestRecordScan(t *testing.T) { t.Fatal("mismatch") } - // Record another scan 1 hour after the previous one. + // Record another scan 1 hour after the previous one. We don't pass any + // subnets this time. secondScanTime := firstScanTime.Add(time.Hour) pt.HostBlockHeight = 456 - if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, secondScanTime, settings, pt, true)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, secondScanTime, settings, pt, true, nil)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -572,9 +584,14 @@ func TestRecordScan(t *testing.T) { t.Fatal("mismatch") } + // The host should still have the subnets. + if !reflect.DeepEqual(host.Subnets, subnets) { + t.Fatal("mismatch") + } + // Record another scan 2 hours after the second one. This time it fails. thirdScanTime := secondScanTime.Add(2 * time.Hour) - if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, thirdScanTime, settings, pt, false)}); err != nil { + if err := ss.RecordHostScans(ctx, []api.HostScan{newTestScan(hk, thirdScanTime, settings, pt, false, nil)}); err != nil { t.Fatal(err) } host, err = ss.Host(ctx, hk) @@ -633,8 +650,8 @@ func TestRemoveHosts(t *testing.T) { pt := rhpv3.HostPriceTable{} t1 := now.Add(-time.Minute * 120) // 2 hours ago t2 := now.Add(-time.Minute * 90) // 1.5 hours ago (30min downtime) - hi1 := newTestScan(hk, t1, rhpv2.HostSettings{NetAddress: "host.com"}, pt, false) - hi2 := newTestScan(hk, t2, rhpv2.HostSettings{NetAddress: "host.com"}, pt, false) + hi1 := newTestScan(hk, t1, rhpv2.HostSettings{NetAddress: "host.com"}, pt, false, nil) + hi2 := newTestScan(hk, t2, rhpv2.HostSettings{NetAddress: "host.com"}, pt, false, nil) // record interactions if err := ss.RecordHostScans(context.Background(), []api.HostScan{hi1, hi2}); err != nil { @@ -664,7 +681,7 @@ func TestRemoveHosts(t *testing.T) { // record interactions t3 := now.Add(-time.Minute * 60) // 1 hour ago (60min downtime) - hi3 := newTestScan(hk, t3, rhpv2.HostSettings{NetAddress: "host.com"}, pt, false) + hi3 := newTestScan(hk, t3, rhpv2.HostSettings{NetAddress: "host.com"}, pt, false, nil) if err := ss.RecordHostScans(context.Background(), []api.HostScan{hi3}); err != nil { t.Fatal(err) } @@ -1319,13 +1336,14 @@ func hostByPubKey(tx *gorm.DB, hostKey types.PublicKey) (dbHost, error) { } // newTestScan returns a host interaction with given parameters. -func newTestScan(hk types.PublicKey, scanTime time.Time, settings rhpv2.HostSettings, pt rhpv3.HostPriceTable, success bool) api.HostScan { +func newTestScan(hk types.PublicKey, scanTime time.Time, settings rhpv2.HostSettings, pt rhpv3.HostPriceTable, success bool, subnets []string) api.HostScan { return api.HostScan{ HostKey: hk, + PriceTable: pt, + Settings: settings, + Subnets: subnets, Success: success, Timestamp: scanTime, - Settings: settings, - PriceTable: pt, } } diff --git a/stores/sql/main.go b/stores/sql/main.go index 9cdee70d4..8758b8301 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -1228,7 +1228,8 @@ func RecordHostScans(ctx context.Context, tx sql.Tx, scans []api.HostScan) error price_table = CASE WHEN ? AND (price_table_expiry IS NULL OR ? > price_table_expiry) THEN ? ELSE price_table END, price_table_expiry = CASE WHEN ? AND (price_table_expiry IS NULL OR ? > price_table_expiry) THEN ? ELSE price_table_expiry END, successful_interactions = CASE WHEN ? THEN successful_interactions + 1 ELSE successful_interactions END, - failed_interactions = CASE WHEN ? THEN failed_interactions + 1 ELSE failed_interactions END + failed_interactions = CASE WHEN ? THEN failed_interactions + 1 ELSE failed_interactions END, + subnets = CASE WHEN ? THEN ? ELSE subnets END WHERE public_key = ? `) if err != nil { @@ -1252,6 +1253,7 @@ func RecordHostScans(ctx context.Context, tx sql.Tx, scans []api.HostScan) error scan.Success, now, now, // price_table_expiry scan.Success, // successful_interactions !scan.Success, // failed_interactions + len(scan.Subnets) > 0, strings.Join(scan.Subnets, ","), PublicKey(scan.HostKey), ) if err != nil { @@ -1511,7 +1513,7 @@ func SearchHosts(ctx context.Context, tx sql.Tx, autopilot, filterMode, usabilit SELECT h.id, h.created_at, h.last_announcement, h.public_key, h.net_address, h.price_table, h.price_table_expiry, h.settings, h.total_scans, h.last_scan, h.last_scan_success, h.second_to_last_scan_success, h.uptime, h.downtime, h.successful_interactions, h.failed_interactions, COALESCE(h.lost_sectors, 0), - h.scanned, %s + h.scanned, h.subnets, %s FROM hosts h %s %s @@ -1526,17 +1528,21 @@ func SearchHosts(ctx context.Context, tx sql.Tx, autopilot, filterMode, usabilit var h api.Host var hostID int64 var pte dsql.NullTime + var subnets string err := rows.Scan(&hostID, &h.KnownSince, &h.LastAnnouncement, (*PublicKey)(&h.PublicKey), &h.NetAddress, (*PriceTable)(&h.PriceTable.HostPriceTable), &pte, (*HostSettings)(&h.Settings), &h.Interactions.TotalScans, (*UnixTimeNS)(&h.Interactions.LastScan), &h.Interactions.LastScanSuccess, &h.Interactions.SecondToLastScanSuccess, &h.Interactions.Uptime, &h.Interactions.Downtime, &h.Interactions.SuccessfulInteractions, &h.Interactions.FailedInteractions, &h.Interactions.LostSectors, - &h.Scanned, &h.Blocked, + &h.Scanned, &subnets, &h.Blocked, ) if err != nil { return nil, fmt.Errorf("failed to scan host: %w", err) } + if subnets != "" { + h.Subnets = strings.Split(subnets, ",") + } h.PriceTable.Expiry = pte.Time h.StoredData = storedDataMap[hostID] hosts = append(hosts, h) diff --git a/stores/sql/mysql/migrations/main/migration_00011_host_subnets.sql b/stores/sql/mysql/migrations/main/migration_00011_host_subnets.sql new file mode 100644 index 000000000..ef70b7843 --- /dev/null +++ b/stores/sql/mysql/migrations/main/migration_00011_host_subnets.sql @@ -0,0 +1,2 @@ +ALTER TABLE `hosts` ADD COLUMN `subnets` VARCHAR(255) NOT NULL DEFAULT ''; +UPDATE `hosts` SET last_scan = 0; \ No newline at end of file diff --git a/stores/sql/mysql/migrations/main/schema.sql b/stores/sql/mysql/migrations/main/schema.sql index a8b52e0f3..145fa9452 100644 --- a/stores/sql/mysql/migrations/main/schema.sql +++ b/stores/sql/mysql/migrations/main/schema.sql @@ -98,6 +98,7 @@ CREATE TABLE `hosts` ( `lost_sectors` bigint unsigned DEFAULT NULL, `last_announcement` datetime(3) DEFAULT NULL, `net_address` varchar(191) DEFAULT NULL, + `subnets` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY `public_key` (`public_key`), KEY `idx_hosts_public_key` (`public_key`), diff --git a/stores/sql/sqlite/migrations/main/migration_00011_host_subnets.sql b/stores/sql/sqlite/migrations/main/migration_00011_host_subnets.sql new file mode 100644 index 000000000..4bdb4472d --- /dev/null +++ b/stores/sql/sqlite/migrations/main/migration_00011_host_subnets.sql @@ -0,0 +1,2 @@ +ALTER TABLE `hosts` ADD COLUMN `subnets` text NOT NULL DEFAULT ''; +UPDATE `hosts` SET last_scan = 0; \ No newline at end of file diff --git a/stores/sql/sqlite/migrations/main/schema.sql b/stores/sql/sqlite/migrations/main/schema.sql index aaff040ee..eadbc425c 100644 --- a/stores/sql/sqlite/migrations/main/schema.sql +++ b/stores/sql/sqlite/migrations/main/schema.sql @@ -12,7 +12,7 @@ CREATE INDEX `idx_archived_contracts_state` ON `archived_contracts`(`state`); CREATE INDEX `idx_archived_contracts_renewed_from` ON `archived_contracts`(`renewed_from`); -- dbHost -CREATE TABLE `hosts` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`public_key` blob NOT NULL UNIQUE,`settings` text,`price_table` text,`price_table_expiry` datetime,`total_scans` integer,`last_scan` integer,`last_scan_success` numeric,`second_to_last_scan_success` numeric,`scanned` numeric,`uptime` integer,`downtime` integer,`recent_downtime` integer,`recent_scan_failures` integer,`successful_interactions` real,`failed_interactions` real,`lost_sectors` integer,`last_announcement` datetime,`net_address` text); +CREATE TABLE `hosts` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`public_key` blob NOT NULL UNIQUE,`settings` text,`price_table` text,`price_table_expiry` datetime,`total_scans` integer,`last_scan` integer,`last_scan_success` numeric,`second_to_last_scan_success` numeric,`scanned` numeric,`uptime` integer,`downtime` integer,`recent_downtime` integer,`recent_scan_failures` integer,`successful_interactions` real,`failed_interactions` real,`lost_sectors` integer,`last_announcement` datetime,`net_address` text,`subnets` text NOT NULL DEFAULT ''); CREATE INDEX `idx_hosts_recent_scan_failures` ON `hosts`(`recent_scan_failures`); CREATE INDEX `idx_hosts_recent_downtime` ON `hosts`(`recent_downtime`); CREATE INDEX `idx_hosts_scanned` ON `hosts`(`scanned`); diff --git a/worker/net.go b/worker/net.go index f751be9c9..8401062ab 100644 --- a/worker/net.go +++ b/worker/net.go @@ -2,40 +2,9 @@ package worker import ( "context" - "fmt" "net" ) -var privateSubnets []*net.IPNet - -func init() { - for _, subnet := range []string{ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - "100.64.0.0/10", - } { - _, subnet, err := net.ParseCIDR(subnet) - if err != nil { - panic(fmt.Sprintf("failed to parse subnet: %v", err)) - } - privateSubnets = append(privateSubnets, subnet) - } -} - -func isPrivateIP(addr net.IP) bool { - if addr.IsLoopback() || addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() { - return true - } - - for _, block := range privateSubnets { - if block.Contains(addr) { - return true - } - } - return false -} - func dial(ctx context.Context, hostIP string) (net.Conn, error) { conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", hostIP) return conn, err diff --git a/worker/worker.go b/worker/worker.go index a2ce63685..03ce9c874 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -8,7 +8,6 @@ import ( "io" "math" "math/big" - "net" "net/http" "os" "runtime" @@ -1388,42 +1387,22 @@ func (w *worker) Shutdown(ctx context.Context) error { func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey types.PublicKey, hostIP string) (rhpv2.HostSettings, rhpv3.HostPriceTable, time.Duration, error) { logger := w.logger.With("host", hostKey).With("hostIP", hostIP).With("timeout", timeout) - // prepare a helper for scanning - scan := func() (rhpv2.HostSettings, rhpv3.HostPriceTable, time.Duration, error) { - // helper to prepare a context for scanning - withTimeoutCtx := func() (context.Context, context.CancelFunc) { - if timeout > 0 { - return context.WithTimeout(ctx, timeout) - } - return ctx, func() {} - } - // resolve the address - { - scanCtx, cancel := withTimeoutCtx() - defer cancel() - // resolve hostIP. We don't want to scan hosts on private networks. - if !w.allowPrivateIPs { - host, _, err := net.SplitHostPort(hostIP) - if err != nil { - return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, err - } - addrs, err := (&net.Resolver{}).LookupIPAddr(scanCtx, host) - if err != nil { - return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, err - } - for _, addr := range addrs { - if isPrivateIP(addr.IP) { - return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, api.ErrHostOnPrivateNetwork - } - } - } + + // prepare a helper to create a context for scanning + timeoutCtx := func() (context.Context, context.CancelFunc) { + if timeout > 0 { + return context.WithTimeout(ctx, timeout) } + return ctx, func() {} + } + // prepare a helper for scanning + scan := func() (rhpv2.HostSettings, rhpv3.HostPriceTable, time.Duration, error) { // fetch the host settings start := time.Now() var settings rhpv2.HostSettings { - scanCtx, cancel := withTimeoutCtx() + scanCtx, cancel := timeoutCtx() defer cancel() err := w.withTransportV2(scanCtx, hostKey, hostIP, func(t *rhpv2.Transport) error { var err error @@ -1443,7 +1422,7 @@ func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey ty // fetch the host pricetable var pt rhpv3.HostPriceTable { - scanCtx, cancel := withTimeoutCtx() + scanCtx, cancel := timeoutCtx() defer cancel() err := w.transportPoolV3.withTransportV3(scanCtx, hostKey, settings.SiamuxAddr(), func(ctx context.Context, t *transportV3) error { if hpt, err := RPCPriceTable(ctx, t, func(pt rhpv3.HostPriceTable) (rhpv3.PaymentMethod, error) { return nil, nil }); err != nil { @@ -1460,6 +1439,16 @@ func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey ty return settings, pt, time.Since(start), nil } + // resolve host ip, don't scan if the host is on a private network or if it + // resolves to more than two addresses of the same type, if it fails for + // another reason the host scan won't have subnets + subnets, private, err := utils.ResolveHostIP(ctx, hostIP) + if errors.Is(err, api.ErrHostTooManyAddresses) { + return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, err + } else if private && !w.allowPrivateIPs { + return rhpv2.HostSettings{}, rhpv3.HostPriceTable{}, 0, api.ErrHostOnPrivateNetwork + } + // scan: first try settings, pt, duration, err := scan() if err != nil { @@ -1496,10 +1485,11 @@ func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey ty scanErr := w.bus.RecordHostScans(ctx, []api.HostScan{ { HostKey: hostKey, + PriceTable: pt, + Subnets: subnets, Success: isSuccessfulInteraction(err), - Timestamp: time.Now(), Settings: settings, - PriceTable: pt, + Timestamp: time.Now(), }, }) if scanErr != nil {