Merge branch 'master' into akaladarshi/lotus-shed-prune-all-indexes-cmd

CHANGELOG.md

@@ -23,33 +23,10 @@
 
 ## ☢️ Upgrade Warnings ☢️
 
-- lotus-gateway behaviour, CLI arguments and APIs have received minor changes. See the improvements section below.
-
 ## New features
 
-- feat: Add trace filter API supporting RPC method `trace_filter` ([filecoin-project/lotus#12123](https://github.com/filecoin-project/lotus/pull/12123)). Configuring `EthTraceFilterMaxResults` sets a limit on how many results are returned in any individual `trace_filter` RPC API call.
-- feat: `FilecoinAddressToEthAddress` RPC can now return ETH addresses for all Filecoin address types ("f0"/"f1"/"f2"/"f3") based on client's re-org tolerance. This is a breaking change if you are using the API via the go-jsonrpc library or by using Lotus as a library, but is a non-breaking change when using the API via any other RPC method as it adds an optional second argument.
-([filecoin-project/lotus#12324](https://github.com/filecoin-project/lotus/pull/12324)).
-- feat: Added `lotus-shed indexes inspect-events` health-check command ([filecoin-project/lotus#12346](https://github.com/filecoin-project/lotus/pull/12346)).
-
-## Enhancements
-
-- fix: add datacap balance to circ supply internal accounting as unCirc #12348
-- feat: Use a block cache to speed up the `EthGetBlockByHash` API: https://github.com/filecoin-project/lotus/pull/12359
-
 ## Improvements
 
-- feat!: gateway: fix rate limiting, better stateful handling ([filecoin-project/lotus#12315](https://github.com/filecoin-project/lotus/pull/12315)).
-  - CLI usage documentation has been improved for `lotus-gateway`
-  - `--per-conn-rate-limit` now works as advertised.
-  - `--eth-max-filters-per-conn` is new and allows you to set the maximum number of filters and subscription per connection, it defaults to 16.
-    - Previously, this limit was set to `16` and applied separately to filters and subscriptions. This limit is now unified and applies to both filters and subscriptions.
-  - Stateful Ethereum APIs (those involving filters and subscriptions) are now disabled for plain HTTP connections. A client must be using websockets to access these APIs.
-    - These APIs are also now automatically removed from the node by the gateway when a client disconnects.
-  - Some APIs have changed which may impact users consuming Lotus Gateway code as a library.
-  - The default value for the `Events.FilterTTL` config option has been reduced from 24h to 1h. This means that filters will expire on a Lotus node after 1 hour of not being accessed by the client.
-- feat(f3): F3 has been updated with many performance improvements and additional metrics.
-
 # 1.28.2 / 2024-08-15
 
 This is a Lotus patch release v1.28.2 for Node operators and Storage Providers.

build/openrpc/full.json

@@ -2,7 +2,7 @@
     "openrpc": "1.2.6",
     "info": {
         "title": "Lotus RPC API",
-        "version": "1.28.3-dev"
+        "version": "1.29.1-dev"
     },
     "methods": [
         {

build/openrpc/gateway.json

@@ -2,7 +2,7 @@
     "openrpc": "1.2.6",
     "info": {
         "title": "Lotus RPC API",
-        "version": "1.28.3-dev"
+        "version": "1.29.1-dev"
     },
     "methods": [
         {

build/openrpc/miner.json

@@ -2,7 +2,7 @@
     "openrpc": "1.2.6",
     "info": {
         "title": "Lotus RPC API",
-        "version": "1.28.3-dev"
+        "version": "1.29.1-dev"
     },
     "methods": [
         {

build/openrpc/worker.json

@@ -2,7 +2,7 @@
     "openrpc": "1.2.6",
     "info": {
         "title": "Lotus RPC API",
-        "version": "1.28.3-dev"
+        "version": "1.29.1-dev"
     },
     "methods": [
         {

build/version.go

@@ -7,7 +7,7 @@ import (
 )
 
 // NodeBuildVersion is the local build version of the Lotus daemon
-const NodeBuildVersion string = "1.28.3-dev"
+const NodeBuildVersion string = "1.29.1-dev"
 
 func NodeUserVersion() BuildVersion {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/lf3/f3.go

@@ -10,9 +10,6 @@ import (
 	logging "github.com/ipfs/go-log/v2"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/host"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/exporters/prometheus"
-	"go.opentelemetry.io/otel/sdk/metric"
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
@@ -53,20 +50,6 @@ type F3Params struct {
 
 var log = logging.Logger("f3")
 
-func init() {
-	// Set up otel to prometheus reporting so that F3 metrics are reported via lotus
-	// prometheus metrics. This bridge eventually gets picked up by opencensus
-	// exporter as HTTP handler. This by default registers an otel collector against
-	// the global prometheus registry. In the future, we should clean up metrics in
-	// Lotus and move it all to use otel. For now, bridge away.
-	if bridge, err := prometheus.New(); err != nil {
-		log.Errorf("could not create the otel prometheus exporter: %v", err)
-	} else {
-		provider := metric.NewMeterProvider(metric.WithReader(bridge))
-		otel.SetMeterProvider(provider)
-	}
-}
-
 func New(mctx helpers.MetricsCtx, lc fx.Lifecycle, params F3Params) (*F3, error) {
 
 	ds := namespace.Wrap(params.Datastore, datastore.NewKey("/f3"))

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.28.3-dev
+   1.29.1-dev
 
 COMMANDS:
    daemon   Start a lotus daemon process

go.mod

@@ -147,6 +147,7 @@ require (
 	go.opentelemetry.io/otel/bridge/opencensus v1.28.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.14.0
 	go.opentelemetry.io/otel/exporters/prometheus v0.50.0
+	go.opentelemetry.io/otel/metric v1.28.0
 	go.opentelemetry.io/otel/sdk v1.28.0
 	go.opentelemetry.io/otel/sdk/metric v1.28.0
 	go.uber.org/atomic v1.11.0
@@ -323,7 +324,6 @@ require (
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 	github.com/zondax/hid v0.9.2 // indirect
 	github.com/zondax/ledger-go v0.14.3 // indirect
-	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.uber.org/dig v1.17.1 // indirect
 	go.uber.org/mock v0.4.0 // indirect

metrics/otel_bridge.go

@@ -0,0 +1,21 @@
+package metrics
+
+import (
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/sdk/metric"
+)
+
+func init() {
+	// Set up otel to prometheus reporting so that F3 metrics are reported via lotus
+	// prometheus metrics. This bridge eventually gets picked up by opencensus
+	// exporter as HTTP handler. This by default registers an otel collector against
+	// the global prometheus registry. In the future, we should clean up metrics in
+	// Lotus and move it all to use otel. For now, bridge away.
+	if bridge, err := prometheus.New(); err != nil {
+		log.Errorf("could not create the otel prometheus exporter: %v", err)
+	} else {
+		provider := metric.NewMeterProvider(metric.WithReader(bridge))
+		otel.SetMeterProvider(provider)
+	}
+}

node/modules/lp2p/host.go

@@ -62,12 +62,6 @@ func Host(mctx helpers.MetricsCtx, buildVersion build.BuildVersion, lc fx.Lifecy
 		return nil, err
 	}
 
-	lc.Append(fx.Hook{
-		OnStop: func(ctx context.Context) error {
-			return h.Close()
-		},
-	})
-
 	return h, nil
 }
 

node/modules/lp2p/metrics.go

@@ -0,0 +1,30 @@
+package lp2p
+
+import (
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+var otelmeter = otel.Meter("libp2p")
+
+var attrIdentity = attribute.Key("identity")
+var attrProtocolID = attribute.Key("protocol")
+var attrDirectionInbound = attribute.String("direction", "inbound")
+var attrDirectionOutbound = attribute.String("direction", "outbound")
+
+var otelmetrics = struct {
+	bandwidth metric.Int64ObservableCounter
+}{
+	bandwidth: must(otelmeter.Int64ObservableCounter("lotus_libp2p_bandwidth",
+		metric.WithDescription("Libp2p stream bandwidth."),
+		metric.WithUnit("By"),
+	)),
+}
+
+func must[T any](v T, err error) T {
+	if err != nil {
+		panic(err)
+	}
+	return v
+}

node/modules/lp2p/transport.go

@@ -1,11 +1,16 @@
 package lp2p
 
 import (
+	"context"
+
 	"github.com/libp2p/go-libp2p"
 	"github.com/libp2p/go-libp2p/core/metrics"
+	"github.com/libp2p/go-libp2p/core/peer"
 	noise "github.com/libp2p/go-libp2p/p2p/security/noise"
 	tls "github.com/libp2p/go-libp2p/p2p/security/tls"
 	libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
+	"go.opentelemetry.io/otel/metric"
+	"go.uber.org/fx"
 )
 
 var DefaultTransports = simpleOpt(libp2p.DefaultTransports)
@@ -31,8 +36,33 @@ func Security(enabled, preferTLS bool) interface{} {
 	}
 }
 
-func BandwidthCounter() (opts Libp2pOpts, reporter metrics.Reporter) {
+func BandwidthCounter(lc fx.Lifecycle, id peer.ID) (opts Libp2pOpts, reporter metrics.Reporter, err error) {
 	reporter = metrics.NewBandwidthCounter()
 	opts.Opts = append(opts.Opts, libp2p.BandwidthReporter(reporter))
-	return opts, reporter
+
+	// Register it with open telemetry. We report by-callback instead of implementing a custom
+	// bandwidth counter to avoid allocating every time we read/write to a stream (and to stay
+	// out of the hot path).
+	//
+	// Identity is required to ensure this observer observes with unique attributes.
+	identityAttr := attrIdentity.String(id.String())
+	registration, err := otelmeter.RegisterCallback(func(ctx context.Context, obs metric.Observer) error {
+		for p, bw := range reporter.GetBandwidthByProtocol() {
+			if p == "" {
+				p = "<unknown>"
+			}
+			protoAttr := attrProtocolID.String(string(p))
+			obs.ObserveInt64(otelmetrics.bandwidth, bw.TotalOut,
+				metric.WithAttributes(identityAttr, protoAttr, attrDirectionOutbound))
+			obs.ObserveInt64(otelmetrics.bandwidth, bw.TotalIn,
+				metric.WithAttributes(identityAttr, protoAttr, attrDirectionInbound))
+		}
+		return nil
+	}, otelmetrics.bandwidth)
+	if err != nil {
+		return Libp2pOpts{}, nil, err
+	}
+	lc.Append(fx.StopHook(registration.Unregister))
+
+	return opts, reporter, nil
 }

storage/pipeline/pledge.go

@@ -104,7 +104,7 @@ func (m *Sealing) sectorWeight(ctx context.Context, sector SectorInfo, expiratio
 
 		alloc, err := piece.GetAllocation(ctx, m.Api, ts.Key())
 		if err != nil || alloc == nil {
-			if err == nil {
+			if err != nil {
 				log.Errorw("failed to get allocation", "error", err)
 			}
 			w = big.Add(w, big.Mul(sectorDuration, abi.NewStoragePower(int64(piece.Piece().Size))))