Add GCP Cloud profiler support and update Go to 1.21 (thought-machine…

…#274)

* Add GCP Cloud profiler support

* Undo a couple of Puku changes

* Update Go and all packages to latest

* Attempt to update to later versions of Go

* Missed the dockerfile

* And latest Docker tags as well

* Try removing systemtoolchain?

* Revert to older image

* Attempt to calm the linter

* Actually revert the SDK override

* Attempt to get around the linter?

* Bump golang CI?

* Fix defer for timer

* Attempt go.mod surgery

* Silence linter, add

.github/workflows/codeql-analysis.yml

@@ -17,8 +17,7 @@ on:
 jobs:
   analyze:
     name: Analyze
-    # TODO(jpoole): set this back to ubuntu-latest once https://github.com/actions/virtual-environments/issues/1816 lands
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false

.github/workflows/golangci-lint.yml

@@ -14,9 +14,9 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '^1.20'
+          go-version: '^1.21'
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.52.0
+          version: v1.55.2
           args: cli/... discern/... elan/... flair/... grpcutil/... lucidity/... mettle/... purity/... rexclient/... zeal/...

.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
     needs: [test]
     runs-on: ubuntu-latest
     container:
-      image: thoughtmachine/please-servers:20220816
+      image: thoughtmachine/please-servers:20230319
     if: github.ref == 'refs/heads/master'
     steps:
       - name: Checkout code

.golangci.yml

@@ -11,6 +11,7 @@ issues:
     - SA5008  # Warns for duplicate struct tags which are meaningful to go-flags
     - unslice  # This may be useful in some places.
     - ifElseChain  # Generally don't agree.
+    - indent-error-flow # Similar to above, seems to be making questionable choices
     - appendAssign
     - halp  # Make misspell be quiet about this.
     - exitAfterDefer # Potentially useful but not in any cases it fires right now.

.plzconfig.ci

@@ -4,8 +4,9 @@ dir = .plz-cache
 dircompress = true
 
 [plugin "go"]
+target = //plugins:go
 defaultstatic = true
-gotool = //third_party/go:system_toolchain|go
+gotool = //third_party/go:toolchain|go
 coverageredesign = true
 
 [Plugin "proto"]

BUILD

@@ -8,4 +8,4 @@ filegroup(
     name = "go.mod",
     srcs = ["go.mod"],
     visibility = ["PUBLIC"],
-)
+)

cli/BUILD

@@ -8,6 +8,7 @@ go_library(
         "///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
         "///third_party/go/github.com_thought-machine_http-admin//:http-admin",
         "///third_party/go/go.uber.org_automaxprocs//maxprocs",
+        "///third_party/go/cloud.google.com_go_profiler//:profiler",
     ],
 )
 
@@ -16,6 +17,6 @@ go_test(
     srcs = ["cli_test.go"],
     deps = [
         ":cli",
-        "//third_party/go:testify",
+        "///third_party/go/github.com_stretchr_testify//assert",
     ],
 )

cli/cli.go

@@ -10,8 +10,10 @@ import (
 	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
 	"github.com/peterebden/go-cli-init/v4/flags"
 	"github.com/peterebden/go-cli-init/v4/logging"
-	"github.com/thought-machine/http-admin"
+	admin "github.com/thought-machine/http-admin"
 	"go.uber.org/automaxprocs/maxprocs"
+
+	"cloud.google.com/go/profiler"
 )
 
 var log = logging.MustGetLogger()
@@ -29,24 +31,41 @@ type LoggingOpts struct {
 }
 
 // AdminOpts is a re-export of the admin type so servers don't need to import it directly.
-type AdminOpts = admin.Opts
+type AdminOpts struct {
+	Admin              admin.Opts
+	EnableGcpProfiling bool `long:"gcp_profiling" description:"Enable pushing profiles to GCP Cloud profiling." env:"ADMIN_GCP_PROFILING"`
+}
 
 // ParseFlagsOrDie parses incoming flags and sets up logging etc.
 func ParseFlagsOrDie(name string, opts interface{}, loggingOpts *LoggingOpts) (string, logging.LogLevelInfo) {
 	cmd := flags.ParseFlagsOrDie(name, opts)
 	info := logging.MustInitStructuredLogging(loggingOpts.Verbosity, loggingOpts.FileVerbosity, loggingOpts.LogFile, loggingOpts.Structured)
 	if _, err := maxprocs.Set(maxprocs.Logger(log.Notice), maxprocs.Min(1)); err != nil {
-		log.Error("Failed to set GOMAXPROCS: %s", err)
+		log.Errorf("Failed to set GOMAXPROCS: %s", err)
 	}
 	return cmd, info
 }
 
 // ServeAdmin starts the admin HTTP server.
 // It will block forever so the caller may well want to use a goroutine.
-func ServeAdmin(opts AdminOpts, info logging.LogLevelInfo) {
-	opts.Logger = logging.MustGetLoggerNamed("github.com.thought-machine.http-admin")
-	opts.LogInfo = info
-	go admin.Serve(opts)
+func ServeAdmin(serviceName string, opts AdminOpts, info logging.LogLevelInfo) {
+	opts.Admin.Logger = logging.MustGetLoggerNamed("github.com.thought-machine.http-admin")
+	opts.Admin.LogInfo = info
+	if opts.EnableGcpProfiling {
+		setupProfiling(serviceName)
+	}
+	go admin.Serve(opts.Admin)
+}
+
+func setupProfiling(serviceName string) {
+	cfg := profiler.Config{
+		Service: serviceName,
+	}
+
+	// Profiler initialization, best done as early as possible.
+	if err := profiler.Start(cfg); err != nil {
+		log.Warningf("Failed to set up profiling, continuing anyway: %s", err)
+	}
 }
 
 // An Action represents a combined hash / size pair written like

elan/main.go

@@ -33,6 +33,6 @@ modes are intended for testing only.
 
 func main() {
 	_, info := cli.ParseFlagsOrDie("Elan", &opts, &opts.Logging)
-	go cli.ServeAdmin(opts.Admin, info)
+	go cli.ServeAdmin("Elan", opts.Admin, info)
 	rpc.ServeForever(opts.GRPC, opts.Storage, opts.Parallelism, opts.DirCacheSize, int64(opts.KnownBlobCacheSize))
 }

elan/rpc/BUILD

@@ -48,7 +48,8 @@ go_test(
         "///third_party/go/github.com_klauspost_compress//zstd",
         "///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
         "//grpcutil",
-        "//third_party/go:testify",
+        "///third_party/go/github.com_stretchr_testify//assert",
+        "///third_party/go/github.com_stretchr_testify//require",
     ],
 )
 

elan/rpc/rpc.go

@@ -652,7 +652,9 @@ func (s *server) writeBlob(ctx context.Context, prefix string, digest *pb.Digest
 	s.limiter <- struct{}{}
 	defer func() { <-s.limiter }()
 	start := time.Now()
-	defer writeLatencies.Observe(time.Since(start).Seconds())
+	defer func() {
+		writeLatencies.Observe(time.Since(start).Seconds())
+	}()
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	w, err := s.bucket.NewWriter(ctx, key, &blob.WriterOptions{BufferSize: s.bufferSize(digest)})

flair/main.go

@@ -47,7 +47,7 @@ want to have more than the minimum number of instances of it (hopefully more tha
 
 func main() {
 	_, info := cli.ParseFlagsOrDie("Flair", &opts, &opts.Logging)
-	go cli.ServeAdmin(opts.Admin, info)
+	go cli.ServeAdmin("Flair", opts.Admin, info)
 	cr := newReplicator(opts.Geometry, opts.Replicas, opts.LoadBalance)
 	ar := newReplicator(opts.AssetGeometry, opts.Replicas, opts.LoadBalance)
 	er := newReplicator(opts.ExecutorGeometry, opts.Replicas, opts.LoadBalance)

flair/trie/BUILD

@@ -24,6 +24,6 @@ go_test(
         ":trie",
         "///third_party/go/github.com_hashicorp_go-multierror//:go-multierror",
         "//third_party/go:grpc",
-        "//third_party/go:testify",
+        "///third_party/go/github.com_stretchr_testify//assert",
     ],
 )

go.mod

@@ -1,93 +1,97 @@
 module github.com/thought-machine/please-servers
 
-go 1.18
+go 1.21
 
 require (
-	cloud.google.com/go/pubsub v1.30.0
-	cloud.google.com/go/storage v1.28.1
+	cloud.google.com/go/profiler v0.4.0
+	cloud.google.com/go/pubsub v1.33.0
+	cloud.google.com/go/storage v1.36.0
 	github.com/bazelbuild/remote-apis v0.0.0-20230411132548-35aee1c4a425
 	github.com/bazelbuild/remote-apis-sdks v0.0.0-20230419185642-269815af5db1
-	github.com/dgraph-io/ristretto v0.0.3
-	github.com/dustin/go-humanize v1.0.0
-	github.com/go-redis/redis/v8 v8.8.3
+	github.com/dgraph-io/ristretto v0.1.1
+	github.com/dustin/go-humanize v1.0.1
+	github.com/go-redis/redis/v8 v8.11.5
 	github.com/golang/protobuf v1.5.3
-	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0
-	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5
-	github.com/hashicorp/go-multierror v1.1.0
-	github.com/hashicorp/go-retryablehttp v0.6.6
-	github.com/klauspost/compress v1.12.3
-	github.com/mostynb/go-grpc-compression v1.1.6
-	github.com/peterebden/go-cli-init/v4 v4.0.1
+	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0
+	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1
+	github.com/hashicorp/go-multierror v1.1.1
+	github.com/hashicorp/go-retryablehttp v0.7.5
+	github.com/klauspost/compress v1.17.4
+	github.com/mostynb/go-grpc-compression v1.2.2
+	github.com/peterebden/go-cli-init/v4 v4.0.2
 	github.com/peterebden/go-copyfile v0.0.0-20200424115000-bc0baf74909c
 	github.com/peterebden/go-sri v1.1.1
-	github.com/prometheus/client_golang v1.14.0
-	github.com/prometheus/common v0.37.0
-	github.com/shirou/gopsutil v3.21.1+incompatible
-	github.com/stretchr/testify v1.8.1
+	github.com/prometheus/client_golang v1.18.0
+	github.com/prometheus/common v0.45.0
+	github.com/shirou/gopsutil v3.21.11+incompatible
+	github.com/stretchr/testify v1.8.4
 	github.com/thought-machine/http-admin v1.1.0
-	go.uber.org/automaxprocs v1.4.0
-	gocloud.dev v0.20.0
-	golang.org/x/crypto v0.1.0
-	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
-	golang.org/x/sync v0.1.0
-	golang.org/x/time v0.3.0
-	google.golang.org/api v0.114.0
-	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc
-	google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc
-	google.golang.org/grpc v1.55.0
-	google.golang.org/protobuf v1.30.0
+	go.uber.org/automaxprocs v1.5.3
+	gocloud.dev v0.36.0
+	github.com/golang/snappy v0.0.4 // This is a required dep that Go mod tidy gets rid of. This breaks Please if it gets removed though.
+    github.com/google/go-cmp v0.6.0 // This is a required dep that Go mod tidy gets rid of. This breaks Please if it gets removed though.
+	golang.org/x/crypto v0.17.0
+	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
+	golang.org/x/sync v0.6.0
+	golang.org/x/time v0.5.0
+	google.golang.org/api v0.155.0
+	google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917
+	google.golang.org/genproto/googleapis/bytestream v0.0.0-20240102182953-50ed04b92917
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917
+	google.golang.org/grpc v1.60.1
+	google.golang.org/protobuf v1.32.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 )
 
 require (
-	cloud.google.com/go v0.110.0 // indirect
-	cloud.google.com/go/compute v1.19.0 // indirect
+	cloud.google.com/go v0.111.0 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.13.0 // indirect
-	cloud.google.com/go/longrunning v0.4.1 // indirect
-	github.com/StackExchange/wmi v1.2.1 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
+	cloud.google.com/go/longrunning v0.5.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/go-ole/go-ole v1.2.5 // indirect
-	github.com/golang/glog v1.1.0 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/golang/snappy v0.0.3 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/google/wire v0.4.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
-	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
-	github.com/googleapis/gax-go/v2 v2.7.1 // indirect
-	github.com/gorilla/mux v1.7.4 // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/mostynb/zstdpool-syncpool v0.0.7 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/golang/glog v1.2.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/pprof v0.0.0-20231229205709-960ae82b1e42 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/google/wire v0.5.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/mostynb/zstdpool-syncpool v0.0.13 // indirect
 	github.com/pborman/uuid v1.2.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pkg/xattr v0.4.4 // indirect
+	github.com/pkg/xattr v0.4.9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/google/s2a-go v0.1.4 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/thought-machine/go-flags v1.5.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/thought-machine/go-flags v1.6.3 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/otel v0.20.0 // indirect
-	go.opentelemetry.io/otel/metric v0.20.0 // indirect
-	go.opentelemetry.io/otel/trace v0.20.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/oauth2 v0.6.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
+	go.opentelemetry.io/otel/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.21.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/oauth2 v0.15.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/term v0.16.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )