⭐ allow specifying a proxy just for Mondoo API connections (#470)

Allow cnspec to be told to use a specific proxy for connections to the
Mondoo API. In high-to-low order of precedence:

1) MONDOO_API_PROXY env var
2) --api-proxy CLI param
3) api_proxy value in config file

Update the various places in cnspec where connections are set up for the
Mondoo API to use the httpclient with proxy.

---------

Signed-off-by: Joel Diaz <joel@mondoo.com>
Co-authored-by: Christoph Hartmann <chris@lollyrock.com>

apps/cnspec/cmd/backgroundjob/healthping.go

@@ -2,6 +2,7 @@ package backgroundjob
 
 import (
 	"context"
+	"net/http"
 	"sync"
 	"time"
 
@@ -10,26 +11,28 @@ import (
 )
 
 type healthPinger struct {
-	ctx      context.Context
-	interval time.Duration
-	quit     chan struct{}
-	wg       sync.WaitGroup
-	endpoint string
+	ctx        context.Context
+	interval   time.Duration
+	quit       chan struct{}
+	wg         sync.WaitGroup
+	endpoint   string
+	httpClient *http.Client
 }
 
-func NewHealthPinger(ctx context.Context, endpoint string, interval time.Duration) *healthPinger {
+func NewHealthPinger(ctx context.Context, httpClient *http.Client, endpoint string, interval time.Duration) *healthPinger {
 	return &healthPinger{
-		ctx:      ctx,
-		interval: interval,
-		quit:     make(chan struct{}),
-		endpoint: endpoint,
+		ctx:        ctx,
+		interval:   interval,
+		quit:       make(chan struct{}),
+		endpoint:   endpoint,
+		httpClient: httpClient,
 	}
 }
 
 func (h *healthPinger) Start() {
 	h.wg.Add(1)
 	runHealthCheck := func() {
-		_, err := health.CheckApiHealth(h.endpoint)
+		_, err := health.CheckApiHealth(h.httpClient, h.endpoint)
 		if err != nil {
 			log.Info().Err(err).Msg("could not perform health check")
 		}

apps/cnspec/cmd/bundle.go

@@ -16,7 +16,6 @@ import (
 	"go.mondoo.com/cnquery/upstream"
 	"go.mondoo.com/cnspec/internal/bundle"
 	"go.mondoo.com/cnspec/policy"
-	"go.mondoo.com/ranger-rpc"
 )
 
 func init() {
@@ -202,7 +201,12 @@ var policyPublishCmd = &cobra.Command{
 			log.Error().Err(err).Msg(errorMessageServiceAccount)
 			os.Exit(cnquery_cmd.ConfigurationErrorCode)
 		}
-		queryHubServices, err := policy.NewPolicyHubClient(opts.UpstreamApiEndpoint(), ranger.DefaultHttpClient(), certAuth)
+
+		httpClient, err := opts.GetHttpClient()
+		if err != nil {
+			log.Fatal().Err(err).Msg("error while creating Mondoo API client")
+		}
+		queryHubServices, err := policy.NewPolicyHubClient(opts.UpstreamApiEndpoint(), httpClient, certAuth)
 		if err != nil {
 			log.Fatal().Err(err).Msg("could not connect to policy hub")
 		}

apps/cnspec/cmd/login.go

@@ -63,6 +63,13 @@ func register(token string) {
 	apiEndpoint := viper.GetString("api_endpoint")
 	token = strings.TrimSpace(token)
 
+	// NOTE: login is special because we do not have a config yet
+	proxy, err := cnquery_config.GetAPIProxy()
+	if err != nil {
+		log.Fatal().Err(err).Msg("could not parse proxy URL")
+	}
+	httpClient := ranger.NewHttpClient(ranger.WithProxy(proxy))
+
 	// we handle three cases here:
 	// 1. user has a token provided
 	// 2. user has no token provided, but has a service account file is already there
@@ -92,7 +99,7 @@ func register(token string) {
 		plugins = append(plugins, defaultPlugins...)
 		plugins = append(plugins, statictoken.NewRangerPlugin(token))
 
-		client, err := upstream.NewAgentManagerClient(apiEndpoint, ranger.DefaultHttpClient(), plugins...)
+		client, err := upstream.NewAgentManagerClient(apiEndpoint, httpClient, plugins...)
 		if err != nil {
 			log.Fatal().Err(err).Msg("could not connect to mondoo platform")
 		}
@@ -145,6 +152,11 @@ func register(token string) {
 		// print the used config to the user
 		config.DisplayUsedConfig()
 
+		httpClient, err = opts.GetHttpClient()
+		if err != nil {
+			log.Fatal().Err(err).Msg("could not create http client")
+		}
+
 		if opts.AgentMrn != "" {
 			// already authenticated
 			log.Info().Msg("client is already logged in, skipping")
@@ -162,7 +174,7 @@ func register(token string) {
 			}
 			plugins = append(plugins, certAuth)
 
-			client, err := upstream.NewAgentManagerClient(apiEndpoint, ranger.DefaultHttpClient(), plugins...)
+			client, err := upstream.NewAgentManagerClient(apiEndpoint, httpClient, plugins...)
 			if err != nil {
 				log.Fatal().Err(err).Msg("could not connect to Mondoo Platform")
 			}
@@ -211,7 +223,7 @@ func register(token string) {
 		os.Exit(cnquery_cmd.ConfigurationErrorCode)
 	}
 	plugins = append(plugins, certAuth)
-	client, err := upstream.NewAgentManagerClient(apiEndpoint, ranger.DefaultHttpClient(), plugins...)
+	client, err := upstream.NewAgentManagerClient(apiEndpoint, httpClient, plugins...)
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not connect to mondoo platform")
 	}

apps/cnspec/cmd/logout.go

@@ -12,7 +12,6 @@ import (
 	"go.mondoo.com/cnquery/cli/config"
 	"go.mondoo.com/cnquery/cli/sysinfo"
 	"go.mondoo.com/cnquery/upstream"
-	"go.mondoo.com/ranger-rpc"
 	"sigs.k8s.io/yaml"
 )
 
@@ -65,7 +64,11 @@ the credentials cannot be used in future anymore.
 		}
 		plugins = append(plugins, certAuth)
 
-		client, err := upstream.NewAgentManagerClient(opts.UpstreamApiEndpoint(), ranger.DefaultHttpClient(), plugins...)
+		httpClient, err := opts.GetHttpClient()
+		if err != nil {
+			log.Fatal().Err(err).Msg("error while creating Mondoo API client")
+		}
+		client, err := upstream.NewAgentManagerClient(opts.UpstreamApiEndpoint(), httpClient, plugins...)
 		if err != nil {
 			log.Error().Err(err).Msg("could not initialize connection to Mondoo Platform")
 			os.Exit(cnquery_cmd.ConfigurationErrorCode)

apps/cnspec/cmd/root.go

@@ -94,8 +94,10 @@ func init() {
 
 	rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose output")
 	rootCmd.PersistentFlags().String("log-level", "info", "Set log level: error, warn, info, debug, trace")
+	rootCmd.PersistentFlags().String("api-proxy", "", "Set proxy for communications with Mondoo API")
 	viper.BindPFlag("verbose", rootCmd.PersistentFlags().Lookup("verbose"))
 	viper.BindPFlag("log-level", rootCmd.PersistentFlags().Lookup("log-level"))
+	viper.BindPFlag("api-proxy", rootCmd.PersistentFlags().Lookup("api-proxy"))
 	viper.BindEnv("features")
 
 	config.Init(rootCmd)

apps/cnspec/cmd/scan.go

@@ -506,11 +506,18 @@ func getCobraScanConfig(cmd *cobra.Command, args []string, provider providers.Pr
 			log.Warn().Err(err).Msg("could not gather client information")
 		}
 		plugins = append(plugins, defaultRangerPlugins(sysInfo, opts.GetFeatures())...)
+		httpClient, err := opts.GetHttpClient()
+		if err != nil {
+			log.Error().Err(err).Msg("error setting up httpclient")
+			os.Exit(cnquery_cmd.ConfigurationErrorCode)
+
+		}
 		log.Info().Msg("using service account credentials")
 		conf.UpstreamConfig = &resources.UpstreamConfig{
 			SpaceMrn:    opts.GetParentMrn(),
 			ApiEndpoint: opts.UpstreamApiEndpoint(),
 			Plugins:     plugins,
+			HttpClient:  httpClient,
 		}
 	}
 
@@ -562,7 +569,7 @@ func RunScan(config *scanConfig, opts ...scan.ScannerOption) (*policy.ReportColl
 	scannerOpts = append(scannerOpts, opts...)
 
 	if config.UpstreamConfig != nil {
-		scannerOpts = append(scannerOpts, scan.WithUpstream(config.UpstreamConfig.ApiEndpoint, config.UpstreamConfig.SpaceMrn), scan.WithPlugins(config.UpstreamConfig.Plugins))
+		scannerOpts = append(scannerOpts, scan.WithUpstream(config.UpstreamConfig.ApiEndpoint, config.UpstreamConfig.SpaceMrn, config.UpstreamConfig.HttpClient), scan.WithPlugins(config.UpstreamConfig.Plugins))
 	}
 
 	// show warning to the user of the policy filter container a bundle file name

apps/cnspec/cmd/serve.go

@@ -75,7 +75,7 @@ var serveCmd = &cobra.Command{
 		ctx := cnquery.SetFeatures(context.Background(), cnquery.DefaultFeatures)
 
 		if conf != nil && conf.UpstreamConfig != nil {
-			hc := backgroundjob.NewHealthPinger(ctx, conf.UpstreamConfig.ApiEndpoint, 5*time.Minute)
+			hc := backgroundjob.NewHealthPinger(ctx, conf.UpstreamConfig.HttpClient, conf.UpstreamConfig.ApiEndpoint, 5*time.Minute)
 			hc.Start()
 			defer hc.Stop()
 		}
@@ -159,7 +159,17 @@ func getServeConfig() (*scanConfig, error) {
 		}
 	}
 
-	var err error
+	// set up the http client to include proxy config
+	httpClient, err := opts.GetHttpClient()
+	if err != nil {
+		log.Error().Err(err).Msg("error while setting up httpclient")
+		os.Exit(ConfigurationErrorCode)
+	}
+	if conf.UpstreamConfig == nil {
+		conf.UpstreamConfig = &resources.UpstreamConfig{}
+	}
+	conf.UpstreamConfig.HttpClient = httpClient
+
 	conf.Inventory, err = inventoryloader.ParseOrUse(nil, viper.GetBool("insecure"))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not load configuration")

apps/cnspec/cmd/serve_api.go

@@ -70,14 +70,20 @@ var serveApiCmd = &cobra.Command{
 			log.Warn().Err(err).Msg("could not gather client information")
 		}
 		plugins = append(plugins, defaultRangerPlugins(sysInfo, opts.GetFeatures())...)
+		httpClient, err := opts.GetHttpClient()
+		if err != nil {
+			log.Error().Err(err).Msg("error seting up http client")
+			os.Exit(cnquery_cmd.ConfigurationErrorCode)
+		}
 		log.Info().Msg("using service account credentials")
 		upstreamConfig := resources.UpstreamConfig{
 			SpaceMrn:    opts.GetParentMrn(),
 			ApiEndpoint: opts.UpstreamApiEndpoint(),
 			Plugins:     plugins,
+			HttpClient:  httpClient,
 		}
 
-		scanner := scan.NewLocalScanner(scan.WithUpstream(upstreamConfig.ApiEndpoint, upstreamConfig.SpaceMrn), scan.WithPlugins(plugins), scan.DisableProgressBar())
+		scanner := scan.NewLocalScanner(scan.WithUpstream(upstreamConfig.ApiEndpoint, upstreamConfig.SpaceMrn, upstreamConfig.HttpClient), scan.WithPlugins(plugins), scan.DisableProgressBar())
 		if err := scanner.EnableQueue(); err != nil {
 			log.Fatal().Err(err).Msg("could not enable scan queue")
 		}

apps/cnspec/cmd/status.go

@@ -66,6 +66,11 @@ Status sends a ping to Mondoo Platform to verify the credentials.
 			log.Fatal().Err(err).Send()
 		}
 
+		httpClient, err := opts.GetHttpClient()
+		if err != nil {
+			log.Fatal().Err(err).Msg("failed to set up Mondoo API client")
+		}
+
 		sysInfo, err := sysinfo.GatherSystemInfo(sysinfo.WithMotor(m))
 		if err == nil {
 			s.Client.Platform = sysInfo.Platform
@@ -74,7 +79,7 @@ Status sends a ping to Mondoo Platform to verify the credentials.
 		}
 
 		// check server health and clock skew
-		upstreamStatus, err := health.CheckApiHealth(opts.UpstreamApiEndpoint())
+		upstreamStatus, err := health.CheckApiHealth(httpClient, opts.UpstreamApiEndpoint())
 		if err != nil {
 			log.Error().Err(err).Msg("could not check upstream health")
 		}
@@ -103,7 +108,7 @@ Status sends a ping to Mondoo Platform to verify the credentials.
 			plugins = append(plugins, certAuth)
 
 			// try to ping the server
-			client, err := upstream.NewAgentManagerClient(s.Upstream.API.Endpoint, ranger.DefaultHttpClient(), plugins...)
+			client, err := upstream.NewAgentManagerClient(s.Upstream.API.Endpoint, httpClient, plugins...)
 			if err == nil {
 				_, err = client.PingPong(context.Background(), &upstream.Ping{})
 				if err != nil {

go.mod

@@ -2,6 +2,8 @@ module go.mondoo.com/cnspec
 
 go 1.19
 
+replace github.com/slack-go/slack v0.12.1 => github.com/imilchev/slack v0.0.0-20230324120548-5380d7dd00a5
+
 require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/cockroachdb/errors v1.9.1
@@ -26,10 +28,10 @@ require (
 	github.com/spf13/pflag v1.0.6-0.20201009195203-85dd5c8bc61c
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.2
-	go.mondoo.com/cnquery v0.0.0-20230321163439-9786ebf33b26
-	go.mondoo.com/ranger-rpc v0.5.1-0.20220923135836-9e7732899d34
+	go.mondoo.com/cnquery v0.0.0-20230328163439-27fcddaefc71
+	go.mondoo.com/ranger-rpc v0.0.0-20230328135530-12135c17095f
 	go.opentelemetry.io/otel v1.14.0
-	golang.org/x/sync v0.1.0
+	golang.org/x/sync v0.1.0 // indirect
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/yaml.v3 v3.0.1
@@ -421,7 +423,7 @@ require (
 	github.com/sivchari/containedctx v1.0.2 // indirect
 	github.com/sivchari/nosnakecase v1.7.0 // indirect
 	github.com/sivchari/tenv v1.7.1 // indirect
-	github.com/slack-go/slack v0.11.4 // indirect
+	github.com/slack-go/slack v0.12.1 // indirect
 	github.com/sonatard/noctx v0.0.1 // indirect
 	github.com/sourcegraph/go-diff v0.7.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect

go.sum

@@ -791,6 +791,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imilchev/slack v0.0.0-20230324120548-5380d7dd00a5 h1:aNFjgSeqhIzc+g0SyzXxypHI87MzN32fCd2kI6OD/1o=
+github.com/imilchev/slack v0.0.0-20230324120548-5380d7dd00a5/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -1213,8 +1215,6 @@ github.com/sivchari/nosnakecase v1.7.0 h1:7QkpWIRMe8x25gckkFd2A5Pi6Ymo0qgr4JrhGt
 github.com/sivchari/nosnakecase v1.7.0/go.mod h1:CwDzrzPea40/GB6uynrNLiorAlgFRvRbFSgJx2Gs+QY=
 github.com/sivchari/tenv v1.7.1 h1:PSpuD4bu6fSmtWMxSGWcvqUUgIn7k3yOJhOIzVWn8Ak=
 github.com/sivchari/tenv v1.7.1/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl018Icvg=
-github.com/slack-go/slack v0.11.4 h1:ojSa7KlPm3PqY2AomX4VTxEsK5eci5JaxCjlzGV5zoM=
-github.com/slack-go/slack v0.11.4/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -1344,10 +1344,10 @@ github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0
 github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
 gitlab.com/bosi/decorder v0.2.3 h1:gX4/RgK16ijY8V+BRQHAySfQAb354T7/xQpDB2n10P0=
 gitlab.com/bosi/decorder v0.2.3/go.mod h1:9K1RB5+VPNQYtXtTDAzd2OEftsZb1oV0IrJrzChSdGE=
-go.mondoo.com/cnquery v0.0.0-20230321163439-9786ebf33b26 h1:y8qE52qMNRqb/9B5/4EY04Y8hoj5DMC8EM4GDEjC47w=
-go.mondoo.com/cnquery v0.0.0-20230321163439-9786ebf33b26/go.mod h1:8GAmoT3wwGRp7lru1K13r/mDIN2Yq8Ri0mgEPGjK0SE=
-go.mondoo.com/ranger-rpc v0.5.1-0.20220923135836-9e7732899d34 h1:mtPZ1J+nRI/ivV+n41bjIwY6Rfxb2Jf49svZSQMGHIA=
-go.mondoo.com/ranger-rpc v0.5.1-0.20220923135836-9e7732899d34/go.mod h1:3YKcqFrlPgaB4FZ4EoLgdmRtwMQdO7RoAkZYFn+F1eY=
+go.mondoo.com/cnquery v0.0.0-20230328163439-27fcddaefc71 h1:6e9rvxGVnwxA2uwjSG7sVv5SDuSCTz5FaW5xsaoK5dc=
+go.mondoo.com/cnquery v0.0.0-20230328163439-27fcddaefc71/go.mod h1:+s3BWL2KwtEm36pm+FNtdZ1op1n+hdIzWfTkb6hkI6E=
+go.mondoo.com/ranger-rpc v0.0.0-20230328135530-12135c17095f h1:l8N+cU5Ul8+NzC3DtyjrUqpzSDpPQDnGqvxpwMz7CMw=
+go.mondoo.com/ranger-rpc v0.0.0-20230328135530-12135c17095f/go.mod h1:3YKcqFrlPgaB4FZ4EoLgdmRtwMQdO7RoAkZYFn+F1eY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

policy/hub.go

@@ -278,6 +278,8 @@ func (s *LocalServices) DefaultPolicies(ctx context.Context, req *DefaultPolicie
 		registryEndpoint = defaultRegistryUrl
 	}
 
+	// Note, this does not use the proxy config override from the mondoo.yml since we only get here when
+	// it is used without upstream config
 	client, err := NewPolicyHubClient(registryEndpoint, ranger.DefaultHttpClient())
 	if err != nil {
 		return nil, err