Add flag to delay osquery healthchecks

kolide · Aug 18, 2023 · 198bd6e · 198bd6e
1 parent ce017a3
commit 198bd6e
Show file tree

Hide file tree

Showing 9 changed files with 159 additions and 70 deletions.
diff --git a/cmd/launcher/options.go b/cmd/launcher/options.go
@@ -64,29 +64,30 @@ func parseOptions(subcommandName string, args []string) (*launcher.Options, erro
 
 	var (
 		// Primary options
-		flAutoloadedExtensions   arrayFlags
-		flCertPins               = flagset.String("cert_pins", "", "Comma separated, hex encoded SHA256 hashes of pinned subject public key info")
-		flControlRequestInterval = flagset.Duration("control_request_interval", 60*time.Second, "The interval at which the control server requests will be made")
-		flEnrollSecret           = flagset.String("enroll_secret", "", "The enroll secret that is used in your environment")
-		flEnrollSecretPath       = flagset.String("enroll_secret_path", "", "Optionally, the path to your enrollment secret")
-		flInitialRunner          = flagset.Bool("with_initial_runner", false, "Run differential queries from config ahead of scheduled interval.")
-		flKolideServerURL        = flagset.String("hostname", "", "The hostname of the gRPC server")
-		flKolideHosted           = flagset.Bool("kolide_hosted", false, "Use Kolide SaaS settings for defaults")
-		flTransport              = flagset.String("transport", "grpc", "The transport protocol that should be used to communicate with remote (default: grpc)")
-		flLoggingInterval        = flagset.Duration("logging_interval", 60*time.Second, "The interval at which logs should be flushed to the server")
-		flOsquerydPath           = flagset.String("osqueryd_path", "", "Path to the osqueryd binary to use (Default: find osqueryd in $PATH)")
-		flRootDirectory          = flagset.String("root_directory", defaultRootDirectoryPath, "The location of the local database, pidfiles, etc.")
-		flRootPEM                = flagset.String("root_pem", "", "Path to PEM file including root certificates to verify against")
-		flVersion                = flagset.Bool("version", false, "Print Launcher version and exit")
-		flLogMaxBytesPerBatch    = flagset.Int("log_max_bytes_per_batch", 0, "Maximum size of a batch of logs. Recommend leaving unset, and launcher will determine")
-		flOsqueryFlags           arrayFlags // set below with flagset.Var
-		flCompactDbMaxTx         = flagset.Int64("compactdb-max-tx", 65536, "Maximum transaction size used when compacting the internal DB")
-		flConfigFilePath         = flagset.String("config", defaultConfigFilePath, "config file to parse options from (optional)")
-		flExportTraces           = flagset.Bool("export_traces", false, "Whether to export traces")
-		flTraceSamplingRate      = flagset.Float64("trace_sampling_rate", 0.0, "What fraction of traces should be sampled")
-		flLogIngestServerURL     = flagset.String("log_ingest_url", "", "Where to export logs")
-		flTraceIngestServerURL   = flagset.String("trace_ingest_url", "", "Where to export traces")
-		flDisableIngestTLS       = flagset.Bool("disable_trace_ingest_tls", false, "Disable TLS for observability ingest server communication")
+		flAutoloadedExtensions           arrayFlags
+		flCertPins                       = flagset.String("cert_pins", "", "Comma separated, hex encoded SHA256 hashes of pinned subject public key info")
+		flControlRequestInterval         = flagset.Duration("control_request_interval", 60*time.Second, "The interval at which the control server requests will be made")
+		flEnrollSecret                   = flagset.String("enroll_secret", "", "The enroll secret that is used in your environment")
+		flEnrollSecretPath               = flagset.String("enroll_secret_path", "", "Optionally, the path to your enrollment secret")
+		flInitialRunner                  = flagset.Bool("with_initial_runner", false, "Run differential queries from config ahead of scheduled interval.")
+		flKolideServerURL                = flagset.String("hostname", "", "The hostname of the gRPC server")
+		flKolideHosted                   = flagset.Bool("kolide_hosted", false, "Use Kolide SaaS settings for defaults")
+		flTransport                      = flagset.String("transport", "grpc", "The transport protocol that should be used to communicate with remote (default: grpc)")
+		flLoggingInterval                = flagset.Duration("logging_interval", 60*time.Second, "The interval at which logs should be flushed to the server")
+		flOsquerydPath                   = flagset.String("osqueryd_path", "", "Path to the osqueryd binary to use (Default: find osqueryd in $PATH)")
+		flOsqueryHealthcheckStartupDelay = flagset.Duration("osquery_healthcheck_startup_delay", 10*time.Minute, "time to wait before beginning osquery healthchecks")
+		flRootDirectory                  = flagset.String("root_directory", defaultRootDirectoryPath, "The location of the local database, pidfiles, etc.")
+		flRootPEM                        = flagset.String("root_pem", "", "Path to PEM file including root certificates to verify against")
+		flVersion                        = flagset.Bool("version", false, "Print Launcher version and exit")
+		flLogMaxBytesPerBatch            = flagset.Int("log_max_bytes_per_batch", 0, "Maximum size of a batch of logs. Recommend leaving unset, and launcher will determine")
+		flOsqueryFlags                   arrayFlags // set below with flagset.Var
+		flCompactDbMaxTx                 = flagset.Int64("compactdb-max-tx", 65536, "Maximum transaction size used when compacting the internal DB")
+		flConfigFilePath                 = flagset.String("config", defaultConfigFilePath, "config file to parse options from (optional)")
+		flExportTraces                   = flagset.Bool("export_traces", false, "Whether to export traces")
+		flTraceSamplingRate              = flagset.Float64("trace_sampling_rate", 0.0, "What fraction of traces should be sampled")
+		flLogIngestServerURL             = flagset.String("log_ingest_url", "", "Where to export logs")
+		flTraceIngestServerURL           = flagset.String("trace_ingest_url", "", "Where to export traces")
+		flDisableIngestTLS               = flagset.Bool("disable_trace_ingest_tls", false, "Disable TLS for observability ingest server communication")
 
 		// osquery TLS endpoints
 		flOsqTlsConfig    = flagset.String("config_tls_endpoint", "", "Config endpoint for the osquery tls transport")
@@ -272,6 +273,7 @@ func parseOptions(subcommandName string, args []string) (*launcher.Options, erro
 		OsqueryTlsLoggerEndpoint:           *flOsqTlsLogger,
 		OsqueryVerbose:                     *flOsqueryVerbose,
 		OsquerydPath:                       osquerydPath,
+		OsqueryHealthcheckStartupDelay:     *flOsqueryHealthcheckStartupDelay,
 		RootDirectory:                      *flRootDirectory,
 		RootPEM:                            *flRootPEM,
 		TraceSamplingRate:                  *flTraceSamplingRate,

diff --git a/pkg/agent/flags/flag_controller.go b/pkg/agent/flags/flag_controller.go
@@ -500,3 +500,14 @@ func (fc *FlagController) InModernStandby() bool {
 		WithDefaultBool(false),
 	).get(fc.getControlServerValue(keys.InModernStandby))
 }
+
+func (fc *FlagController) SetOsqueryHealthcheckStartupDelay(delay time.Duration) error {
+	return fc.setControlServerValue(keys.OsqueryHealthcheckStartupDelay, durationToBytes(delay))
+}
+func (fc *FlagController) OsqueryHealthcheckStartupDelay() time.Duration {
+	return NewDurationFlagValue(fc.logger, keys.OsqueryHealthcheckStartupDelay,
+		WithDefault(fc.cmdLineOpts.OsqueryHealthcheckStartupDelay),
+		WithMin(0*time.Second),
+		WithMax(1*time.Hour),
+	).get(fc.getControlServerValue(keys.OsqueryHealthcheckStartupDelay))
+}
diff --git a/pkg/agent/flags/keys/keys.go b/pkg/agent/flags/keys/keys.go
@@ -11,43 +11,44 @@ type FlagKey string
 // 4. Implement tests for any new APIs, sanitizers, limits, overrides.
 // 5. Update mocks -- in pkg/agent/types, run `mockery --name Knapsack` and `mockery --name Flags`.
 const (
-	KolideServerURL            FlagKey = "hostname"
-	KolideHosted               FlagKey = "kolide_hosted"
-	Transport                  FlagKey = "transport"
-	LoggingInterval            FlagKey = "logging_interval"
-	OsquerydPath               FlagKey = "osqueryd_path"
-	RootDirectory              FlagKey = "root_directory"
-	RootPEM                    FlagKey = "root_pem"
-	DesktopEnabled             FlagKey = "desktop_enabled_v1"
-	DesktopUpdateInterval      FlagKey = "desktop_update_interval"
-	DesktopMenuRefreshInterval FlagKey = "desktop_menu_refresh_interval"
-	DebugServerData            FlagKey = "debug_server_data"
-	ForceControlSubsystems     FlagKey = "force_control_subsystems"
-	ControlServerURL           FlagKey = "control_server_url"
-	ControlRequestInterval     FlagKey = "control_request_interval"
-	DisableControlTLS          FlagKey = "disable_control_tls"
-	InsecureControlTLS         FlagKey = "insecure_control_tls"
-	InsecureTLS                FlagKey = "insecure_tls"
-	InsecureTransportTLS       FlagKey = "insecure_transport"
-	IAmBreakingEELicense       FlagKey = "i-am-breaking-ee-license"
-	Debug                      FlagKey = "debug"
-	DebugLogFile               FlagKey = "debug_log_file"
-	OsqueryVerbose             FlagKey = "osquery_verbose"
-	Autoupdate                 FlagKey = "autoupdate"
-	NotaryServerURL            FlagKey = "notary_url"
-	TufServerURL               FlagKey = "tuf_url"
-	MirrorServerURL            FlagKey = "mirror_url"
-	AutoupdateInterval         FlagKey = "autoupdate_interval"
-	UpdateChannel              FlagKey = "update_channel"
-	NotaryPrefix               FlagKey = "notary_prefix"
-	AutoupdateInitialDelay     FlagKey = "autoupdater_initial_delay"
-	UpdateDirectory            FlagKey = "update_directory"
-	ExportTraces               FlagKey = "export_traces"
-	TraceSamplingRate          FlagKey = "trace_sampling_rate"
-	LogIngestServerURL         FlagKey = "log_ingest_url"
-	TraceIngestServerURL       FlagKey = "trace_ingest_url"
-	DisableTraceIngestTLS      FlagKey = "disable_trace_ingest_tls"
-	InModernStandby            FlagKey = "in_modern_standby"
+	KolideServerURL                FlagKey = "hostname"
+	KolideHosted                   FlagKey = "kolide_hosted"
+	Transport                      FlagKey = "transport"
+	LoggingInterval                FlagKey = "logging_interval"
+	OsquerydPath                   FlagKey = "osqueryd_path"
+	OsqueryHealthcheckStartupDelay FlagKey = "osquery_healthcheck_startup_delay"
+	RootDirectory                  FlagKey = "root_directory"
+	RootPEM                        FlagKey = "root_pem"
+	DesktopEnabled                 FlagKey = "desktop_enabled_v1"
+	DesktopUpdateInterval          FlagKey = "desktop_update_interval"
+	DesktopMenuRefreshInterval     FlagKey = "desktop_menu_refresh_interval"
+	DebugServerData                FlagKey = "debug_server_data"
+	ForceControlSubsystems         FlagKey = "force_control_subsystems"
+	ControlServerURL               FlagKey = "control_server_url"
+	ControlRequestInterval         FlagKey = "control_request_interval"
+	DisableControlTLS              FlagKey = "disable_control_tls"
+	InsecureControlTLS             FlagKey = "insecure_control_tls"
+	InsecureTLS                    FlagKey = "insecure_tls"
+	InsecureTransportTLS           FlagKey = "insecure_transport"
+	IAmBreakingEELicense           FlagKey = "i-am-breaking-ee-license"
+	Debug                          FlagKey = "debug"
+	DebugLogFile                   FlagKey = "debug_log_file"
+	OsqueryVerbose                 FlagKey = "osquery_verbose"
+	Autoupdate                     FlagKey = "autoupdate"
+	NotaryServerURL                FlagKey = "notary_url"
+	TufServerURL                   FlagKey = "tuf_url"
+	MirrorServerURL                FlagKey = "mirror_url"
+	AutoupdateInterval             FlagKey = "autoupdate_interval"
+	UpdateChannel                  FlagKey = "update_channel"
+	NotaryPrefix                   FlagKey = "notary_prefix"
+	AutoupdateInitialDelay         FlagKey = "autoupdater_initial_delay"
+	UpdateDirectory                FlagKey = "update_directory"
+	ExportTraces                   FlagKey = "export_traces"
+	TraceSamplingRate              FlagKey = "trace_sampling_rate"
+	LogIngestServerURL             FlagKey = "log_ingest_url"
+	TraceIngestServerURL           FlagKey = "trace_ingest_url"
+	DisableTraceIngestTLS          FlagKey = "disable_trace_ingest_tls"
+	InModernStandby                FlagKey = "in_modern_standby"
 )
 
 func (key FlagKey) String() string {

diff --git a/pkg/agent/knapsack/knapsack.go b/pkg/agent/knapsack/knapsack.go
@@ -389,3 +389,10 @@ func (k *knapsack) SetInModernStandby(enabled bool) error {
 func (k *knapsack) InModernStandby() bool {
 	return k.flags.InModernStandby()
 }
+
+func (k *knapsack) SetOsqueryHealthcheckStartupDelay(delay time.Duration) error {
+	return k.flags.SetOsqueryHealthcheckStartupDelay(delay)
+}
+func (k *knapsack) OsqueryHealthcheckStartupDelay() time.Duration {
+	return k.flags.OsqueryHealthcheckStartupDelay()
+}
diff --git a/pkg/agent/types/flags.go b/pkg/agent/types/flags.go
@@ -192,4 +192,8 @@ type Flags interface {
 	// InModernStandby indicates whether a Windows machine is awake or in modern standby
 	SetInModernStandby(enabled bool) error
 	InModernStandby() bool
+
+	// OsqueryHealthcheckStartupDelay is the time to wait before beginning osquery healthchecks
+	SetOsqueryHealthcheckStartupDelay(delay time.Duration) error
+	OsqueryHealthcheckStartupDelay() time.Duration
 }
diff --git a/pkg/agent/types/mocks/flags.go b/pkg/agent/types/mocks/flags.go
diff --git a/pkg/agent/types/mocks/knapsack.go b/pkg/agent/types/mocks/knapsack.go
diff --git a/pkg/launcher/options.go b/pkg/launcher/options.go
@@ -25,6 +25,8 @@ type Options struct {
 	RootDirectory string
 	// OsquerydPath is the path to the osqueryd binary.
 	OsquerydPath string
+	// OsqueryHealthcheckStartupDelay is the time to wait before beginning osquery healthchecks
+	OsqueryHealthcheckStartupDelay time.Duration
 	// CertPins are optional hashes of subject public key info to use for
 	// certificate pinning.
 	CertPins [][]byte

diff --git a/pkg/osquery/runtime/runner.go b/pkg/osquery/runtime/runner.go
@@ -438,6 +438,10 @@ func (r *Runner) launchOsqueryInstance() error {
 
 	// Health check on interval
 	o.errgroup.Go(func() error {
+		level.Debug(o.logger).Log("msg", "entering delay before starting osquery healthchecks")
+		time.Sleep(o.knapsack.OsqueryHealthcheckStartupDelay())
+		level.Debug(o.logger).Log("msg", "exiting delay before starting osquery healthchecks")
+
 		ticker := time.NewTicker(healthCheckInterval)
 		defer ticker.Stop()
 		for {