fix: [CI-12353]: Adding control for log upload through settings

api/api.go

@@ -162,10 +162,11 @@ type (
 	}
 
 	LogConfig struct {
-		AccountID      string `json:"account_id,omitempty"`
-		IndirectUpload bool   `json:"indirect_upload,omitempty"` // Whether to directly upload via signed link or using log service
-		URL            string `json:"url,omitempty"`
-		Token          string `json:"token,omitempty"`
+		AccountID             string `json:"account_id,omitempty"`
+		IndirectUpload        bool   `json:"indirect_upload,omitempty"` // Whether to directly upload via signed link or using log service
+		URL                   string `json:"url,omitempty"`
+		Token                 string `json:"token,omitempty"`
+		IndirectUploadSetting bool   `json:"indirect_upload_setting,omitempty"`
 	}
 
 	TIConfig struct {

cli/client/client.go

@@ -147,10 +147,11 @@ func runStage(client Client, remoteLog bool) error {
 	}
 	if remoteLog {
 		setupParams.LogConfig = api.LogConfig{
-			URL:            "http://localhost:8079",
-			AccountID:      "kmpy",
-			Token:          "token",
-			IndirectUpload: true,
+			URL:                   "http://localhost:8079",
+			AccountID:             "kmpy",
+			Token:                 "token",
+			IndirectUpload:        true,
+			IndirectUploadSetting: true,
 		}
 	}
 	logrus.Infof("starting setup")

logstream/remote/http.go

@@ -37,13 +37,14 @@ var defaultClient = &http.Client{
 }
 
 // NewHTTPClient returns a new HTTPClient.
-func NewHTTPClient(endpoint, accountID, token string, indirectUpload, skipverify bool) *HTTPClient {
+func NewHTTPClient(endpoint, accountID, token string, indirectUpload, indirectUploadSetting, skipverify bool) *HTTPClient {
 	client := &HTTPClient{
-		Endpoint:       endpoint,
-		AccountID:      accountID,
-		Token:          token,
-		SkipVerify:     skipverify,
-		IndirectUpload: indirectUpload,
+		Endpoint:              endpoint,
+		AccountID:             accountID,
+		Token:                 token,
+		SkipVerify:            skipverify,
+		IndirectUpload:        indirectUpload,
+		IndirectUploadSetting: indirectUploadSetting,
 	}
 	if skipverify {
 		client.Client = &http.Client{
@@ -63,16 +64,17 @@ func NewHTTPClient(endpoint, accountID, token string, indirectUpload, skipverify
 
 // HTTPClient provides an http service client.
 type HTTPClient struct {
-	Client         *http.Client
-	Endpoint       string // Example: http://localhost:port
-	Token          string // Per account token to validate against
-	AccountID      string
-	SkipVerify     bool
-	IndirectUpload bool
+	Client                *http.Client
+	Endpoint              string // Example: http://localhost:port
+	Token                 string // Per account token to validate against
+	AccountID             string
+	SkipVerify            bool
+	IndirectUpload        bool
+	IndirectUploadSetting bool
 }
 
 // UploadFile uploads the file directly to data store or via log service
-// if indirectUpload is true, logs go through log service instead of using an uploadable link.
+// if indirectUpload or IndirectUploadSetting is true, logs go through log service instead of using an uploadable link.
 func (c *HTTPClient) Upload(ctx context.Context, key string, lines []*logstream.Line) error {
 	data := new(bytes.Buffer)
 	for _, line := range convertLines(lines) {
@@ -84,9 +86,11 @@ func (c *HTTPClient) Upload(ctx context.Context, key string, lines []*logstream.
 		}
 		data.Write(buf.Bytes())
 	}
-	if c.IndirectUpload {
+	logrus.Infoln(fmt.Sprintf("Value of ff indirectUpload is %t", c.IndirectUpload))
+	logrus.Infoln(fmt.Sprintf("Value of setting indirectUploadSetting is %t", c.IndirectUploadSetting))
+	if c.IndirectUpload || c.IndirectUploadSetting {
 		logrus.WithField("key", key).
-			Infoln("uploading logs through log service as indirectUpload is specified as true")
+			Infoln("uploading logs through log service as indirectUpload or IndirectUploadSetting is specified as true")
 		err := c.uploadToRemoteStorage(ctx, key, data)
 		if err != nil {
 			logrus.WithError(err).WithField("key", key).

pipeline/runtime/common.go

@@ -180,7 +180,7 @@ func setTiEnvVariables(step *spec.Step, config *tiCfg.Cfg) {
 
 func getLogServiceClient(cfg api.LogConfig) logstream.Client {
 	if cfg.URL != "" {
-		return remote.NewHTTPClient(cfg.URL, cfg.AccountID, cfg.Token, cfg.IndirectUpload, false)
+		return remote.NewHTTPClient(cfg.URL, cfg.AccountID, cfg.Token, cfg.IndirectUpload, cfg.IndirectUploadSetting, false)
 	}
 	return stdout.New()
 }

pipeline/state.go

@@ -66,7 +66,7 @@ func (s *State) GetLogStreamClient() logstream.Client {
 	if s.logClient == nil {
 		if s.logConfig.URL != "" {
 			s.logClient = remote.NewHTTPClient(s.logConfig.URL, s.logConfig.AccountID,
-				s.logConfig.Token, s.logConfig.IndirectUpload, false)
+				s.logConfig.Token, s.logConfig.IndirectUpload, s.logConfig.IndirectUploadSetting, false)
 		} else {
 			s.logClient = filestore.New(SharedVolPath)
 		}