[CI-13836]: Override registry for internal harness images

engine/docker/docker.go

@@ -256,9 +256,9 @@ func (e *Docker) Destroy(ctx context.Context, pipelineConfig *spec.PipelineConfi
 
 // Run runs the pipeline step.
 func (e *Docker) Run(ctx context.Context, pipelineConfig *spec.PipelineConfig, step *spec.Step,
-	output io.Writer, isDrone bool) (*runtime.State, error) {
+	output io.Writer, isDrone bool, isHosted bool) (*runtime.State, error) {
 	// create the container
-	err := e.create(ctx, pipelineConfig, step, output)
+	err := e.create(ctx, pipelineConfig, step, output, isHosted)
 	if err != nil {
 		return nil, errors.TrimExtraInfo(err)
 	}
@@ -314,7 +314,7 @@ func (e *Docker) startContainer(ctx context.Context, stepID string, tty bool, ou
 // emulate docker commands
 //
 
-func (e *Docker) create(ctx context.Context, pipelineConfig *spec.PipelineConfig, step *spec.Step, output io.Writer) error {
+func (e *Docker) create(ctx context.Context, pipelineConfig *spec.PipelineConfig, step *spec.Step, output io.Writer, isHosted bool) error {
 	// create pull options with encoded authorization credentials.
 	pullopts := types.ImagePullOptions{}
 	if step.Auth != nil {
@@ -324,13 +324,28 @@ func (e *Docker) create(ctx context.Context, pipelineConfig *spec.PipelineConfig
 		)
 	}
 
+	finalImage := step.Image
+
+	// override image registry for internal images
+	// this is short term solution
+	// override if no auth is present
+	if isHosted && (step.Auth == nil || step.Auth.Username == "" || step.Auth.Password == "") {
+		finalImage = image.OverrideRegistry(step.Image)
+	}
+
 	// automatically pull the latest version of the image if requested
 	// by the process configuration, or if the image is :latest
 	if step.Pull == spec.PullAlways ||
-		(step.Pull == spec.PullDefault && image.IsLatest(step.Image)) {
-		pullerr := e.pullImageWithRetries(ctx, step.Image, pullopts, output)
+		(step.Pull == spec.PullDefault && image.IsLatest(finalImage)) {
+		pullerr := e.pullImageWithRetries(ctx, finalImage, pullopts, output)
 		if pullerr != nil {
-			return pullerr
+			// if for some reason overriden image does not work then fallback
+			if finalImage != step.Image {
+				pullerr = e.pullImageWithRetries(ctx, step.Image, pullopts, output)
+			}
+			if pullerr != nil {
+				return pullerr
+			}
 		}
 	}
 
@@ -347,9 +362,15 @@ func (e *Docker) create(ctx context.Context, pipelineConfig *spec.PipelineConfig
 	// automatically pull and try to re-create the image if the
 	// failure is caused because the image does not exist.
 	if client.IsErrNotFound(err) && step.Pull != spec.PullNever {
-		pullerr := e.pullImageWithRetries(ctx, step.Image, pullopts, output)
+		pullerr := e.pullImageWithRetries(ctx, finalImage, pullopts, output)
 		if pullerr != nil {
-			return pullerr
+			// if for some reason overriden image does not work then fallback
+			if finalImage != step.Image {
+				pullerr = e.pullImageWithRetries(ctx, step.Image, pullopts, output)
+			}
+			if pullerr != nil {
+				return pullerr
+			}
 		}
 
 		// once the image is successfully pulled we attempt to

engine/docker/image/image.go

@@ -15,6 +15,13 @@ import (
 	"github.com/docker/distribution/reference"
 )
 
+var (
+	internalImages = []string{"harness/drone-git", "plugins/docker", "plugins/acr", "plugins/ecr", "plugins/gcr",
+		"plugins/gar", "plugins/gcs", "plugins/s3", "harness/sto-plugin", "plugins/artifactory", "plugins/cache",
+		"harness/ssca-plugin", "harness/slsa-plugin", "harness/ssca-compliance-plugin"}
+	garRegistry = "us-docker.pkg.dev/gar-prod-setup/harness-public/"
+)
+
 // Trim returns the short image name without tag.
 func Trim(name string) string {
 	ref, err := reference.ParseAnyReference(name)
@@ -94,3 +101,27 @@ func MatchHostname(image, hostname string) bool {
 func IsLatest(s string) bool {
 	return strings.HasSuffix(Expand(s), ":latest")
 }
+
+// Overrides registry if image is an internal image
+func OverrideRegistry(imageWithTag string) string {
+	parts := strings.Split(imageWithTag, ":")
+	if len(parts) < 1 || len(parts) > 2 {
+		return imageWithTag
+	}
+
+	imageName := parts[0]
+	tagName := ""
+	if len(parts) == 2 {
+		tagName = parts[1]
+	}
+
+	for _, im := range internalImages {
+		if imageName == im {
+			if tagName == "" {
+				return garRegistry + imageName
+			}
+			return garRegistry + imageName + ":" + tagName
+		}
+	}
+	return imageWithTag
+}

engine/engine.go

@@ -97,7 +97,7 @@ func (e *Engine) Destroy(ctx context.Context) error {
 	return e.docker.Destroy(ctx, cfg)
 }
 
-func (e *Engine) Run(ctx context.Context, step *spec.Step, output io.Writer, isDrone bool) (*runtime.State, error) {
+func (e *Engine) Run(ctx context.Context, step *spec.Step, output io.Writer, isDrone bool, isHosted bool) (*runtime.State, error) {
 	e.mu.Lock()
 	cfg := e.pipelineConfig
 	e.mu.Unlock()
@@ -111,7 +111,7 @@ func (e *Engine) Run(ctx context.Context, step *spec.Step, output io.Writer, isD
 	}
 
 	if step.Image != "" {
-		return e.docker.Run(ctx, cfg, step, output, isDrone)
+		return e.docker.Run(ctx, cfg, step, output, isDrone, isHosted)
 	}
 
 	return exec.Run(ctx, step, output)

engine/helper.go

@@ -66,6 +66,7 @@ func RunStep(
 	output io.Writer,
 	cfg *spec.PipelineConfig,
 	isDrone bool,
+	isHosted bool,
 ) (*runtime.State, error) {
 	d, err := docker.NewEnv(opts.Opts)
 	if err != nil {
@@ -80,7 +81,7 @@ func RunStep(
 		printCommand(step, output)
 	}
 	if step.Image != "" {
-		return d.Run(ctx, cfg, step, output, isDrone)
+		return d.Run(ctx, cfg, step, output, isDrone, isHosted)
 	}
 
 	return exec.Run(ctx, step, output)

pipeline/runtime/run.go

@@ -91,7 +91,10 @@ func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out
 	log := logrus.New()
 	log.Out = out
 
-	exited, err := f(ctx, step, out, r.LogDrone)
+	// stageRuntimeID is only passed for dlite
+	isHosted := r.StageRuntimeID != ""
+
+	exited, err := f(ctx, step, out, r.LogDrone, isHosted)
 	timeTakenMs := time.Since(start).Milliseconds()
 
 	reportStart := time.Now()

pipeline/runtime/runtest.go

@@ -72,7 +72,7 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest,
 	artifactFile := fmt.Sprintf("%s/%s-artifact", pipeline.SharedVolPath, step.ID)
 	step.Envs["PLUGIN_ARTIFACT_FILE"] = artifactFile
 
-	exited, err := f(ctx, step, out, false)
+	exited, err := f(ctx, step, out, false, false)
 	timeTakenMs := time.Since(start).Milliseconds()
 	collectionErr := collectRunTestData(ctx, log, r, start, step.Name, tiConfig)
 	if err == nil {

pipeline/runtime/runtestsV2.go

@@ -91,7 +91,7 @@ func executeRunTestsV2Step(ctx context.Context, f RunFunc, r *api.StartStepReque
 		step.Envs["PLUGIN_METADATA_FILE"] = fmt.Sprintf("%s/%s-%s", pipeline.SharedVolPath, step.ID, metadataFile)
 	}
 
-	exited, err := f(ctx, step, out, r.LogDrone)
+	exited, err := f(ctx, step, out, r.LogDrone, false)
 	timeTakenMs := time.Since(start).Milliseconds()
 	collectionErr := collectTestReportsAndCg(ctx, log, r, start, step.Name, tiConfig)
 	if err == nil {

pipeline/runtime/step_executor.go

@@ -31,7 +31,7 @@ import (
 
 type ExecutionStatus int
 
-type RunFunc func(ctx context.Context, step *spec.Step, output io.Writer, isDrone bool) (*runtime.State, error)
+type RunFunc func(ctx context.Context, step *spec.Step, output io.Writer, isDrone bool, isHosted bool) (*runtime.State, error)
 
 type StepStatus struct {
 	Status            ExecutionStatus

pipeline/runtime/step_executor_stateless.go

@@ -58,8 +58,8 @@ func (e *StepExecutorStateless) executeStep( //nolint:gocritic
 	writer logstream.Writer,
 ) (*runtime.State, map[string]string,
 	map[string]string, []byte, []*api.OutputV2, string, error) {
-	runFunc := func(ctx context.Context, step *spec.Step, output io.Writer, isDrone bool) (*runtime.State, error) {
-		return engine.RunStep(ctx, engine.Opts{}, step, output, cfg, isDrone)
+	runFunc := func(ctx context.Context, step *spec.Step, output io.Writer, isDrone bool, isHosted bool) (*runtime.State, error) {
+		return engine.RunStep(ctx, engine.Opts{}, step, output, cfg, isDrone, isHosted)
 	}
 	// Temporary: this should be removed once we have a better way of handling test intelligence.
 	tiConfig := getTiCfg(&r.TIConfig)