From 6cdfeeed1eff3ee24b6e1b4cd821925a23d7646f Mon Sep 17 00:00:00 2001 From: Patrick Zhao Date: Sun, 29 Sep 2024 10:03:07 +0800 Subject: [PATCH] support reference repos in workflow Signed-off-by: Patrick Zhao --- .../common/repository/models/workflow_v4.go | 31 +- .../aslan/core/common/service/build.go | 83 +++++ .../core/workflow/service/webhook/gitlab.go | 45 --- .../service/webhook/gitlab_pipeline_task.go | 249 -------------- .../service/webhook/gitlab_workflowv4_task.go | 1 + .../workflow/service/webhook/utils_test.go | 110 ------- .../core/workflow/service/workflow/job/job.go | 32 +- .../service/workflow/job/job_build.go | 311 +++++++++--------- .../service/workflow/job/job_freestyle.go | 117 ++++++- .../service/workflow/job/job_scanning.go | 3 + .../service/workflow/job/job_testing.go | 3 + .../service/workflow/job/job_vm_deploy.go | 24 +- 12 files changed, 399 insertions(+), 610 deletions(-) delete mode 100644 pkg/microservice/aslan/core/workflow/service/webhook/gitlab_pipeline_task.go delete mode 100644 pkg/microservice/aslan/core/workflow/service/webhook/utils_test.go diff --git a/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go b/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go index d5e0125592..99e34a51b3 100644 --- a/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go +++ b/pkg/microservice/aslan/core/common/repository/models/workflow_v4.go @@ -286,7 +286,8 @@ type FreestyleJobSpec struct { // fromjob/runtime, runtime 表示运行时输入，fromjob 表示从上游构建任务中获取 Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"` // 当 source 为 fromjob 时需要，指定部署镜像来源是上游哪一个构建任务 - JobName string `bson:"job_name" yaml:"job_name" json:"job_name"` + JobName string `bson:"job_name" yaml:"job_name" json:"job_name"` + RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"` // save the origin quoted job name OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"` Properties *JobProperties `bson:"properties" yaml:"properties" json:"properties"` @@ -297,15 +298,26 @@ type FreestyleJobSpec struct { type FreeStyleServiceInfo struct { ServiceName string `bson:"service_name" yaml:"service_name" json:"service_name"` - ServiceModule string `bson:"service_module" yaml:"service_module" json:"service_module"` + ServiceModule string `bson:"service_module" yaml:"service_module" json:"service_module"` Repos []*types.Repository `bson:"repos" yaml:"repos" json:"repos"` KeyVals []*KeyVal `bson:"key_vals" yaml:"key_vals" json:"key_vals"` } +func (i *FreeStyleServiceInfo) GetKey() string { + if i == nil { + return "" + } + return i.ServiceName + "-" + i.ServiceModule +} + type ZadigBuildJobSpec struct { - DockerRegistryID string `bson:"docker_registry_id" yaml:"docker_registry_id" json:"docker_registry_id"` - ServiceAndBuilds []*ServiceAndBuild `bson:"service_and_builds" yaml:"service_and_builds" json:"service_and_builds"` - ServiceAndBuildsOptions []*ServiceAndBuild `bson:"-" yaml:"service_and_builds_options" json:"service_and_builds_options"` + Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"` + JobName string `bson:"job_name" yaml:"job_name" json:"job_name"` + OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"` + RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"` + DockerRegistryID string `bson:"docker_registry_id" yaml:"docker_registry_id" json:"docker_registry_id"` + ServiceAndBuilds []*ServiceAndBuild `bson:"service_and_builds" yaml:"service_and_builds" json:"service_and_builds"` + ServiceAndBuildsOptions []*ServiceAndBuild `bson:"-" yaml:"service_and_builds_options" json:"service_and_builds_options"` } type ServiceAndBuild struct { @@ -320,6 +332,13 @@ type ServiceAndBuild struct { ShareStorageInfo *ShareStorageInfo `bson:"share_storage_info" yaml:"share_storage_info" json:"share_storage_info"` } +func (i *ServiceAndBuild) GetKey() string { + if i == nil { + return "" + } + return i.ServiceName + "-" + i.ServiceModule +} + type ZadigDeployJobSpec struct { Env string `bson:"env" yaml:"env" json:"env"` EnvOptions []*ZadigDeployEnvInformation `bson:"-" yaml:"env_options" json:"env_options"` @@ -481,6 +500,7 @@ type ZadigTestingJobSpec struct { Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"` JobName string `bson:"job_name" yaml:"job_name" json:"job_name"` OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"` + RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"` // selected service in service testing TargetServices []*ServiceTestTarget `bson:"target_services" yaml:"target_services" json:"target_services"` // field for non-service tests. @@ -513,6 +533,7 @@ type ZadigScanningJobSpec struct { Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"` JobName string `bson:"job_name" yaml:"job_name" json:"job_name"` OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"` + RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"` // Scannings used only for normal scanning. for service scanning we use Scannings []*ScanningModule `bson:"scannings" yaml:"scannings" json:"scannings"` // ServiceAndScannings is the configured field for this job. It includes all the services along with its configured scanning. diff --git a/pkg/microservice/aslan/core/common/service/build.go b/pkg/microservice/aslan/core/common/service/build.go index 2a7e9a4d4e..b5c365d79a 100644 --- a/pkg/microservice/aslan/core/common/service/build.go +++ b/pkg/microservice/aslan/core/common/service/build.go @@ -17,7 +17,9 @@ limitations under the License. package service import ( + "fmt" "strings" + "sync" "time" "go.uber.org/zap" @@ -219,3 +221,84 @@ func MergeParams(templateEnvs []*commonmodels.Param, customEnvs []*commonmodels. } return retEnvs } + +type BuildService struct { + BuildMap sync.Map + BuildTemplateMap sync.Map +} + +func NewBuildService() *BuildService { + return &BuildService{ + BuildMap: sync.Map{}, + BuildTemplateMap: sync.Map{}, + } +} + +func (c *BuildService) GetBuild(buildName, serviceName, serviceModule string) (*commonmodels.Build, error) { + buildInfo := &commonmodels.Build{} + buildMapValue, ok := c.BuildMap.Load(buildName) + if !ok { + buildInfo, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: buildName}) + if err != nil { + return nil, fmt.Errorf("find build: %s error: %v", buildName, err) + } + c.BuildMap.Store(buildName, buildInfo) + } else { + buildInfo = buildMapValue.(*commonmodels.Build) + } + + if err := FillBuildDetail(buildInfo, serviceName, serviceModule, &c.BuildTemplateMap); err != nil { + return nil, err + } + return buildInfo, nil +} + +func FillBuildDetail(moduleBuild *commonmodels.Build, serviceName, serviceModule string, buildTemplateMap *sync.Map) error { + if moduleBuild.TemplateID == "" { + return nil + } + + var err error + var buildTemplate *commonmodels.BuildTemplate + buildTemplateMapValue, ok := buildTemplateMap.Load(moduleBuild.TemplateID) + if !ok { + buildTemplate, err = commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ + ID: moduleBuild.TemplateID, + }) + if err != nil { + return fmt.Errorf("failed to find build template with id: %s, err: %s", moduleBuild.TemplateID, err) + } + buildTemplateMap.Store(moduleBuild.TemplateID, buildTemplate) + } else { + buildTemplate = buildTemplateMapValue.(*commonmodels.BuildTemplate) + } + + moduleBuild.Timeout = buildTemplate.Timeout + moduleBuild.PreBuild = buildTemplate.PreBuild + moduleBuild.JenkinsBuild = buildTemplate.JenkinsBuild + moduleBuild.ScriptType = buildTemplate.ScriptType + moduleBuild.Scripts = buildTemplate.Scripts + moduleBuild.PostBuild = buildTemplate.PostBuild + moduleBuild.SSHs = buildTemplate.SSHs + moduleBuild.PMDeployScripts = buildTemplate.PMDeployScripts + moduleBuild.CacheEnable = buildTemplate.CacheEnable + moduleBuild.CacheDirType = buildTemplate.CacheDirType + moduleBuild.CacheUserDir = buildTemplate.CacheUserDir + moduleBuild.AdvancedSettingsModified = buildTemplate.AdvancedSettingsModified + moduleBuild.Outputs = buildTemplate.Outputs + moduleBuild.Infrastructure = buildTemplate.Infrastructure + moduleBuild.VMLabels = buildTemplate.VmLabels + + // repos are configured by service modules + for _, serviceConfig := range moduleBuild.Targets { + if serviceConfig.ServiceName == serviceName && serviceConfig.ServiceModule == serviceModule { + moduleBuild.Repos = serviceConfig.Repos + if moduleBuild.PreBuild == nil { + moduleBuild.PreBuild = &commonmodels.PreBuild{} + } + moduleBuild.PreBuild.Envs = MergeBuildEnvs(moduleBuild.PreBuild.Envs, serviceConfig.Envs) + break + } + } + return nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go index 8fb975f68b..96b45dcc4c 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go @@ -115,24 +115,6 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log commonrepo.NewWebHookUserColl().Upsert(webhookUser) } - //产品工作流webhook - //wg.Add(1) - //go func() { - // defer wg.Done() - // if err = TriggerWorkflowByGitlabEvent(pushEvent, baseURI, requestID, log); err != nil { - // errorList = multierror.Append(errorList, err) - // } - //}() - - //单服务工作流webhook - wg.Add(1) - go func() { - defer wg.Done() - if err = TriggerPipelineByGitlabEvent(pushEvent, baseURI, requestID, log); err != nil { - errorList = multierror.Append(errorList, err) - } - }() - //测试管理webhook wg.Add(1) go func() { @@ -160,24 +142,6 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log } if mergeEvent != nil { - //多服务工作流webhook - //wg.Add(1) - //go func() { - // defer wg.Done() - // if err = TriggerWorkflowByGitlabEvent(mergeEvent, baseURI, requestID, log); err != nil { - // errorList = multierror.Append(errorList, err) - // } - //}() - - //单服务工作流webhook - wg.Add(1) - go func() { - defer wg.Done() - if err = TriggerPipelineByGitlabEvent(mergeEvent, baseURI, requestID, log); err != nil { - errorList = multierror.Append(errorList, err) - } - }() - //测试管理webhook wg.Add(1) go func() { @@ -205,15 +169,6 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log } if tagEvent != nil { - //// workflow webhook - //wg.Add(1) - //go func() { - // defer wg.Done() - // if err = TriggerWorkflowByGitlabEvent(tagEvent, baseURI, requestID, log); err != nil { - // errorList = multierror.Append(errorList, err) - // } - //}() - //test webhook wg.Add(1) go func() { diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_pipeline_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_pipeline_task.go deleted file mode 100644 index 7f178c336f..0000000000 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_pipeline_task.go +++ /dev/null @@ -1,249 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "github.com/hashicorp/go-multierror" - "github.com/xanzy/go-gitlab" - "go.uber.org/zap" - - "github.com/koderover/zadig/v2/pkg/microservice/aslan/config" - commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" - commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/scmnotify" - workflowservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/workflow/service/workflow" - "github.com/koderover/zadig/v2/pkg/setting" - e "github.com/koderover/zadig/v2/pkg/tool/errors" - "github.com/koderover/zadig/v2/pkg/types" -) - -type pipelineGitEventMatcher interface { - Match(*commonmodels.MainHookRepo, string) (*commonmodels.TaskArgs, error) -} - -type pipelineGitlabPushEventMatcher struct { - log *zap.SugaredLogger - pipeline *commonmodels.Pipeline - event *gitlab.PushEvent -} - -func (pgpem *pipelineGitlabPushEventMatcher) Match(hookRepo *commonmodels.MainHookRepo, requestID string) (*commonmodels.TaskArgs, error) { - ev := pgpem.event - var ( - changedFiles = make([]string, 0) - ) - if checkRepoNamespaceMatch(hookRepo, ev.Project.PathWithNamespace) { - if hookRepo.Branch == getBranchFromRef(ev.Ref) && EventConfigured(hookRepo, config.HookEventPush) { - for _, commit := range ev.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Removed...) - changedFiles = append(changedFiles, commit.Modified...) - } - } - } - if MatchChanges(hookRepo, changedFiles) { - var ( - branch = getBranchFromRef(ev.Ref) - ref = ev.Ref - commitID = ev.Commits[0].ID - commitMessage = ev.Commits[0].Message - ) - - eventRepo := &types.Repository{ - RepoOwner: hookRepo.RepoOwner, - RepoNamespace: hookRepo.GetRepoNamespace(), - RepoName: hookRepo.RepoName, - Branch: branch, - CommitID: commitID, - CommitMessage: commitMessage, - IsPrimary: true, - } - - ptargs := &commonmodels.TaskArgs{ - PipelineName: pgpem.pipeline.Name, - TaskCreator: setting.WebhookTaskCreator, - ReqID: requestID, - HookPayload: &commonmodels.HookPayload{ - Owner: hookRepo.RepoOwner, - Repo: hookRepo.RepoName, - Branch: branch, - Ref: ref, - IsPr: false, - }, - Builds: []*types.Repository{eventRepo}, - Test: commonmodels.TestArgs{ - Builds: []*types.Repository{eventRepo}, - }, - } - return ptargs, nil - } - - return nil, nil -} - -type pipelineGitlabMergeEventMatcher struct { - diffFunc gitlabMergeRequestDiffFunc - log *zap.SugaredLogger - pipeline *commonmodels.Pipeline - event *gitlab.MergeEvent -} - -func (pgmem *pipelineGitlabMergeEventMatcher) Match(hookRepo *commonmodels.MainHookRepo, requestID string) (*commonmodels.TaskArgs, error) { - ev := pgmem.event - var ( - err error - changedFiles = make([]string, 0) - ) - if checkRepoNamespaceMatch(hookRepo, ev.ObjectAttributes.Target.PathWithNamespace) { - if EventConfigured(hookRepo, config.HookEventPr) && (hookRepo.Branch == ev.ObjectAttributes.TargetBranch) { - if ev.ObjectAttributes.State == "opened" { - changedFiles, err = pgmem.diffFunc(ev, hookRepo.CodehostID) - if err != nil { - pgmem.log.Warnf("failed to get changes of event %v , err:%v", ev, err) - return nil, err - } - pgmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) - } - } - } - if MatchChanges(hookRepo, changedFiles) { - var ( - branch = ev.ObjectAttributes.TargetBranch - commitID = ev.ObjectAttributes.LastCommit.ID - commitMessage = ev.ObjectAttributes.LastCommit.Message - ) - - eventRepo := &types.Repository{ - RepoOwner: hookRepo.RepoOwner, - RepoNamespace: hookRepo.GetRepoNamespace(), - RepoName: hookRepo.RepoName, - PR: ev.ObjectAttributes.IID, - Branch: branch, - CommitID: commitID, - CommitMessage: commitMessage, - IsPrimary: true, - } - - ptargs := &commonmodels.TaskArgs{ - PipelineName: pgmem.pipeline.Name, - TaskCreator: setting.WebhookTaskCreator, - ReqID: requestID, - HookPayload: &commonmodels.HookPayload{ - Owner: hookRepo.RepoOwner, - Repo: hookRepo.RepoName, - Branch: branch, - Ref: commitID, - IsPr: true, - }, - Builds: []*types.Repository{eventRepo}, - Test: commonmodels.TestArgs{ - Builds: []*types.Repository{eventRepo}, - }, - } - return ptargs, nil - } - return nil, nil -} - -func pipelineCreateGitlabEventMatcher( - event interface{}, diffSrv gitlabMergeRequestDiffFunc, pipeline *commonmodels.Pipeline, log *zap.SugaredLogger, -) pipelineGitEventMatcher { - switch evt := event.(type) { - case *gitlab.PushEvent: - return &pipelineGitlabPushEventMatcher{ - pipeline: pipeline, - log: log, - event: evt, - } - case *gitlab.MergeEvent: - return &pipelineGitlabMergeEventMatcher{ - diffFunc: diffSrv, - log: log, - event: evt, - pipeline: pipeline, - } - } - - return nil -} - -func TriggerPipelineByGitlabEvent(event interface{}, baseURI, requestID string, log *zap.SugaredLogger) error { - pipelineList, err := commonrepo.NewPipelineColl().List(&commonrepo.PipelineListOption{}) - if err != nil { - log.Errorf("list PipelineV2 error: %v", err) - return e.ErrListPipeline - } - - mErr := &multierror.Error{} - diffSrv := func(mergeEvent *gitlab.MergeEvent, codehostId int) ([]string, error) { - return findChangedFilesOfMergeRequest(mergeEvent, codehostId) - } - - var notification *commonmodels.Notification - - for _, pipelineObject := range pipelineList { - if pipelineObject.Hook != nil && pipelineObject.Hook.Enabled { - log.Debugf("find %d hooks in pipeline %s", len(pipelineObject.Hook.GitHooks), pipelineObject.Name) - for _, item := range pipelineObject.Hook.GitHooks { - matcher := pipelineCreateGitlabEventMatcher(event, diffSrv, pipelineObject, log) - if matcher == nil { - continue - } - hookRepo := new(commonmodels.MainHookRepo) - hookRepo.RepoOwner = item.Owner - hookRepo.RepoName = item.Repo - hookRepo.Branch = item.Branch - hookRepo.MatchFolders = item.MatchFolders - HookEvents := make([]config.HookEventType, 0) - for _, event := range item.Events { - if config.HookEventPush == config.HookEventType(event) { - HookEvents = append(HookEvents, config.HookEventPush) - } else if config.HookEventPr == config.HookEventType(event) { - HookEvents = append(HookEvents, config.HookEventPr) - } - } - hookRepo.Events = HookEvents - hookRepo.CodehostID = item.CodehostID - taskargs, err := matcher.Match(hookRepo, requestID) - if taskargs == nil || err != nil { - log.Infof("[Webhook] %s Match none , [task] : %s , [error] : %v", pipelineObject.Name, pipelineObject.Name, err) - continue - } - log.Infof("TriggerPipelineByGitlabEvent event match hook, pipelineObject.Name:%s\n", pipelineObject.Name) - if taskargs.HookPayload.IsPr { - hookRepo.RepoOwner = taskargs.HookPayload.Owner - if notification == nil { - notification, _ = scmnotify.NewService().SendInitWebhookComment( - hookRepo, taskargs.Builds[0].PR, baseURI, true, false, false, false, log, - ) - } - } - if notification != nil { - taskargs.NotificationID = notification.ID.Hex() - } - resp, err1 := workflowservice.CreatePipelineTask(taskargs, log) - if err1 != nil { - log.Errorf("[Webhook] CreatePipelineTask task %s error: %v", pipelineObject.Name, err) - continue - } - log.Infof("[Webhook] triggered task %s:%d", pipelineObject.Name, resp.TaskID) - } - } - } - - return mErr.ErrorOrNil() -} diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflowv4_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflowv4_task.go index 58b6e8f73d..34bb440331 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflowv4_task.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/gitlab_workflowv4_task.go @@ -335,6 +335,7 @@ func TriggerWorkflowV4ByGitlabEvent(event interface{}, baseURI, requestID string } log.Infof("event match hook %v of %s", item.MainRepo, workflow.Name) eventRepo := matcher.GetHookRepo(item.MainRepo) + log.Debugf("eventRepo: %s/%s/%s/%s", eventRepo.Source, eventRepo.GetRepoNamespace(), eventRepo.RepoName, eventRepo.Branch) autoCancelOpt := &AutoCancelOpt{ TaskType: config.WorkflowType, diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/utils_test.go b/pkg/microservice/aslan/core/workflow/service/webhook/utils_test.go deleted file mode 100644 index 2adcd3d4cf..0000000000 --- a/pkg/microservice/aslan/core/workflow/service/webhook/utils_test.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2021 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/koderover/zadig/v2/pkg/tool/kube/serializer" -) - -var testDeployment = ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: test - namespace: test -spec: - template: - spec: - containers: - - image: test-image - name: test -` - -var testStatefulSet = ` -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test - namespace: test -spec: - template: - spec: - containers: - - image: test-image - name: test -` - -var testJob = ` -apiVersion: batch/v1 -kind: Job -metadata: - name: test - namespace: test -spec: - template: - spec: - containers: - - image: test-image - name: test -` - -var _ = Describe("Testing utils", func() { - - Context("test getContainers", func() { - It("should work for Deployment", func() { - u, err := serializer.NewDecoder().YamlToUnstructured([]byte(testDeployment)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(u.GetKind()).To(Equal("Deployment")) - Expect(u.GetName()).To(Equal("test")) - - cs, err := getContainers(u) - Expect(err).ShouldNot(HaveOccurred()) - Expect(cs).To(HaveLen(1)) - Expect(cs[0].Name).To(Equal("test")) - Expect(cs[0].Image).To(Equal("test-image")) - }) - - It("should work for StatefulSet", func() { - u, err := serializer.NewDecoder().YamlToUnstructured([]byte(testStatefulSet)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(u.GetKind()).To(Equal("StatefulSet")) - Expect(u.GetName()).To(Equal("test")) - - cs, err := getContainers(u) - Expect(err).ShouldNot(HaveOccurred()) - Expect(cs).To(HaveLen(1)) - Expect(cs[0].Name).To(Equal("test")) - Expect(cs[0].Image).To(Equal("test-image")) - }) - - It("should work for Job", func() { - u, err := serializer.NewDecoder().YamlToUnstructured([]byte(testJob)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(u.GetKind()).To(Equal("Job")) - Expect(u.GetName()).To(Equal("test")) - - cs, err := getContainers(u) - Expect(err).ShouldNot(HaveOccurred()) - Expect(cs).To(HaveLen(1)) - Expect(cs[0].Name).To(Equal("test")) - Expect(cs[0].Image).To(Equal("test-image")) - }) - }) -}) diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job.go index 1d936ef674..5e37176c0f 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/job/job.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job.go @@ -23,7 +23,6 @@ import ( "path" "regexp" "strings" - "sync" "time" "github.com/pkg/errors" @@ -31,7 +30,7 @@ import ( "github.com/koderover/zadig/v2/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" - commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/v2/pkg/setting" "github.com/koderover/zadig/v2/pkg/types" "github.com/koderover/zadig/v2/pkg/types/job" @@ -307,11 +306,8 @@ type RepoIndex struct { } func GetWorkflowRepoIndex(workflow *commonmodels.WorkflowV4, currentJobName string, log *zap.SugaredLogger) []*RepoIndex { - var ( - buildMap sync.Map - buildTemplateMap sync.Map - ) resp := []*RepoIndex{} + buildService := commonservice.NewBuildService() jobRankMap := getJobRankMap(workflow.Stages) for _, stage := range workflow.Stages { for _, job := range stage.Jobs { @@ -326,27 +322,9 @@ func GetWorkflowRepoIndex(workflow *commonmodels.WorkflowV4, currentJobName stri continue } for _, build := range jobSpec.ServiceAndBuilds { - var err error - var buildInfo *commonmodels.Build - buildMapValue, ok := buildMap.Load(build.BuildName) - if !ok { - buildInfo, err = commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.BuildName}) - if err != nil { - log.Errorf("find build: %s error: %v", build.BuildName, err) - buildMap.Store(build.BuildName, nil) - continue - } - buildMap.Store(build.BuildName, buildInfo) - } else { - if buildMapValue == nil { - log.Errorf("find build: %s error: %v", build.BuildName, err) - continue - } - buildInfo = buildMapValue.(*commonmodels.Build) - } - - if err := fillBuildDetail(buildInfo, build.ServiceName, build.ServiceModule, &buildTemplateMap); err != nil { - log.Errorf("fill build: %s detail error: %v", build.BuildName, err) + buildInfo, err := buildService.GetBuild(build.BuildName, build.ServiceName, build.ServiceModule) + if err != nil { + log.Errorf("get build info failed, err: %v", err) continue } for _, target := range buildInfo.Targets { diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_build.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_build.go index fab91f3ada..32c4d16f87 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_build.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_build.go @@ -22,7 +22,6 @@ import ( "os" "path" "strings" - "sync" "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/sets" @@ -76,30 +75,13 @@ func (j *BuildJob) SetPreset() error { if err != nil { return fmt.Errorf("get services map error: %v", err) } - var buildMap sync.Map - var buildTemplateMap sync.Map + + buildSvc := commonservice.NewBuildService() newBuilds := make([]*commonmodels.ServiceAndBuild, 0) for _, build := range j.spec.ServiceAndBuilds { - var buildInfo *commonmodels.Build - buildMapValue, ok := buildMap.Load(build.BuildName) - if !ok { - buildInfo, err = commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.BuildName, ProductName: j.workflow.Project}) - if err != nil { - log.Errorf("find build: %s error: %v", build.BuildName, err) - buildMap.Store(build.BuildName, nil) - continue - } - buildMap.Store(build.BuildName, buildInfo) - } else { - if buildMapValue == nil { - log.Errorf("find build: %s error: %v", build.BuildName, err) - continue - } - buildInfo = buildMapValue.(*commonmodels.Build) - } - - if err := fillBuildDetail(buildInfo, build.ServiceName, build.ServiceModule, &buildTemplateMap); err != nil { - log.Errorf("fill build: %s detail error: %v", build.BuildName, err) + buildInfo, err := buildSvc.GetBuild(build.BuildName, build.ServiceName, build.ServiceModule) + if err != nil { + log.Errorf("find build: %s error: %v", build.BuildName, err) continue } for _, target := range buildInfo.Targets { @@ -185,30 +167,12 @@ func (j *BuildJob) SetOptions() error { return fmt.Errorf("get services map error: %v", err) } - var buildMap sync.Map - var buildTemplateMap sync.Map + buildSvc := commonservice.NewBuildService() newBuilds := make([]*commonmodels.ServiceAndBuild, 0) for _, build := range originalSpec.ServiceAndBuilds { - var buildInfo *commonmodels.Build - buildMapValue, ok := buildMap.Load(build.BuildName) - if !ok { - buildInfo, err = commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.BuildName, ProductName: j.workflow.Project}) - if err != nil { - log.Errorf("find build: %s error: %v", build.BuildName, err) - buildMap.Store(build.BuildName, nil) - continue - } - buildMap.Store(build.BuildName, buildInfo) - } else { - if buildMapValue == nil { - log.Errorf("find build: %s error: %v", build.BuildName, err) - continue - } - buildInfo = buildMapValue.(*commonmodels.Build) - } - - if err := fillBuildDetail(buildInfo, build.ServiceName, build.ServiceModule, &buildTemplateMap); err != nil { - log.Errorf("fill build: %s detail error: %v", build.BuildName, err) + buildInfo, err := buildSvc.GetBuild(build.BuildName, build.ServiceName, build.ServiceModule) + if err != nil { + log.Errorf("find build: %s error: %v", build.BuildName, err) continue } for _, target := range buildInfo.Targets { @@ -259,32 +223,11 @@ func (j *BuildJob) GetRepos() ([]*types.Repository, error) { return resp, err } - var ( - err error - buildMap sync.Map - buildTemplateMap sync.Map - ) + buildSvc := commonservice.NewBuildService() for _, build := range j.spec.ServiceAndBuilds { - var buildInfo *commonmodels.Build - buildMapValue, ok := buildMap.Load(build.BuildName) - if !ok { - buildInfo, err = commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.BuildName, ProductName: j.workflow.Project}) - if err != nil { - log.Errorf("find build: %s error: %v", build.BuildName, err) - buildMap.Store(build.BuildName, nil) - continue - } - buildMap.Store(build.BuildName, buildInfo) - } else { - if buildMapValue == nil { - log.Errorf("find build: %s error: %v", build.BuildName, err) - continue - } - buildInfo = buildMapValue.(*commonmodels.Build) - } - - if err := fillBuildDetail(buildInfo, build.ServiceName, build.ServiceModule, &buildTemplateMap); err != nil { - log.Errorf("fill build: %s detail error: %v", build.BuildName, err) + buildInfo, err := buildSvc.GetBuild(build.BuildName, build.ServiceName, build.ServiceModule) + if err != nil { + log.Errorf("find build: %s error: %v", build.BuildName, err) continue } for _, target := range buildInfo.Targets { @@ -366,25 +309,19 @@ func (j *BuildJob) UpdateWithLatestSetting() error { userConfiguredService[key] = service } + buildSvc := commonservice.NewBuildService() mergedServiceAndBuilds := make([]*commonmodels.ServiceAndBuild, 0) - var buildTemplateMap sync.Map - for _, buildInfo := range latestSpec.ServiceAndBuilds { key := fmt.Sprintf("%s++%s", buildInfo.ServiceName, buildInfo.ServiceModule) // if a service is selected (in the map above) and is in the latest build job config, add it to the list. // user defined kv and repo should be merged into the newly created list. if userDefinedArgs, ok := userConfiguredService[key]; ok { - latestBuild, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: buildInfo.BuildName, ProductName: j.workflow.Project}) + latestBuild, err := buildSvc.GetBuild(buildInfo.BuildName, buildInfo.ServiceName, buildInfo.ServiceModule) if err != nil { log.Errorf("find build: %s error: %v", buildInfo.BuildName, err) continue } - if err := fillBuildDetail(latestBuild, buildInfo.ServiceName, buildInfo.ServiceModule, &buildTemplateMap); err != nil { - log.Errorf("fill build: %s detail error: %v", buildInfo.BuildName, err) - continue - } - for _, target := range latestBuild.Targets { if target.ServiceName == buildInfo.ServiceName && target.ServiceModule == buildInfo.ServiceModule { buildInfo.Repos = mergeRepos(latestBuild.Repos, buildInfo.Repos) @@ -448,10 +385,7 @@ func (j *BuildJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { return resp, fmt.Errorf("find default s3 storage error: %v", err) } - var ( - buildMap sync.Map - buildTemplateMap sync.Map - ) + buildSvc := commonservice.NewBuildService() for _, build := range j.spec.ServiceAndBuilds { imageTag := commonservice.ReleaseCandidate(build.Repos, taskID, j.workflow.Project, build.ServiceModule, "", build.ImageName, "image") @@ -465,20 +399,9 @@ func (j *BuildJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { pkgFile := fmt.Sprintf("%s.tar.gz", commonservice.ReleaseCandidate(build.Repos, taskID, j.workflow.Project, build.ServiceModule, "", build.ImageName, "tar")) - var buildInfo *commonmodels.Build - buildMapValue, ok := buildMap.Load(build.BuildName) - if !ok { - buildInfo, err = commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: build.BuildName, ProductName: j.workflow.Project}) - if err != nil { - return resp, fmt.Errorf("find build: %s error: %v", build.BuildName, err) - } - buildMap.Store(build.BuildName, buildInfo) - } else { - buildInfo = buildMapValue.(*commonmodels.Build) - } - // it only fills build detail created from template - if err := fillBuildDetail(buildInfo, build.ServiceName, build.ServiceModule, &buildTemplateMap); err != nil { - return resp, err + buildInfo, err := buildSvc.GetBuild(build.BuildName, build.ServiceName, build.ServiceModule) + if err != nil { + return resp, fmt.Errorf("find build: %s error: %v", build.BuildName, err) } basicImage, err := commonrepo.NewBasicImageColl().Find(buildInfo.PreBuild.ImageID) if err != nil { @@ -876,56 +799,6 @@ func modelS3toS3(modelS3 *commonmodels.S3Storage) *step.S3 { return resp } -func fillBuildDetail(moduleBuild *commonmodels.Build, serviceName, serviceModule string, buildTemplateMap *sync.Map) error { - if moduleBuild.TemplateID == "" { - return nil - } - - var err error - var buildTemplate *commonmodels.BuildTemplate - buildTemplateMapValue, ok := buildTemplateMap.Load(moduleBuild.TemplateID) - if !ok { - buildTemplate, err = commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{ - ID: moduleBuild.TemplateID, - }) - if err != nil { - return fmt.Errorf("failed to find build template with id: %s, err: %s", moduleBuild.TemplateID, err) - } - buildTemplateMap.Store(moduleBuild.TemplateID, buildTemplate) - } else { - buildTemplate = buildTemplateMapValue.(*commonmodels.BuildTemplate) - } - - moduleBuild.Timeout = buildTemplate.Timeout - moduleBuild.PreBuild = buildTemplate.PreBuild - moduleBuild.JenkinsBuild = buildTemplate.JenkinsBuild - moduleBuild.ScriptType = buildTemplate.ScriptType - moduleBuild.Scripts = buildTemplate.Scripts - moduleBuild.PostBuild = buildTemplate.PostBuild - moduleBuild.SSHs = buildTemplate.SSHs - moduleBuild.PMDeployScripts = buildTemplate.PMDeployScripts - moduleBuild.CacheEnable = buildTemplate.CacheEnable - moduleBuild.CacheDirType = buildTemplate.CacheDirType - moduleBuild.CacheUserDir = buildTemplate.CacheUserDir - moduleBuild.AdvancedSettingsModified = buildTemplate.AdvancedSettingsModified - moduleBuild.Outputs = buildTemplate.Outputs - moduleBuild.Infrastructure = buildTemplate.Infrastructure - moduleBuild.VMLabels = buildTemplate.VmLabels - - // repos are configured by service modules - for _, serviceConfig := range moduleBuild.Targets { - if serviceConfig.ServiceName == serviceName && serviceConfig.ServiceModule == serviceModule { - moduleBuild.Repos = serviceConfig.Repos - if moduleBuild.PreBuild == nil { - moduleBuild.PreBuild = &commonmodels.PreBuild{} - } - moduleBuild.PreBuild.Envs = commonservice.MergeBuildEnvs(moduleBuild.PreBuild.Envs, serviceConfig.Envs) - break - } - } - return nil -} - func renderEnv(data string, kvs []*commonmodels.KeyVal) string { mapper := func(data string) string { for _, envar := range kvs { @@ -947,7 +820,7 @@ func mergeRepos(templateRepos []*types.Repository, customRepos []*types.Reposito if repo.RepoNamespace == "" { repo.RepoNamespace = repo.RepoOwner } - repoKey := strings.Join([]string{repo.Source, repo.RepoNamespace, repo.RepoName}, "/") + repoKey := strings.Join([]string{repo.Source, repo.GetRepoNamespace(), repo.RepoName}, "/") customRepoMap[repoKey] = repo } for _, repo := range templateRepos { @@ -1047,3 +920,147 @@ func ensureBuildInOutputs(outputs []*commonmodels.Output) []*commonmodels.Output func getBuildJobCacheObjectPath(workflowName, serviceName, serviceModule string) string { return fmt.Sprintf("%s/cache/%s/%s", workflowName, serviceName, serviceModule) } + +func (j *BuildJob) getOriginReferedJobTargets(jobName string) ([]*commonmodels.ServiceAndBuild, error) { + servicetargets := []*commonmodels.ServiceAndBuild{} + originTargetMap := make(map[string]*commonmodels.ServiceAndBuild) + + servicesMap, err := repository.GetMaxRevisionsServicesMap(j.workflow.Project, false) + if err != nil { + return nil, fmt.Errorf("get services map error: %v", err) + } + + buildSvc := commonservice.NewBuildService() + for _, build := range j.spec.ServiceAndBuilds { + buildInfo, err := buildSvc.GetBuild(build.BuildName, build.ServiceName, build.ServiceModule) + if err != nil { + err = fmt.Errorf("get build %s failed, err: %s", build.BuildName, err) + log.Error(err) + return nil, err + } + + target := &commonmodels.ServiceAndBuild{ + ServiceName: build.ServiceName, + ServiceModule: build.ServiceModule, + KeyVals: buildInfo.PreBuild.Envs, + Repos: buildInfo.Repos, + } + + service := servicesMap[build.ServiceName] + if service == nil { + return nil, fmt.Errorf("service %s not found", build.ServiceName) + } + for _, container := range service.Containers { + if container.Name == build.ServiceModule { + target.ImageName = container.ImageName + break + } + } + + originTargetMap[target.GetKey()] = target + } + + for _, stage := range j.workflow.Stages { + for _, job := range stage.Jobs { + if job.Name != jobName { + continue + } + if job.JobType == config.JobZadigBuild { + buildSpec := &commonmodels.ZadigBuildJobSpec{} + if err := commonmodels.IToi(job.Spec, buildSpec); err != nil { + return servicetargets, err + } + for _, build := range buildSpec.ServiceAndBuilds { + target := &commonmodels.ServiceAndBuild{ + ServiceName: build.ServiceName, + ServiceModule: build.ServiceModule, + } + if originService, ok := originTargetMap[build.GetKey()]; ok { + target.KeyVals = originService.KeyVals + target.Repos = originService.Repos + target.BuildName = originService.BuildName + target.ImageName = originService.ImageName + target.ShareStorageInfo = originService.ShareStorageInfo + } else { + return nil, fmt.Errorf("service %s not found in workflow config", target.GetKey()) + } + + servicetargets = append(servicetargets, target) + } + return servicetargets, nil + } + if job.JobType == config.JobZadigDistributeImage { + distributeSpec := &commonmodels.ZadigDistributeImageJobSpec{} + if err := commonmodels.IToi(job.Spec, distributeSpec); err != nil { + return servicetargets, err + } + for _, distribute := range distributeSpec.Targets { + target := &commonmodels.ServiceAndBuild{ + ServiceName: distribute.ServiceName, + ServiceModule: distribute.ServiceModule, + } + if originService, ok := originTargetMap[target.GetKey()]; ok { + target.KeyVals = originService.KeyVals + target.Repos = originService.Repos + target.BuildName = originService.BuildName + target.ImageName = originService.ImageName + target.ShareStorageInfo = originService.ShareStorageInfo + } else { + return nil, fmt.Errorf("service %s not found in workflow config", target.GetKey()) + } + servicetargets = append(servicetargets, target) + } + return servicetargets, nil + } + if job.JobType == config.JobZadigDeploy { + deploySpec := &commonmodels.ZadigDeployJobSpec{} + if err := commonmodels.IToi(job.Spec, deploySpec); err != nil { + return servicetargets, err + } + for _, svc := range deploySpec.Services { + for _, module := range svc.Modules { + target := &commonmodels.ServiceAndBuild{ + ServiceName: svc.ServiceName, + ServiceModule: module.ServiceModule, + } + if originService, ok := originTargetMap[target.GetKey()]; ok { + target.KeyVals = originService.KeyVals + target.Repos = originService.Repos + target.BuildName = originService.BuildName + target.ImageName = originService.ImageName + target.ShareStorageInfo = originService.ShareStorageInfo + } else { + return nil, fmt.Errorf("service %s not found in workflow config", target.GetKey()) + } + servicetargets = append(servicetargets, target) + } + } + return servicetargets, nil + } + if job.JobType == config.JobZadigScanning { + scanningSpec := &commonmodels.ZadigScanningJobSpec{} + if err := commonmodels.IToi(job.Spec, scanningSpec); err != nil { + return servicetargets, err + } + for _, svc := range scanningSpec.TargetServices { + target := &commonmodels.ServiceAndBuild{ + ServiceName: svc.ServiceName, + ServiceModule: svc.ServiceModule, + } + if originService, ok := originTargetMap[target.GetKey()]; ok { + target.KeyVals = originService.KeyVals + target.Repos = originService.Repos + target.BuildName = originService.BuildName + target.ImageName = originService.ImageName + target.ShareStorageInfo = originService.ShareStorageInfo + } else { + return nil, fmt.Errorf("service %s not found in workflow config", target.GetKey()) + } + servicetargets = append(servicetargets, target) + } + return servicetargets, nil + } + } + } + return nil, fmt.Errorf("build job %s not found", jobName) +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_freestyle.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_freestyle.go index 0c97f7163d..b4e9ce199b 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_freestyle.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_freestyle.go @@ -285,15 +285,26 @@ func (j *FreeStyleJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { if j.spec.FreestyleJobType == config.ServiceFreeStyleJobType { tasks := []*commonmodels.JobTask{} - switch j.spec.Source { - case config.SourceRuntime, config.SourceFromJob: - for _, service := range j.spec.Services { - task, err := j.toJob(taskID, registries, service, logger) - if err != nil { - return nil, err - } - tasks = append(tasks, task) + // get info from previous job + if j.spec.Source == config.SourceFromJob { + // adapt to the front end, use the direct quoted job name + if j.spec.OriginJobName != "" { + j.spec.JobName = j.spec.OriginJobName + } + targets, err := j.getOriginReferedJobTargets(j.spec.JobName) + if err != nil { + return resp, fmt.Errorf("get origin refered job: %s targets failed, err: %v", j.spec.JobName, err) + } + // clear service list to prevent old data from remaining + j.spec.Services = targets + } + + for _, service := range j.spec.Services { + task, err := j.toJob(taskID, registries, service, logger) + if err != nil { + return nil, err } + tasks = append(tasks, task) } return tasks, nil } else { @@ -526,3 +537,93 @@ func (j *FreeStyleJob) GetOutPuts(log *zap.SugaredLogger) []string { resp = append(resp, getOutputKey(jobKey, j.spec.Outputs)...) return resp } + +func (j *FreeStyleJob) getOriginReferedJobTargets(jobName string) ([]*commonmodels.FreeStyleServiceInfo, error) { + servicetargets := []*commonmodels.FreeStyleServiceInfo{} + + orginRepos := []*types.Repository{} + for _, step := range j.spec.Steps { + if step.StepType == config.StepGit { + stepSpec := &steptypes.StepGitSpec{} + if err := commonmodels.IToi(step.Spec, stepSpec); err != nil { + return servicetargets, fmt.Errorf("parse git step spec error: %v", err) + } + orginRepos = stepSpec.Repos + } + } + + for _, stage := range j.workflow.Stages { + for _, job := range stage.Jobs { + if job.Name != jobName { + continue + } + if job.JobType == config.JobZadigBuild { + buildSpec := &commonmodels.ZadigBuildJobSpec{} + if err := commonmodels.IToi(job.Spec, buildSpec); err != nil { + return servicetargets, err + } + for _, build := range buildSpec.ServiceAndBuilds { + info := &commonmodels.FreeStyleServiceInfo{ + ServiceName: build.ServiceName, + ServiceModule: build.ServiceModule, + KeyVals: j.spec.Properties.Envs, + Repos: orginRepos, + } + servicetargets = append(servicetargets, info) + } + return servicetargets, nil + } + if job.JobType == config.JobZadigDistributeImage { + distributeSpec := &commonmodels.ZadigDistributeImageJobSpec{} + if err := commonmodels.IToi(job.Spec, distributeSpec); err != nil { + return servicetargets, err + } + for _, distribute := range distributeSpec.Targets { + target := &commonmodels.FreeStyleServiceInfo{ + ServiceName: distribute.ServiceName, + ServiceModule: distribute.ServiceModule, + KeyVals: j.spec.Properties.Envs, + Repos: orginRepos, + } + servicetargets = append(servicetargets, target) + } + return servicetargets, nil + } + if job.JobType == config.JobZadigDeploy { + deploySpec := &commonmodels.ZadigDeployJobSpec{} + if err := commonmodels.IToi(job.Spec, deploySpec); err != nil { + return servicetargets, err + } + for _, svc := range deploySpec.Services { + for _, module := range svc.Modules { + target := &commonmodels.FreeStyleServiceInfo{ + ServiceName: svc.ServiceName, + ServiceModule: module.ServiceModule, + KeyVals: j.spec.Properties.Envs, + Repos: orginRepos, + } + servicetargets = append(servicetargets, target) + } + } + return servicetargets, nil + } + if job.JobType == config.JobZadigScanning { + scanningSpec := &commonmodels.ZadigScanningJobSpec{} + if err := commonmodels.IToi(job.Spec, scanningSpec); err != nil { + return servicetargets, err + } + for _, svc := range scanningSpec.TargetServices { + target := &commonmodels.FreeStyleServiceInfo{ + ServiceName: svc.ServiceName, + ServiceModule: svc.ServiceModule, + KeyVals: j.spec.Properties.Envs, + Repos: orginRepos, + } + servicetargets = append(servicetargets, target) + } + return servicetargets, nil + } + } + } + return nil, fmt.Errorf("freestyle job %s not found", jobName) +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go index be53e61767..6c178138ac 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_scanning.go @@ -255,6 +255,9 @@ func (j *ScanningJob) MergeWebhookRepo(webhookRepo *types.Repository) error { for _, scanning := range j.spec.Scannings { scanning.Repos = mergeRepos(scanning.Repos, []*types.Repository{webhookRepo}) } + for _, serviceAndScaning := range j.spec.ServiceAndScannings { + serviceAndScaning.Repos = mergeRepos(serviceAndScaning.Repos, []*types.Repository{webhookRepo}) + } j.job.Spec = j.spec return nil } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_testing.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_testing.go index 2c96a993f0..1cd9adb354 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_testing.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_testing.go @@ -210,6 +210,9 @@ func (j *TestingJob) MergeWebhookRepo(webhookRepo *types.Repository) error { for _, testing := range j.spec.TestModules { testing.Repos = mergeRepos(testing.Repos, []*types.Repository{webhookRepo}) } + for _, serviceAndTest := range j.spec.ServiceAndTests { + serviceAndTest.Repos = mergeRepos(serviceAndTest.Repos, []*types.Repository{webhookRepo}) + } j.job.Spec = j.spec return nil } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_vm_deploy.go b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_vm_deploy.go index f93f267f75..ab1f5aa45f 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/job/job_vm_deploy.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/job/job_vm_deploy.go @@ -21,7 +21,6 @@ import ( "fmt" "strconv" "strings" - "sync" "github.com/koderover/zadig/v2/pkg/util" "go.uber.org/zap" @@ -31,6 +30,7 @@ import ( commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb/template" + commonservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service" codehostdb "github.com/koderover/zadig/v2/pkg/microservice/systemconfig/core/codehost/repository/mongodb" "github.com/koderover/zadig/v2/pkg/setting" "github.com/koderover/zadig/v2/pkg/tool/log" @@ -418,10 +418,7 @@ func (j *VMDeployJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { originS3StorageSubfolder = s3Storage.Subfolder } - var ( - buildMap sync.Map - buildTemplateMap sync.Map - ) + buildSvc := commonservice.NewBuildService() for _, vmDeployInfo := range j.spec.ServiceAndVMDeploys { s3Storage.Subfolder = originS3StorageSubfolder @@ -429,20 +426,9 @@ func (j *VMDeployJob) ToJobs(taskID int64) ([]*commonmodels.JobTask, error) { if !ok { return resp, fmt.Errorf("service %s not found", vmDeployInfo.ServiceName) } - var buildInfo *commonmodels.Build - buildMapValue, ok := buildMap.Load(service.BuildName) - if !ok { - buildInfo, err = commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: service.BuildName, ProductName: j.workflow.Project}) - if err != nil { - return resp, fmt.Errorf("find build: %s error: %v", service.BuildName, err) - } - buildMap.Store(service.BuildName, buildInfo) - } else { - buildInfo = buildMapValue.(*commonmodels.Build) - } - // it only fills build detail created from template - if err := fillBuildDetail(buildInfo, vmDeployInfo.ServiceName, vmDeployInfo.ServiceName, &buildTemplateMap); err != nil { - return resp, err + buildInfo, err := buildSvc.GetBuild(service.BuildName, vmDeployInfo.ServiceName, vmDeployInfo.ServiceModule) + if err != nil { + return resp, fmt.Errorf("get build info for service %s error: %v", vmDeployInfo.ServiceName, err) } basicImage, err := commonrepo.NewBasicImageColl().Find(buildInfo.PreDeploy.ImageID) if err != nil {