Skip to content

Commit

Permalink
support reference repos in workflow
Browse files Browse the repository at this point in the history 
Signed-off-by: Patrick Zhao <zhaoyu@koderover.com>
  • Loading branch information
@PetrusZ
PetrusZ committed Sep 29, 2024
1 parent 21088b5 commit 6cdfeee
Show file tree
Hide file tree
Showing 12 changed files with 399 additions and 610 deletions.
31 changes: 26 additions & 5 deletions pkg/microservice/aslan/core/common/repository/models/workflow_v4.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,8 @@ type FreestyleJobSpec struct {
// fromjob/runtime, runtime 表示运行时输入，fromjob 表示从上游构建任务中获取
Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"`
// 当 source 为 fromjob 时需要，指定部署镜像来源是上游哪一个构建任务
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"`
// save the origin quoted job name
OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"`
Properties *JobProperties `bson:"properties" yaml:"properties" json:"properties"`
Expand All @@ -297,15 +298,26 @@ type FreestyleJobSpec struct {

type FreeStyleServiceInfo struct {
ServiceName string `bson:"service_name" yaml:"service_name" json:"service_name"`
ServiceModule string `bson:"service_module" yaml:"service_module" json:"service_module"`
ServiceModule string `bson:"service_module" yaml:"service_module" json:"service_module"`
Repos []*types.Repository `bson:"repos" yaml:"repos" json:"repos"`
KeyVals []*KeyVal `bson:"key_vals" yaml:"key_vals" json:"key_vals"`
}

func (i *FreeStyleServiceInfo) GetKey() string {
if i == nil {
return ""
}
return i.ServiceName + "-" + i.ServiceModule
}

type ZadigBuildJobSpec struct {
DockerRegistryID string `bson:"docker_registry_id" yaml:"docker_registry_id" json:"docker_registry_id"`
ServiceAndBuilds []*ServiceAndBuild `bson:"service_and_builds" yaml:"service_and_builds" json:"service_and_builds"`
ServiceAndBuildsOptions []*ServiceAndBuild `bson:"-" yaml:"service_and_builds_options" json:"service_and_builds_options"`
Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"`
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"`
RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"`
DockerRegistryID string `bson:"docker_registry_id" yaml:"docker_registry_id" json:"docker_registry_id"`
ServiceAndBuilds []*ServiceAndBuild `bson:"service_and_builds" yaml:"service_and_builds" json:"service_and_builds"`
ServiceAndBuildsOptions []*ServiceAndBuild `bson:"-" yaml:"service_and_builds_options" json:"service_and_builds_options"`
}

type ServiceAndBuild struct {
Expand All @@ -320,6 +332,13 @@ type ServiceAndBuild struct {
ShareStorageInfo *ShareStorageInfo `bson:"share_storage_info" yaml:"share_storage_info" json:"share_storage_info"`
}

func (i *ServiceAndBuild) GetKey() string {
if i == nil {
return ""
}
return i.ServiceName + "-" + i.ServiceModule
}

type ZadigDeployJobSpec struct {
Env string `bson:"env" yaml:"env" json:"env"`
EnvOptions []*ZadigDeployEnvInformation `bson:"-" yaml:"env_options" json:"env_options"`
Expand Down Expand Up @@ -481,6 +500,7 @@ type ZadigTestingJobSpec struct {
Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"`
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"`
RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"`
// selected service in service testing
TargetServices []*ServiceTestTarget `bson:"target_services" yaml:"target_services" json:"target_services"`
// field for non-service tests.
Expand Down Expand Up @@ -513,6 +533,7 @@ type ZadigScanningJobSpec struct {
Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"`
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"`
RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"`
// Scannings used only for normal scanning. for service scanning we use
Scannings []*ScanningModule `bson:"scannings" yaml:"scannings" json:"scannings"`
// ServiceAndScannings is the configured field for this job. It includes all the services along with its configured scanning.
Expand Down
83 changes: 83 additions & 0 deletions pkg/microservice/aslan/core/common/service/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ limitations under the License.
package service

import (
"fmt"
"strings"
"sync"
"time"

"go.uber.org/zap"
Expand Down Expand Up @@ -219,3 +221,84 @@ func MergeParams(templateEnvs []*commonmodels.Param, customEnvs []*commonmodels.
}
return retEnvs
}

type BuildService struct {
BuildMap sync.Map
BuildTemplateMap sync.Map
}

func NewBuildService() *BuildService {
return &BuildService{
BuildMap: sync.Map{},
BuildTemplateMap: sync.Map{},
}
}

func (c *BuildService) GetBuild(buildName, serviceName, serviceModule string) (*commonmodels.Build, error) {
buildInfo := &commonmodels.Build{}
buildMapValue, ok := c.BuildMap.Load(buildName)
if !ok {
buildInfo, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: buildName})
if err != nil {
return nil, fmt.Errorf("find build: %s error: %v", buildName, err)
}
c.BuildMap.Store(buildName, buildInfo)
} else {
buildInfo = buildMapValue.(*commonmodels.Build)
}

if err := FillBuildDetail(buildInfo, serviceName, serviceModule, &c.BuildTemplateMap); err != nil {
return nil, err
}
return buildInfo, nil
}

func FillBuildDetail(moduleBuild *commonmodels.Build, serviceName, serviceModule string, buildTemplateMap *sync.Map) error {
if moduleBuild.TemplateID == "" {
return nil
}

var err error
var buildTemplate *commonmodels.BuildTemplate
buildTemplateMapValue, ok := buildTemplateMap.Load(moduleBuild.TemplateID)
if !ok {
buildTemplate, err = commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{
ID: moduleBuild.TemplateID,
})
if err != nil {
return fmt.Errorf("failed to find build template with id: %s, err: %s", moduleBuild.TemplateID, err)
}
buildTemplateMap.Store(moduleBuild.TemplateID, buildTemplate)
} else {
buildTemplate = buildTemplateMapValue.(*commonmodels.BuildTemplate)
}

moduleBuild.Timeout = buildTemplate.Timeout
moduleBuild.PreBuild = buildTemplate.PreBuild
moduleBuild.JenkinsBuild = buildTemplate.JenkinsBuild
moduleBuild.ScriptType = buildTemplate.ScriptType
moduleBuild.Scripts = buildTemplate.Scripts
moduleBuild.PostBuild = buildTemplate.PostBuild
moduleBuild.SSHs = buildTemplate.SSHs
moduleBuild.PMDeployScripts = buildTemplate.PMDeployScripts
moduleBuild.CacheEnable = buildTemplate.CacheEnable
moduleBuild.CacheDirType = buildTemplate.CacheDirType
moduleBuild.CacheUserDir = buildTemplate.CacheUserDir
moduleBuild.AdvancedSettingsModified = buildTemplate.AdvancedSettingsModified
moduleBuild.Outputs = buildTemplate.Outputs
moduleBuild.Infrastructure = buildTemplate.Infrastructure
moduleBuild.VMLabels = buildTemplate.VmLabels

// repos are configured by service modules
for _, serviceConfig := range moduleBuild.Targets {
if serviceConfig.ServiceName == serviceName && serviceConfig.ServiceModule == serviceModule {
moduleBuild.Repos = serviceConfig.Repos
if moduleBuild.PreBuild == nil {
moduleBuild.PreBuild = &commonmodels.PreBuild{}
}
moduleBuild.PreBuild.Envs = MergeBuildEnvs(moduleBuild.PreBuild.Envs, serviceConfig.Envs)
break
}
}
return nil
}
45 changes: 0 additions & 45 deletions pkg/microservice/aslan/core/workflow/service/webhook/gitlab.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,24 +115,6 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log
commonrepo.NewWebHookUserColl().Upsert(webhookUser)
}

//产品工作流webhook
//wg.Add(1)
//go func() {
// defer wg.Done()
// if err = TriggerWorkflowByGitlabEvent(pushEvent, baseURI, requestID, log); err != nil {
// errorList = multierror.Append(errorList, err)
// }
//}()

//单服务工作流webhook
wg.Add(1)
go func() {
defer wg.Done()
if err = TriggerPipelineByGitlabEvent(pushEvent, baseURI, requestID, log); err != nil {
errorList = multierror.Append(errorList, err)
}
}()

//测试管理webhook
wg.Add(1)
go func() {
Expand Down Expand Up @@ -160,24 +142,6 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log
}

if mergeEvent != nil {
//多服务工作流webhook
//wg.Add(1)
//go func() {
// defer wg.Done()
// if err = TriggerWorkflowByGitlabEvent(mergeEvent, baseURI, requestID, log); err != nil {
// errorList = multierror.Append(errorList, err)
// }
//}()

//单服务工作流webhook
wg.Add(1)
go func() {
defer wg.Done()
if err = TriggerPipelineByGitlabEvent(mergeEvent, baseURI, requestID, log); err != nil {
errorList = multierror.Append(errorList, err)
}
}()

//测试管理webhook
wg.Add(1)
go func() {
Expand Down Expand Up @@ -205,15 +169,6 @@ func ProcessGitlabHook(payload []byte, req *http.Request, requestID string, log
}

if tagEvent != nil {
//// workflow webhook
//wg.Add(1)
//go func() {
// defer wg.Done()
// if err = TriggerWorkflowByGitlabEvent(tagEvent, baseURI, requestID, log); err != nil {
// errorList = multierror.Append(errorList, err)
// }
//}()

//test webhook
wg.Add(1)
go func() {
Expand Down
Loading

0 comments on commit 6cdfeee

Please sign in to comment.