Skip to content

Commit

Permalink
support reference repos in workflow
Browse files Browse the repository at this point in the history 
Signed-off-by: Patrick Zhao <zhaoyu@koderover.com>
  • Loading branch information
@PetrusZ
PetrusZ committed Oct 15, 2024
1 parent d4b0432 commit 156fca6
Show file tree
Hide file tree
Showing 21 changed files with 654 additions and 703 deletions.
2 changes: 1 addition & 1 deletion debug/debug.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ dlv --headless --log --listen :9009 --api-version 2 --accept-multiclient --wd=$(
DLV_PID=$!

# 捕获Ctrl-C信号并终止Delve进程
trap "kill -9 $DLV_PID" SIGINT
trap "pkill dlv $SERVICE_NAME"

# 等待Delve进程结束
wait $DLV_PID
2 changes: 1 addition & 1 deletion debug/golang-debug.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ ENV VERSION=1.4.0
# COPY debug debug

RUN apk update
RUN apk --no-cache add bash git curl
RUN apk --no-cache add bash git curl make
RUN go install github.com/go-delve/delve/cmd/dlv@latest

# https://wiki.alpinelinux.org/wiki/Setting_the_timezone
Expand Down
32 changes: 27 additions & 5 deletions pkg/microservice/aslan/core/common/repository/models/workflow_v4.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,8 @@ type FreestyleJobSpec struct {
// fromjob/runtime, runtime 表示运行时输入，fromjob 表示从上游构建任务中获取
Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"`
// 当 source 为 fromjob 时需要，指定部署镜像来源是上游哪一个构建任务
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"`
// save the origin quoted job name
OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"`
Properties *JobProperties `bson:"properties" yaml:"properties" json:"properties"`
Expand All @@ -297,15 +298,27 @@ type FreestyleJobSpec struct {

type FreeStyleServiceInfo struct {
ServiceName string `bson:"service_name" yaml:"service_name" json:"service_name"`
ServiceModule string `bson:"service_module" yaml:"service_module" json:"service_module"`
ServiceModule string `bson:"service_module" yaml:"service_module" json:"service_module"`
Repos []*types.Repository `bson:"repos" yaml:"repos" json:"repos"`
KeyVals []*KeyVal `bson:"key_vals" yaml:"key_vals" json:"key_vals"`
}

func (i *FreeStyleServiceInfo) GetKey() string {
if i == nil {
return ""
}
return i.ServiceName + "-" + i.ServiceModule
}

type ZadigBuildJobSpec struct {
DockerRegistryID string `bson:"docker_registry_id" yaml:"docker_registry_id" json:"docker_registry_id"`
ServiceAndBuilds []*ServiceAndBuild `bson:"service_and_builds" yaml:"service_and_builds" json:"service_and_builds"`
ServiceAndBuildsOptions []*ServiceAndBuild `bson:"-" yaml:"service_and_builds_options" json:"service_and_builds_options"`
Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"`
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"`
RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"`
DockerRegistryID string `bson:"docker_registry_id" yaml:"docker_registry_id" json:"docker_registry_id"`
DefaultServiceAndBuilds []*ServiceAndBuild `bson:"default_service_and_builds" yaml:"default_service_and_builds" json:"default_service_and_builds"`
ServiceAndBuilds []*ServiceAndBuild `bson:"service_and_builds" yaml:"service_and_builds" json:"service_and_builds"`
ServiceAndBuildsOptions []*ServiceAndBuild `bson:"-" yaml:"service_and_builds_options" json:"service_and_builds_options"`
}

type ServiceAndBuild struct {
Expand All @@ -320,6 +333,13 @@ type ServiceAndBuild struct {
ShareStorageInfo *ShareStorageInfo `bson:"share_storage_info" yaml:"share_storage_info" json:"share_storage_info"`
}

func (i *ServiceAndBuild) GetKey() string {
if i == nil {
return ""
}
return i.ServiceName + "-" + i.ServiceModule
}

type ZadigDeployJobSpec struct {
Env string `bson:"env" yaml:"env" json:"env"`
EnvOptions []*ZadigDeployEnvInformation `bson:"-" yaml:"env_options" json:"env_options"`
Expand Down Expand Up @@ -481,6 +501,7 @@ type ZadigTestingJobSpec struct {
Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"`
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"`
RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"`
// selected service in service testing
TargetServices []*ServiceTestTarget `bson:"target_services" yaml:"target_services" json:"target_services"`
// field for non-service tests.
Expand Down Expand Up @@ -513,6 +534,7 @@ type ZadigScanningJobSpec struct {
Source config.DeploySourceType `bson:"source" yaml:"source" json:"source"`
JobName string `bson:"job_name" yaml:"job_name" json:"job_name"`
OriginJobName string `bson:"origin_job_name" yaml:"origin_job_name" json:"origin_job_name"`
RefRepos bool `bson:"ref_repos" yaml:"ref_repos" json:"ref_repos"`
// Scannings used only for normal scanning. for service scanning we use
Scannings []*ScanningModule `bson:"scannings" yaml:"scannings" json:"scannings"`
// ServiceAndScannings is the configured field for this job. It includes all the services along with its configured scanning.
Expand Down
4 changes: 4 additions & 0 deletions pkg/microservice/aslan/core/common/repository/mongodb/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,10 @@ func (c *BuildColl) Find(opt *BuildFindOption) (*models.Build, error) {
query["product_name"] = opt.ProductName
}

if len(query) == 0 {
return nil, errors.New("empty query")
}

resp := new(models.Build)
err := c.Collection.FindOne(context.TODO(), query).Decode(resp)
if err != nil {
Expand Down
98 changes: 98 additions & 0 deletions pkg/microservice/aslan/core/common/service/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ limitations under the License.
package service

import (
"fmt"
"strings"
"sync"
"time"

"go.uber.org/zap"
Expand Down Expand Up @@ -219,3 +221,99 @@ func MergeParams(templateEnvs []*commonmodels.Param, customEnvs []*commonmodels.
}
return retEnvs
}

type BuildService struct {
BuildMap sync.Map
BuildTemplateMap sync.Map
}

func NewBuildService() *BuildService {
return &BuildService{
BuildMap: sync.Map{},
BuildTemplateMap: sync.Map{},
}
}

func (c *BuildService) GetBuild(buildName, serviceName, serviceModule string) (*commonmodels.Build, error) {
var err error
buildInfo := &commonmodels.Build{}
buildMapValue, ok := c.BuildMap.Load(buildName)
if !ok {
buildInfo, err = commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{Name: buildName})
if err != nil {
c.BuildMap.Store(buildName, nil)
return nil, fmt.Errorf("find build: %s error: %v", buildName, err)
}
c.BuildMap.Store(buildName, buildInfo)
} else {
if buildMapValue == nil {
return nil, fmt.Errorf("failed to find build: %s", buildName)
}
buildInfo = buildMapValue.(*commonmodels.Build)
}

if err := FillBuildDetail(buildInfo, serviceName, serviceModule, &c.BuildTemplateMap); err != nil {
return nil, err
}
return buildInfo, nil
}

func FillBuildDetail(moduleBuild *commonmodels.Build, serviceName, serviceModule string, cacheMap *sync.Map) error {
if moduleBuild.TemplateID == "" {
return nil
}

var err error
var buildTemplate *commonmodels.BuildTemplate

if cacheMap == nil {
buildTemplate, err = commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{
ID: moduleBuild.TemplateID,
})
if err != nil {
return fmt.Errorf("failed to find build template with id: %s, err: %s", moduleBuild.TemplateID, err)
}
} else {
buildTemplateMapValue, ok := cacheMap.Load(moduleBuild.TemplateID)
if !ok {
buildTemplate, err = commonrepo.NewBuildTemplateColl().Find(&commonrepo.BuildTemplateQueryOption{
ID: moduleBuild.TemplateID,
})
if err != nil {
return fmt.Errorf("failed to find build template with id: %s, err: %s", moduleBuild.TemplateID, err)
}
cacheMap.Store(moduleBuild.TemplateID, buildTemplate)
} else {
buildTemplate = buildTemplateMapValue.(*commonmodels.BuildTemplate)
}
}

moduleBuild.Timeout = buildTemplate.Timeout
moduleBuild.PreBuild = buildTemplate.PreBuild
moduleBuild.JenkinsBuild = buildTemplate.JenkinsBuild
moduleBuild.ScriptType = buildTemplate.ScriptType
moduleBuild.Scripts = buildTemplate.Scripts
moduleBuild.PostBuild = buildTemplate.PostBuild
moduleBuild.SSHs = buildTemplate.SSHs
moduleBuild.PMDeployScripts = buildTemplate.PMDeployScripts
moduleBuild.CacheEnable = buildTemplate.CacheEnable
moduleBuild.CacheDirType = buildTemplate.CacheDirType
moduleBuild.CacheUserDir = buildTemplate.CacheUserDir
moduleBuild.AdvancedSettingsModified = buildTemplate.AdvancedSettingsModified
moduleBuild.Outputs = buildTemplate.Outputs
moduleBuild.Infrastructure = buildTemplate.Infrastructure
moduleBuild.VMLabels = buildTemplate.VmLabels

// repos are configured by service modules
for _, serviceConfig := range moduleBuild.Targets {
if serviceConfig.ServiceName == serviceName && serviceConfig.ServiceModule == serviceModule {
moduleBuild.Repos = serviceConfig.Repos
if moduleBuild.PreBuild == nil {
moduleBuild.PreBuild = &commonmodels.PreBuild{}
}
moduleBuild.PreBuild.Envs = MergeBuildEnvs(moduleBuild.PreBuild.Envs, serviceConfig.Envs)
break
}
}
return nil
}
48 changes: 36 additions & 12 deletions pkg/microservice/aslan/core/templatestore/service/workflow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -289,7 +291,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -405,7 +409,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -451,7 +457,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -550,7 +558,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -583,7 +593,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -1122,7 +1134,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -1181,7 +1195,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -1240,7 +1256,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -1284,7 +1302,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -1340,7 +1360,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down Expand Up @@ -1457,7 +1479,9 @@ func InitWorkflowTemplateInfos() []*commonmodels.WorkflowV4Template {
{
Name: "build",
JobType: config.JobZadigBuild,
Spec: commonmodels.ZadigBuildJobSpec{},
Spec: commonmodels.ZadigBuildJobSpec{
Source: config.SourceRuntime,
},
},
},
},
Expand Down
Loading

0 comments on commit 156fca6

Please sign in to comment.