Skip to content

Commit 2277ce0

Browse files
pandatixpandatix
committed
ci: add kind testing
1 parent ece85ed commit 2277ce0

File tree

8 files changed

+315
-87
lines changed

8 files changed

+315
-87
lines changed

‎.github/workflows/kind.yaml

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: Create Cluster with Registry
2+
3+
on:
4+
push: {}
5+
6+
jobs:
7+
setup-and-deploy:
8+
runs-on: ubuntu-latest
9+
steps:
10+
- name: Checkout repository
11+
uses: actions/checkout@v3
12+
13+
- name: Set up Docker registry
14+
run: |
15+
docker network create kind || true
16+
docker run -d --network kind --name registry -p 5000:5000 registry:2
17+
18+
- name: Write config file
19+
run: |
20+
cat <<EOF > kind-config.yaml
21+
apiVersion: kind.x-k8s.io/v1alpha4
22+
kind: Cluster
23+
containerdConfigPatches:
24+
- |
25+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
26+
endpoint = ["http://registry:5000"]
27+
28+
kubeadmConfigPatches:
29+
- |
30+
kind: ClusterConfiguration
31+
apiServer:
32+
extraArgs:
33+
"service-node-port-range": "30000-30005"
34+
35+
nodes:
36+
- role: control-plane
37+
extraPortMappings:
38+
- containerPort: 80
39+
hostPort: 80
40+
- containerPort: 30000
41+
hostPort: 30000
42+
- containerPort: 30001
43+
hostPort: 30001
44+
- containerPort: 30002
45+
hostPort: 30002
46+
- containerPort: 30003
47+
hostPort: 30003
48+
- containerPort: 30004
49+
hostPort: 30004
50+
- containerPort: 30005
51+
hostPort: 30005
52+
EOF
53+
54+
- name: Set up Kind cluster
55+
uses: helm/kind-action@v1.7.0
56+
with:
57+
version: v0.20.0
58+
config: kind-config.yaml
59+
cluster_name: kind
60+
env:
61+
KIND_EXPERIMENTAL_DOCKER_NETWORK: kind
62+
63+
- name: Build and push CM
64+
run: |
65+
docker build \
66+
-t localhost:5000/ctferio/chall-manager:${{ github.sha }} \
67+
-f Dockerfile.chall-manager \
68+
.
69+
docker push localhost:5000/ctferio/chall-manager:${{ github.sha }}
70+
71+
- name: Build and push CMJ
72+
run: |
73+
docker build \
74+
-t localhost:5000/ctferio/chall-manager-janitor:${{ github.sha }} \
75+
-f Dockerfile.chall-manager-janitor \
76+
.
77+
docker push localhost:5000/ctferio/chall-manager-janitor:${{ github.sha }}
78+
79+
- name: Install Pulumi
80+
uses: pulumi/actions@v4
81+
82+
- name: Configure stack
83+
run: |
84+
pulumi login --local
85+
export PULUMI_CONFIG_PASSPHRASE=""
86+
87+
cd deploy
88+
pulumi stack init dev
89+
pulumi config set private-registry "localhost:5000/"
90+
pulumi config set tag ${{ github.sha }}
91+
pulumi config set namespace cm-in-ci
92+
pulumi config set no_pvc true
93+
pulumi config set expose true
94+
pulumi up -y
95+
96+
kubectl get svc -n cm-in-ci
97+
URL="http://$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-control-plane):$(pulumi stack output exposed_port)/healthcheck"
98+
echo $URL
99+
curl -v $URL

‎deploy/Pulumi.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,14 @@ config:
3030
type: boolean
3131
description: If set to true, turns on the REST API Swagger UI. Do not activate in production. (Optional)
3232
default: false
33+
no_pvc:
34+
type: boolean
35+
description: Whether to run without a PVC. This is not recommended for production workloads, but for CI purposes.
36+
default: false
37+
expose:
38+
type: boolean
39+
description: Whether to expose to external networking the Chall-Manager service. DO NOT TURN ON IF YOU DON'T UNDERSTAND THE IMPACT.
40+
default: false
3341
otel.endpoint:
3442
type: string
3543
description: The OpenTelemetry Collector endpoint to set signals to. (Optional)

‎deploy/main.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ func main() {
3030
PrivateRegistry: pulumi.String(cfg.PrivateRegistry),
3131
Replicas: pulumi.Int(cfg.Replicas),
3232
Swagger: cfg.Swagger,
33+
NoPVC: cfg.NoPVC,
34+
Expose: cfg.Expose,
3335
EtcdReplicas: nil,
3436
JanitorCron: nil,
3537
Otel: nil,
@@ -53,6 +55,7 @@ func main() {
5355
}
5456

5557
ctx.Export("endpoint", cm.Endpoint)
58+
ctx.Export("exposed_port", cm.ExposedPort)
5659

5760
return nil
5861
})
@@ -67,6 +70,8 @@ type (
6770
Replicas int `json:"replicas"`
6871
Janitor *JanitorConfig `json:"janitor"`
6972
Swagger bool `json:"swagger"`
73+
NoPVC bool `json:"no_pvc"`
74+
Expose bool `json:"expose"`
7075
Otel *OtelConfig `json:"otel"`
7176
}
7277

@@ -92,6 +97,8 @@ func loadConfig(ctx *pulumi.Context) *Config {
9297
PrivateRegistry: cfg.Get("private-registry"),
9398
Replicas: cfg.GetInt("replicas"),
9499
Swagger: cfg.GetBool("swagger"),
100+
NoPVC: cfg.GetBool("no_pvc"),
101+
Expose: cfg.GetBool("expose"),
95102
}
96103

97104
var etcdC EtcdConfig

‎deploy/services/chall-manager.go

Lines changed: 74 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
package services
22

33
import (
4+
"fmt"
5+
"strconv"
46
"strings"
57

68
"github.com/ctfer-io/chall-manager/deploy/common"
79
"github.com/ctfer-io/chall-manager/deploy/services/parts"
10+
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
11+
v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
812
netwv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/networking/v1"
913
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
1014
)
@@ -19,13 +23,17 @@ type (
1923
cm *parts.ChallManager
2024
cmj *parts.ChallManagerJanitor
2125

26+
// Exposure
27+
svc *corev1.Service
28+
2229
// Interface & ports network policies
2330
cmToEtcd *netwv1.NetworkPolicy
2431
cmjToCm *netwv1.NetworkPolicy
2532

2633
// Outputs
2734

28-
Endpoint pulumi.StringOutput
35+
Endpoint pulumi.StringOutput
36+
ExposedPort pulumi.IntPtrOutput
2937
}
3038

3139
// ChallManagerArgs contains all the parametrization of a Chall-Manager
@@ -47,7 +55,7 @@ type (
4755
JanitorCron pulumi.StringPtrInput
4856
janitorCron pulumi.StringOutput
4957

50-
Swagger bool
58+
Swagger, NoPVC, Expose bool
5159

5260
Otel *common.OtelArgs
5361
}
@@ -62,19 +70,41 @@ const (
6270
//
6371
// It is not made to be exposed to outer world (outside of the cluster).
6472
func NewChallManager(ctx *pulumi.Context, name string, args *ChallManagerArgs, opts ...pulumi.ResourceOption) (*ChallManager, error) {
73+
cm := &ChallManager{}
74+
args, err := cm.validate(args)
75+
if err != nil {
76+
return nil, err
77+
}
78+
if err := ctx.RegisterComponentResource("ctfer-io:chall-manager", name, cm, opts...); err != nil {
79+
return nil, err
80+
}
81+
opts = append(opts, pulumi.Parent(cm))
82+
if err := cm.provision(ctx, args, opts...); err != nil {
83+
return nil, err
84+
}
85+
if err := cm.outputs(ctx); err != nil {
86+
return nil, err
87+
}
88+
return cm, nil
89+
}
90+
91+
func (cm *ChallManager) validate(args *ChallManagerArgs) (*ChallManagerArgs, error) {
6592
if args == nil {
6693
args = &ChallManagerArgs{}
6794
}
95+
6896
if args.Tag == nil || args.Tag == pulumi.String("") {
6997
args.tag = pulumi.String("dev").ToStringOutput()
7098
} else {
7199
args.tag = args.Tag.ToStringPtrOutput().Elem()
72100
}
101+
73102
if args.JanitorCron == nil || args.JanitorCron == pulumi.String("") {
74103
args.janitorCron = pulumi.String(defaultCron).ToStringOutput()
75104
} else {
76105
args.janitorCron = args.JanitorCron.ToStringPtrOutput().Elem()
77106
}
107+
78108
if args.PrivateRegistry == nil {
79109
args.privateRegistry = pulumi.String("").ToStringOutput()
80110
} else {
@@ -92,23 +122,14 @@ func NewChallManager(ctx *pulumi.Context, name string, args *ChallManagerArgs, o
92122
return str
93123
}).(pulumi.StringOutput)
94124
}
125+
95126
if args.Replicas == nil {
96127
args.replicas = pulumi.Int(1).ToIntOutput()
97128
} else {
98129
args.replicas = args.Replicas.ToIntPtrOutput().Elem()
99130
}
100131

101-
cm := &ChallManager{}
102-
if err := ctx.RegisterComponentResource("ctfer-io:chall-manager", name, cm, opts...); err != nil {
103-
return nil, err
104-
}
105-
opts = append(opts, pulumi.Parent(cm))
106-
if err := cm.provision(ctx, args, opts...); err != nil {
107-
return nil, err
108-
}
109-
cm.outputs()
110-
111-
return cm, nil
132+
return args, nil
112133
}
113134

114135
func (cm *ChallManager) provision(ctx *pulumi.Context, args *ChallManagerArgs, opts ...pulumi.ResourceOption) (err error) {
@@ -151,6 +172,7 @@ func (cm *ChallManager) provision(ctx *pulumi.Context, args *ChallManagerArgs, o
151172
}).(pulumi.IntOutput),
152173
Etcd: nil,
153174
Swagger: args.Swagger,
175+
NoPVC: args.NoPVC,
154176
Otel: nil,
155177
}
156178
if args.EtcdReplicas != nil {
@@ -172,6 +194,34 @@ func (cm *ChallManager) provision(ctx *pulumi.Context, args *ChallManagerArgs, o
172194
return
173195
}
174196

197+
if args.Expose {
198+
cm.svc, err = corev1.NewService(ctx, "cm-exposed", &corev1.ServiceArgs{
199+
Metadata: v1.ObjectMetaArgs{
200+
Labels: cm.cm.PodLabels,
201+
Namespace: args.Namespace,
202+
},
203+
Spec: corev1.ServiceSpecArgs{
204+
Type: pulumi.String("NodePort"),
205+
Selector: cm.cm.PodLabels,
206+
Ports: corev1.ServicePortArray{
207+
corev1.ServicePortArgs{
208+
Port: cm.cm.Endpoint.ApplyT(func(edp string) int {
209+
// On bootstrap there is no valid URL, but port is assigned
210+
pts := strings.Split(edp, ":")
211+
p := pts[len(pts)-1]
212+
port, _ := strconv.Atoi(p)
213+
fmt.Printf("port: %v\n", port)
214+
return port
215+
}).(pulumi.IntOutput),
216+
},
217+
},
218+
},
219+
}, opts...)
220+
if err != nil {
221+
return
222+
}
223+
}
224+
175225
// Deploy janitor
176226
var cmjOtel *common.OtelArgs
177227
if args.Otel != nil {
@@ -294,6 +344,16 @@ func (cm *ChallManager) provision(ctx *pulumi.Context, args *ChallManagerArgs, o
294344
return
295345
}
296346

297-
func (cm *ChallManager) outputs() {
347+
func (cm *ChallManager) outputs(ctx *pulumi.Context) error {
298348
cm.Endpoint = cm.cm.Endpoint
349+
if cm.svc != nil {
350+
cm.ExposedPort = cm.svc.Spec.ApplyT(func(spec corev1.ServiceSpec) *int {
351+
return spec.Ports[0].NodePort
352+
}).(pulumi.IntPtrOutput)
353+
}
354+
355+
return ctx.RegisterResourceOutputs(cm, pulumi.Map{
356+
"endpoint": cm.Endpoint,
357+
"exposed_port": cm.ExposedPort,
358+
})
299359
}

‎deploy/services/parts/chall-manager-janitor.go

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -53,19 +53,41 @@ const (
5353
)
5454

5555
func NewChallManagerJanitor(ctx *pulumi.Context, name string, args *ChallManagerJanitorArgs, opts ...pulumi.ResourceOption) (*ChallManagerJanitor, error) {
56+
cmj := &ChallManagerJanitor{}
57+
args, err := cmj.validate(args)
58+
if err != nil {
59+
return nil, err
60+
}
61+
if err := ctx.RegisterComponentResource("ctfer-io:chall-manager:chall-manager-janitor", name, cmj, opts...); err != nil {
62+
return nil, err
63+
}
64+
opts = append(opts, pulumi.Parent(cmj))
65+
if err := cmj.provision(ctx, args, opts...); err != nil {
66+
return nil, err
67+
}
68+
if err := cmj.outputs(ctx); err != nil {
69+
return nil, err
70+
}
71+
return cmj, nil
72+
}
73+
74+
func (cmj *ChallManagerJanitor) validate(args *ChallManagerJanitorArgs) (*ChallManagerJanitorArgs, error) {
5675
if args == nil {
5776
args = &ChallManagerJanitorArgs{}
5877
}
78+
5979
if args.Tag == nil || args.Tag == pulumi.String("") {
6080
args.tag = pulumi.String("dev").ToStringOutput()
6181
} else {
6282
args.tag = args.Tag.ToStringPtrOutput().Elem()
6383
}
84+
6485
if args.Cron == nil || args.Cron == pulumi.String("") {
6586
args.cron = pulumi.String(defaultCron).ToStringOutput()
6687
} else {
6788
args.cron = args.Cron.ToStringPtrOutput().Elem()
6889
}
90+
6991
if args.PrivateRegistry == nil || args.PrivateRegistry == pulumi.String("") {
7092
args.privateRegistry = pulumi.String("").ToStringOutput()
7193
} else {
@@ -80,17 +102,7 @@ func NewChallManagerJanitor(ctx *pulumi.Context, name string, args *ChallManager
80102
}).(pulumi.StringOutput)
81103
}
82104

83-
cmj := &ChallManagerJanitor{}
84-
if err := ctx.RegisterComponentResource("ctfer-io:chall-manager:chall-manager-janitor", name, cmj, opts...); err != nil {
85-
return nil, err
86-
}
87-
opts = append(opts, pulumi.Parent(cmj))
88-
if err := cmj.provision(ctx, args, opts...); err != nil {
89-
return nil, err
90-
}
91-
cmj.outputs()
92-
93-
return cmj, nil
105+
return args, nil
94106
}
95107

96108
func (cmj *ChallManagerJanitor) provision(ctx *pulumi.Context, args *ChallManagerJanitorArgs, opts ...pulumi.ResourceOption) (err error) {
@@ -171,6 +183,10 @@ func (cmj *ChallManagerJanitor) provision(ctx *pulumi.Context, args *ChallManage
171183
return
172184
}
173185

174-
func (cmj *ChallManagerJanitor) outputs() {
186+
func (cmj *ChallManagerJanitor) outputs(ctx *pulumi.Context) error {
175187
cmj.PodLabels = cmj.cjob.Spec.JobTemplate().Metadata().Labels()
188+
189+
return ctx.RegisterResourceOutputs(cmj, pulumi.Map{
190+
"podLabels": cmj.PodLabels,
191+
})
176192
}

0 commit comments

Comments
 (0)