From 483d2cc3e713dab2eee3476b5a1fe614fa03550e Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Thu, 6 Jun 2024 11:49:40 +0200 Subject: [PATCH 01/17] add cilium loadbalancer --- go.mod | 49 ++- go.sum | 133 ++++++-- metal/cloud.go | 16 +- pkg/controllers/loadbalancer/addresspool.go | 2 +- pkg/controllers/loadbalancer/cilium/config.go | 258 ++++++++++++++ .../loadbalancer/cilium/config_test.go | 322 ++++++++++++++++++ pkg/controllers/loadbalancer/cilium/peer.go | 38 +++ pkg/controllers/loadbalancer/config.go | 16 + pkg/controllers/loadbalancer/loadbalancer.go | 11 +- .../{metallb.go => metallb/config.go} | 44 +-- .../config_test.go} | 2 +- pkg/controllers/loadbalancer/metallb/peer.go | 51 +++ pkg/controllers/loadbalancer/peer.go | 27 +- pkg/resources/constants/constants.go | 2 + 14 files changed, 892 insertions(+), 79 deletions(-) create mode 100644 pkg/controllers/loadbalancer/cilium/config.go create mode 100644 pkg/controllers/loadbalancer/cilium/config_test.go create mode 100644 pkg/controllers/loadbalancer/cilium/peer.go create mode 100644 pkg/controllers/loadbalancer/config.go rename pkg/controllers/loadbalancer/{metallb.go => metallb/config.go} (85%) rename pkg/controllers/loadbalancer/{metallb_test.go => metallb/config_test.go} (99%) create mode 100644 pkg/controllers/loadbalancer/metallb/peer.go diff --git a/go.mod b/go.mod index f16f832..66bdd92 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22 require ( github.com/avast/retry-go/v4 v4.5.1 + github.com/cilium/cilium v1.15.5 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/metal-stack/metal-go v0.28.4 @@ -17,7 +18,7 @@ require ( k8s.io/client-go v0.29.3 k8s.io/cloud-provider v0.27.2 k8s.io/component-base v0.29.3 - k8s.io/klog/v2 v2.110.1 + k8s.io/klog/v2 v2.120.0 k8s.io/kubectl v0.29.2 sigs.k8s.io/controller-runtime v0.17.2 sigs.k8s.io/yaml v1.4.0 @@ -32,6 +33,8 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cilium/ebpf v0.12.3 // indirect + github.com/cilium/proxy v0.0.0-20231031145409-f19708f3d018 // indirect github.com/coreos/go-oidc/v3 v3.10.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -39,13 +42,14 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/emicklei/go-restful-openapi/v2 v2.10.2 // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -64,19 +68,26 @@ require ( github.com/google/cel-go v0.17.7 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/gopacket v1.1.19 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.5 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/jwx/v2 v2.0.21 // indirect github.com/lestrrat-go/option v1.0.1 // indirect + github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/metal-stack/security v0.8.0 // indirect @@ -86,19 +97,38 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/segmentio/asm v1.2.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/viper v1.18.2 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.10 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect - go.etcd.io/etcd/client/v3 v3.5.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tklauser/go-sysconf v0.3.13 // indirect + github.com/tklauser/numcpus v0.7.0 // indirect + github.com/vishvananda/netlink v1.2.1-beta.2.0.20231127184239-0ced8385386a // indirect + github.com/vishvananda/netns v0.0.4 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.etcd.io/etcd/api/v3 v3.5.11 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.11 // indirect + go.etcd.io/etcd/client/v3 v3.5.11 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect @@ -109,8 +139,10 @@ require ( go.opentelemetry.io/otel/sdk v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.uber.org/dig v1.17.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect golang.org/x/net v0.24.0 // indirect @@ -126,6 +158,7 @@ require ( google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -133,8 +166,8 @@ require ( k8s.io/component-helpers v0.29.3 // indirect k8s.io/controller-manager v0.27.2 // indirect k8s.io/kms v0.29.3 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 // indirect + k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index e7128d2..544aec4 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJd cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= @@ -19,8 +21,18 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/checkmate v1.0.3 h1:CQC5eOmlAZeEjPrVZY3ZwEBH64lHlx9mXYdUehEwI5w= +github.com/cilium/checkmate v1.0.3/go.mod h1:KiBTasf39/F2hf2yAmHw21YFl3hcEyP4Yk6filxc12A= +github.com/cilium/cilium v1.15.5 h1:AFhWniiqVyQXYfpaPZTRfKdS0pLx+8lCDPp7JpAZqfo= +github.com/cilium/cilium v1.15.5/go.mod h1:hsruyj1KCncND7AyIlbKgHUlk7V+ONxTn3EbrOu39dI= +github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= +github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= +github.com/cilium/proxy v0.0.0-20231031145409-f19708f3d018 h1:R/QlThqx099hS6req1k2Q87fvLSRgCEicQGate9vxO4= +github.com/cilium/proxy v0.0.0-20231031145409-f19708f3d018/go.mod h1:p044XccCmONGIUbx3bJ7qvHXK0RcrdvIvbTGiu/RjUA= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU= @@ -48,24 +60,28 @@ github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapw github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= @@ -111,20 +127,23 @@ github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= @@ -133,6 +152,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -164,6 +187,11 @@ github.com/lestrrat-go/jwx/v2 v2.0.21 h1:jAPKupy4uHgrHFEdjVjNkUgoBKtVDgrQPB/h55F github.com/lestrrat-go/jwx/v2 v2.0.21/go.mod h1:09mLW8zto6bWL9GbwnqAli+ArLf+5M33QLQPDggkUWM= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= +github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -193,17 +221,26 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= @@ -212,19 +249,38 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= +github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -234,27 +290,44 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= +github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= +github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/vishvananda/netlink v1.2.1-beta.2.0.20231127184239-0ced8385386a h1:PdKmLjqKUM8AfjGqDbrF/C56RvuGFDMYB0Z+8TMmGpU= +github.com/vishvananda/netlink v1.2.1-beta.2.0.20231127184239-0ced8385386a/go.mod h1:whJevzBpTrid75eZy99s3DqCmy05NfibNaF2Ol5Ox5A= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= -go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= -go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= -go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= +go.etcd.io/etcd/api/v3 v3.5.11 h1:B54KwXbWDHyD3XYAwprxNzTe7vlhR69LuBgZnMVvS7E= +go.etcd.io/etcd/api/v3 v3.5.11/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= +go.etcd.io/etcd/client/pkg/v3 v3.5.11 h1:bT2xVspdiCj2910T0V+/KHcVKjkUrCZVtk8J2JF2z1A= +go.etcd.io/etcd/client/pkg/v3 v3.5.11/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= go.etcd.io/etcd/client/v2 v2.305.10 h1:MrmRktzv/XF8CvtQt+P6wLUlURaNpSDJHFZhe//2QE4= go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA= -go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= -go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= +go.etcd.io/etcd/client/v3 v3.5.11 h1:ajWtgoNSZJ1gmS8k+icvPtqsqEav+iUorF7b0qozgUU= +go.etcd.io/etcd/client/v3 v3.5.11/go.mod h1:a6xQUEqFJ8vztO1agJh/KQKOMfFI8og52ZconzcDJwE= go.etcd.io/etcd/pkg/v3 v3.5.10 h1:WPR8K0e9kWl1gAhB5A7gEa5ZBTNkT9NdNWrR8Qpo1CM= go.etcd.io/etcd/pkg/v3 v3.5.10/go.mod h1:TKTuCKKcF1zxmfKWDkfz5qqYaE3JncKKZPFf8c1nFUs= go.etcd.io/etcd/raft/v3 v3.5.10 h1:cgNAYe7xrsrn/5kXMSaH8kM/Ky8mAdMqGOxyYwpP0LA= @@ -281,6 +354,8 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= +go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -289,6 +364,8 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= go.universe.tf/metallb v0.14.3 h1:iOhPwpieBgw4hzqhVupk2qRebK+r2pZ1FgwqKkW4ZYs= go.universe.tf/metallb v0.14.3/go.mod h1:cPci8OBPmEJLs4ItfKL4nlntzrNJ6bmDFBRbI3BMAYI= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -296,6 +373,8 @@ golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -313,8 +392,17 @@ golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= @@ -327,6 +415,7 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= @@ -352,6 +441,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -364,8 +455,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= -k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= -k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= +k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= +k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= k8s.io/apiserver v0.29.3 h1:xR7ELlJ/BZSr2n4CnD3lfA4gzFivh0wwfNfz9L0WZcE= @@ -380,16 +471,16 @@ k8s.io/component-helpers v0.29.3 h1:1dqZswuZgT2ZMixYeORyCUOAApXxgsvjVSgfoUT+P4o= k8s.io/component-helpers v0.29.3/go.mod h1:yiDqbRQrnQY+sPju/bL7EkwDJb6LVOots53uZNMZBos= k8s.io/controller-manager v0.27.2 h1:S7984FVb5ajp8YqMQGAm8zXEUEl0Omw6FJlOiQU2Ne8= k8s.io/controller-manager v0.27.2/go.mod h1:2HzIhmjKxSH5dJVjYLuJ7/v9HYluNDcHLh6ZyE6rT18= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/klog/v2 v2.120.0 h1:z+q5mfovBj1fKFxiRzsa2DsJLPIVMk/KFL81LMOfK+8= +k8s.io/klog/v2 v2.120.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kms v0.29.3 h1:ReljsAUhYlm2spdT4yXmY+9a8x8dc/OT4mXvwQPPteQ= k8s.io/kms v0.29.3/go.mod h1:TBGbJKpRUMk59neTMDMddjIDL+D4HuFUbpuiuzmOPg0= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU= +k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= k8s.io/kubectl v0.29.2 h1:uaDYaBhumvkwz0S2XHt36fK0v5IdNgL7HyUniwb2IUo= k8s.io/kubectl v0.29.2/go.mod h1:BhizuYBGcKaHWyq+G7txGw2fXg576QbPrrnQdQDZgqI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= diff --git a/metal/cloud.go b/metal/cloud.go index f5a7fb2..7b25564 100644 --- a/metal/cloud.go +++ b/metal/cloud.go @@ -12,6 +12,8 @@ import ( "github.com/metal-stack/metal-ccm/pkg/controllers/housekeeping" "github.com/metal-stack/metal-ccm/pkg/controllers/instances" "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer/cilium" + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer/metallb" "github.com/metal-stack/metal-ccm/pkg/controllers/zones" "github.com/metal-stack/metal-ccm/pkg/resources/constants" "github.com/metal-stack/metal-ccm/pkg/resources/metal" @@ -44,6 +46,7 @@ func NewCloud(_ io.Reader) (cloudprovider.Interface, error) { partitionID := os.Getenv(constants.MetalPartitionIDEnvVar) clusterID := os.Getenv(constants.MetalClusterIDEnvVar) defaultExternalNetworkID := os.Getenv(constants.MetalDefaultExternalNetworkEnvVar) + loadbalancerType := os.Getenv(constants.Loadbalancer) var ( additionalNetworksString = os.Getenv(constants.MetalAdditionalNetworks) @@ -92,7 +95,18 @@ func NewCloud(_ io.Reader) (cloudprovider.Interface, error) { instancesController := instances.New(defaultExternalNetworkID) zonesController := zones.New() - loadBalancerController := loadbalancer.New(partitionID, projectID, clusterID, defaultExternalNetworkID, additionalNetworks) + + var config loadbalancer.Config + switch loadbalancerType { + case "metallb": + config = metallb.NewMetalLBConfig() + case "cilium": + config = cilium.NewCiliumConfig() + default: + return nil, fmt.Errorf("loadbalancer type must be on of 'metallb' or 'cilium'") + } + + loadBalancerController := loadbalancer.New(partitionID, projectID, clusterID, defaultExternalNetworkID, additionalNetworks, config) klog.Info("initialized cloud controller manager") return &cloud{ diff --git a/pkg/controllers/loadbalancer/addresspool.go b/pkg/controllers/loadbalancer/addresspool.go index 464be75..9925fa1 100644 --- a/pkg/controllers/loadbalancer/addresspool.go +++ b/pkg/controllers/loadbalancer/addresspool.go @@ -34,7 +34,7 @@ func (pool *AddressPool) containsCIDR(cidr string) bool { return false } -func (pool *AddressPool) appendIP(ip string) { +func (pool *AddressPool) AppendIP(ip string) { cidr := ip + "/32" if pool.containsCIDR(cidr) { diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go new file mode 100644 index 0000000..7dc2611 --- /dev/null +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -0,0 +1,258 @@ +package cilium + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" + "github.com/metal-stack/metal-lib/pkg/pointer" + "github.com/metal-stack/metal-lib/pkg/tag" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + + "github.com/metal-stack/metal-go/api/models" + + ciliumv2alpha1 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" +) + +const ( + ciliumNamespace = "kube-system" +) + +type CiliumConfig struct { + Peers []*Peer `json:"peers,omitempty" yaml:"peers,omitempty"` + AddressPools []*loadbalancer.AddressPool `json:"address-pools,omitempty" yaml:"address-pools,omitempty"` + namespace string +} + +func NewCiliumConfig() *CiliumConfig { + return &CiliumConfig{namespace: ciliumNamespace} +} + +func (cfg *CiliumConfig) Namespace() string { + return cfg.namespace +} + +func (cfg *CiliumConfig) CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { + err := cfg.computeAddressPools(ips, nws) + if err != nil { + return err + } + err = cfg.computePeers(nodes) + if err != nil { + return err + } + return nil +} + +func (cfg *CiliumConfig) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { + for _, ip := range ips { + if !nws.Has(*ip.Networkid) { + klog.Infof("skipping ip %q: not part of cluster networks", *ip.Ipaddress) + continue + } + net := *ip.Networkid + cfg.addIPToPool(net, *ip) + } + return nil +} + +func (cfg *CiliumConfig) computePeers(nodes []v1.Node) error { + cfg.Peers = []*Peer{} // we want an empty array of peers and not nil if there are no nodes + for _, n := range nodes { + labels := n.GetLabels() + asnString, ok := labels[tag.MachineNetworkPrimaryASN] + if !ok { + return fmt.Errorf("node %q misses label: %s", n.GetName(), tag.MachineNetworkPrimaryASN) + } + asn, err := strconv.ParseInt(asnString, 10, 64) + if err != nil { + return fmt.Errorf("unable to parse valid integer from asn annotation: %w", err) + } + + peer, err := newPeer(n, asn) + if err != nil { + klog.Warningf("skipping peer: %v", err) + continue + } + + cfg.Peers = append(cfg.Peers, peer) + } + return nil +} + +func (cfg *CiliumConfig) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { + for _, pool := range cfg.AddressPools { + if pool.Name == poolName { + return pool + } + } + + pool := loadbalancer.NewBGPAddressPool(poolName) + cfg.AddressPools = append(cfg.AddressPools, pool) + + return pool +} + +func (cfg *CiliumConfig) addIPToPool(network string, ip models.V1IPResponse) { + t := ip.Type + poolType := models.V1IPBaseTypeEphemeral + if t != nil && *t == models.V1IPBaseTypeStatic { + poolType = models.V1IPBaseTypeStatic + } + poolName := fmt.Sprintf("%s-%s", strings.ToLower(network), poolType) + pool := cfg.getOrCreateAddressPool(poolName) + pool.AppendIP(*ip.Ipaddress) +} + +func (cfg *CiliumConfig) ToYAML() (string, error) { + bb, err := yaml.Marshal(cfg) + if err != nil { + return "", err + } + return string(bb), nil +} + +func (cfg *CiliumConfig) WriteCRs(ctx context.Context, c client.Client) error { + err := cfg.writeCiliumBGPPeeringPolicies(ctx, c) + if err != nil { + return fmt.Errorf("failed to write ciliumbgppeeringpolicy resources %w", err) + } + + err = cfg.writeCiliumLoadBalancerIPPools(ctx, c) + if err != nil { + return fmt.Errorf("failed to write ciliumloadbalancerippool resources %w", err) + } + + return nil +} + +func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c client.Client) error { + existingPolicies := ciliumv2alpha1.CiliumBGPPeeringPolicyList{} + err := c.List(ctx, &existingPolicies, client.InNamespace(cfg.namespace)) + if err != nil { + return err + } + for _, existingPolicy := range existingPolicies.Items { + existingPolicy := existingPolicy + found := false + for _, peer := range cfg.Peers { + if fmt.Sprintf("%d", peer.Peer.ASN) == existingPolicy.Name { + found = true + break + } + } + if !found { + err := c.Delete(ctx, &existingPolicy) + if err != nil { + return err + } + } + } + + for _, peer := range cfg.Peers { + bgpPeeringPolicy := &ciliumv2alpha1.CiliumBGPPeeringPolicy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "cilium.io/v2alpha1", + Kind: "CiliumBGPPeeringPolicy", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%d", peer.Peer.ASN), + Namespace: cfg.namespace, + }, + } + res, err := controllerutil.CreateOrUpdate(ctx, c, bgpPeeringPolicy, func() error { + bgpPeeringPolicy.Spec = ciliumv2alpha1.CiliumBGPPeeringPolicySpec{ + NodeSelector: &peer.NodeSelector, + VirtualRouters: []ciliumv2alpha1.CiliumBGPVirtualRouter{ + { + LocalASN: peer.Peer.MyASN, + ExportPodCIDR: pointer.Pointer(true), + Neighbors: []ciliumv2alpha1.CiliumBGPNeighbor{ + { + PeerAddress: "127.0.0.1/32", + PeerASN: peer.Peer.ASN, + GracefulRestart: &ciliumv2alpha1.CiliumBGPNeighborGracefulRestart{Enabled: true}, + }, + }, + }, + }, + } + return nil + }) + if err != nil { + return err + } + if res != controllerutil.OperationResultNone { + klog.Infof("bgppeer: %v", res) + } + } + + return nil +} + +func (cfg *CiliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c client.Client) error { + existingPools := ciliumv2alpha1.CiliumLoadBalancerIPPoolList{} + err := c.List(ctx, &existingPools, client.InNamespace(cfg.namespace)) + if err != nil { + return err + } + for _, existingPool := range existingPools.Items { + existingPool := existingPool + found := false + for _, pool := range cfg.AddressPools { + if pool.Name == existingPool.Name { + found = true + break + } + } + if !found { + err := c.Delete(ctx, &existingPool) + if err != nil { + return err + } + } + } + + for _, pool := range cfg.AddressPools { + ipPool := &ciliumv2alpha1.CiliumLoadBalancerIPPool{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "cilium.io/v2alpha1", + Kind: "CiliumLoadBalancerIpPool", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: pool.Name, + Namespace: cfg.namespace, + }, + } + res, err := controllerutil.CreateOrUpdate(ctx, c, ipPool, func() error { + cidrs := make([]ciliumv2alpha1.CiliumLoadBalancerIPPoolIPBlock, 0) + for _, cidr := range pool.CIDRs { + ipPoolBlock := ciliumv2alpha1.CiliumLoadBalancerIPPoolIPBlock{ + Cidr: ciliumv2alpha1.IPv4orIPv6CIDR(cidr), + } + cidrs = append(cidrs, ipPoolBlock) + } + ipPool.Spec = ciliumv2alpha1.CiliumLoadBalancerIPPoolSpec{ + Cidrs: cidrs, + } + return nil + }) + if err != nil { + return err + } + if res != controllerutil.OperationResultNone { + klog.Infof("ipaddresspool: %v", res) + } + } + + return nil +} diff --git a/pkg/controllers/loadbalancer/cilium/config_test.go b/pkg/controllers/loadbalancer/cilium/config_test.go new file mode 100644 index 0000000..19e1796 --- /dev/null +++ b/pkg/controllers/loadbalancer/cilium/config_test.go @@ -0,0 +1,322 @@ +package cilium + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/metal-stack/metal-go/api/models" + "github.com/metal-stack/metal-lib/pkg/pointer" + "github.com/metal-stack/metal-lib/pkg/tag" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/yaml" +) + +var ( + testNetworks = sets.New( + "internet", + "shared-storage-network", + "mpls-network", + "dmz-network", + ) +) + +func TestCiliumConfig_CalculateConfig(t *testing.T) { + tests := []struct { + name string + nws sets.Set[string] + ips []*models.V1IPResponse + nodes []v1.Node + wantErr error + want map[string]interface{} + }{ + { + name: "one ip acquired, no nodes", + nws: testNetworks, + ips: []*models.V1IPResponse{ + { + Ipaddress: pointer.Pointer("84.1.1.1"), + Name: "acquired-before", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("ephemeral"), + }, + }, + nodes: []v1.Node{}, + wantErr: nil, + want: map[string]interface{}{ + "address-pools": []map[string]interface{}{ + { + "addresses": []string{ + "84.1.1.1/32", + }, + "auto-assign": false, + "name": "internet-ephemeral", + "protocol": "bgp", + }, + }, + }, + }, + { + name: "two ips acquired, no nodes", + nws: testNetworks, + ips: []*models.V1IPResponse{ + { + Ipaddress: pointer.Pointer("84.1.1.1"), + Name: "acquired-before", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("ephemeral"), + }, + { + Ipaddress: pointer.Pointer("84.1.1.2"), + Name: "acquired-before-2", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("ephemeral"), + }, + }, + nodes: []v1.Node{}, + wantErr: nil, + want: map[string]interface{}{ + "address-pools": []map[string]interface{}{ + { + "addresses": []string{ + "84.1.1.1/32", + "84.1.1.2/32", + }, + "auto-assign": false, + "name": "internet-ephemeral", + "protocol": "bgp", + }, + }, + }, + }, + { + name: "two ips acquired, one static ip, no nodes", + nws: testNetworks, + ips: []*models.V1IPResponse{ + { + Ipaddress: pointer.Pointer("84.1.1.1"), + Name: "acquired-before", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("ephemeral"), + }, + { + Ipaddress: pointer.Pointer("84.1.1.2"), + Name: "acquired-before-2", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("ephemeral"), + }, + { + Ipaddress: pointer.Pointer("84.1.1.3"), + Name: "static-ip", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("static"), + }, + }, + nodes: []v1.Node{}, + wantErr: nil, + want: map[string]interface{}{ + "address-pools": []map[string]interface{}{ + { + "addresses": []string{ + "84.1.1.1/32", + "84.1.1.2/32", + }, + "auto-assign": false, + "name": "internet-ephemeral", + "protocol": "bgp", + }, + { + "addresses": []string{ + "84.1.1.3/32", + }, + "auto-assign": false, + "name": "internet-static", + "protocol": "bgp", + }, + }, + }, + }, + + { + name: "connected to internet,storage,dmz and mpls, two ips acquired, one static ip, no nodes", + nws: testNetworks, + ips: []*models.V1IPResponse{ + { + Ipaddress: pointer.Pointer("84.1.1.1"), + Name: "acquired-before", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("ephemeral"), + }, + { + Ipaddress: pointer.Pointer("84.1.1.2"), + Name: "acquired-before-2", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("ephemeral"), + }, + { + Ipaddress: pointer.Pointer("84.1.1.3"), + Name: "static-ip", + Networkid: pointer.Pointer("internet"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("static"), + }, + { + Ipaddress: pointer.Pointer("10.131.44.2"), + Name: "static-ip", + Networkid: pointer.Pointer("shared-storage-network"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("static"), + }, + { + Ipaddress: pointer.Pointer("100.127.130.2"), + Name: "static-ip", + Networkid: pointer.Pointer("mpls-network"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("static"), + }, + { + Ipaddress: pointer.Pointer("100.127.130.3"), + Name: "ephemeral-mpls-ip", + Networkid: pointer.Pointer("mpls-network"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("ephemeral"), + }, + { + Ipaddress: pointer.Pointer("10.129.172.2"), + Name: "static-ip", + Networkid: pointer.Pointer("dmz-network"), + Projectid: pointer.Pointer("project-a"), + Tags: []string{ + fmt.Sprintf("%s=%s", tag.ClusterID, "this-cluster"), + }, + Type: pointer.Pointer("static"), + }, + }, + nodes: []v1.Node{}, + wantErr: nil, + want: map[string]interface{}{ + "address-pools": []map[string]interface{}{ + { + "addresses": []string{ + "84.1.1.1/32", + "84.1.1.2/32", + }, + "auto-assign": false, + "name": "internet-ephemeral", + "protocol": "bgp", + }, + { + "addresses": []string{ + "84.1.1.3/32", + }, + "auto-assign": false, + "name": "internet-static", + "protocol": "bgp", + }, + { + "addresses": []string{ + "10.131.44.2/32", + }, + "auto-assign": false, + "name": "shared-storage-network-static", + "protocol": "bgp", + }, + { + "addresses": []string{ + "100.127.130.2/32", + }, + "auto-assign": false, + "name": "mpls-network-static", + "protocol": "bgp", + }, + { + "addresses": []string{ + "100.127.130.3/32", + }, + "auto-assign": false, + "name": "mpls-network-ephemeral", + "protocol": "bgp", + }, + { + "addresses": []string{ + "10.129.172.2/32", + }, + "auto-assign": false, + "name": "dmz-network-static", + "protocol": "bgp", + }, + }, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + cfg := &CiliumConfig{} + + err := cfg.CalculateConfig(tt.ips, tt.nws, tt.nodes) + if diff := cmp.Diff(err, tt.wantErr); diff != "" { + t.Errorf("CiliumConfig.CalculateConfig() error = %v", diff) + return + } + + yaml, err := cfg.ToYAML() + require.NoError(t, err) + + if diff := cmp.Diff(yaml, mustYAML(tt.want)); diff != "" { + t.Errorf("CiliumConfig.CalculateConfig() = %v", diff) + } + }) + } +} + +func mustYAML(data interface{}) string { + res, _ := yaml.Marshal(data) + return string(res) +} diff --git a/pkg/controllers/loadbalancer/cilium/peer.go b/pkg/controllers/loadbalancer/cilium/peer.go new file mode 100644 index 0000000..f92556a --- /dev/null +++ b/pkg/controllers/loadbalancer/cilium/peer.go @@ -0,0 +1,38 @@ +package cilium + +import ( + slimv1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1" + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" + v1 "k8s.io/api/core/v1" +) + +type Peer struct { + Peer *loadbalancer.Peer `json:"peer,omitempty" yaml:"peer,omitempty"` + NodeSelector slimv1.LabelSelector `json:"node-selectors,omitempty" yaml:"node-selectors,omitempty"` +} + +func newPeer(node v1.Node, asn int64) (*Peer, error) { + hostname := node.GetName() + + matchExpression := slimv1.LabelSelectorRequirement{ + Key: "kubernetes.io/hostname", + Operator: "In", + Values: []string{ + hostname, + }, + } + + peer, err := loadbalancer.NewPeer(node, asn) + if err != nil { + return nil, err + } + + return &Peer{ + Peer: peer, + NodeSelector: slimv1.LabelSelector{ + MatchExpressions: []slimv1.LabelSelectorRequirement{ + matchExpression, + }, + }, + }, nil +} diff --git a/pkg/controllers/loadbalancer/config.go b/pkg/controllers/loadbalancer/config.go new file mode 100644 index 0000000..e4f95d3 --- /dev/null +++ b/pkg/controllers/loadbalancer/config.go @@ -0,0 +1,16 @@ +package loadbalancer + +import ( + "context" + + "github.com/metal-stack/metal-go/api/models" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Config interface { + Namespace() string + CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error + WriteCRs(ctx context.Context, c client.Client) error +} diff --git a/pkg/controllers/loadbalancer/loadbalancer.go b/pkg/controllers/loadbalancer/loadbalancer.go index e2828cd..6e28575 100644 --- a/pkg/controllers/loadbalancer/loadbalancer.go +++ b/pkg/controllers/loadbalancer/loadbalancer.go @@ -29,6 +29,7 @@ import ( ) type LoadBalancerController struct { + Config MetalService *metal.MetalService partitionID string projectID string @@ -43,8 +44,9 @@ type LoadBalancerController struct { } // New returns a new load balancer controller that satisfies the kubernetes cloud provider load balancer interface -func New(partitionID, projectID, clusterID, defaultExternalNetworkID string, additionalNetworks []string) *LoadBalancerController { +func New(partitionID, projectID, clusterID, defaultExternalNetworkID string, additionalNetworks []string, config Config) *LoadBalancerController { return &LoadBalancerController{ + Config: config, partitionID: partitionID, projectID: projectID, clusterID: clusterID, @@ -340,8 +342,7 @@ func (l *LoadBalancerController) updateLoadBalancerConfig(ctx context.Context, n return fmt.Errorf("could not find ips of this project's cluster: %w", err) } - config := newMetalLBConfig() - err = config.CalculateConfig(ips, l.additionalNetworks, nodes) + err = l.Config.CalculateConfig(ips, l.additionalNetworks, nodes) if err != nil { return err } @@ -349,13 +350,13 @@ func (l *LoadBalancerController) updateLoadBalancerConfig(ctx context.Context, n // TODO: in a future release this can be removed err = l.K8sClient.Delete(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ Name: "config", - Namespace: metallbNamespace, + Namespace: l.Config.Namespace(), }}) if client.IgnoreNotFound(err) != nil { return fmt.Errorf("unable to cleanup deprecated metallb configmap: %w", err) } - err = config.WriteCRs(ctx, l.K8sClient) + err = l.Config.WriteCRs(ctx, l.K8sClient) if err != nil { return err } diff --git a/pkg/controllers/loadbalancer/metallb.go b/pkg/controllers/loadbalancer/metallb/config.go similarity index 85% rename from pkg/controllers/loadbalancer/metallb.go rename to pkg/controllers/loadbalancer/metallb/config.go index 4961da7..7c711ac 100644 --- a/pkg/controllers/loadbalancer/metallb.go +++ b/pkg/controllers/loadbalancer/metallb/config.go @@ -1,4 +1,4 @@ -package loadbalancer +package metallb import ( "context" @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" "github.com/metal-stack/metal-lib/pkg/tag" v1 "k8s.io/api/core/v1" @@ -30,12 +31,17 @@ const ( // MetalLBConfig is a struct containing a config for metallb type MetalLBConfig struct { - Peers []*Peer `json:"peers,omitempty" yaml:"peers,omitempty"` - AddressPools []*AddressPool `json:"address-pools,omitempty" yaml:"address-pools,omitempty"` + Peers []*Peer `json:"peers,omitempty" yaml:"peers,omitempty"` + AddressPools []*loadbalancer.AddressPool `json:"address-pools,omitempty" yaml:"address-pools,omitempty"` + namespace string } -func newMetalLBConfig() *MetalLBConfig { - return &MetalLBConfig{} +func NewMetalLBConfig() *MetalLBConfig { + return &MetalLBConfig{namespace: metallbNamespace} +} + +func (cfg *MetalLBConfig) Namespace() string { + return cfg.namespace } // CalculateConfig computes the metallb config from given parameter input. @@ -89,14 +95,14 @@ func (cfg *MetalLBConfig) computePeers(nodes []v1.Node) error { // getOrCreateAddressPool returns the address pool of the given network. // It will be created if it does not exist yet. -func (cfg *MetalLBConfig) getOrCreateAddressPool(poolName string) *AddressPool { +func (cfg *MetalLBConfig) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { for _, pool := range cfg.AddressPools { if pool.Name == poolName { return pool } } - pool := NewBGPAddressPool(poolName) + pool := loadbalancer.NewBGPAddressPool(poolName) cfg.AddressPools = append(cfg.AddressPools, pool) return pool @@ -111,7 +117,7 @@ func (cfg *MetalLBConfig) addIPToPool(network string, ip models.V1IPResponse) { } poolName := fmt.Sprintf("%s-%s", strings.ToLower(network), poolType) pool := cfg.getOrCreateAddressPool(poolName) - pool.appendIP(*ip.Ipaddress) + pool.AppendIP(*ip.Ipaddress) } // ToYAML returns this config in YAML format. @@ -128,7 +134,7 @@ func (cfg *MetalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { // BGPPeers bgpPeerList := metallbv1beta2.BGPPeerList{} - err := c.List(ctx, &bgpPeerList, client.InNamespace(metallbNamespace)) + err := c.List(ctx, &bgpPeerList, client.InNamespace(cfg.namespace)) if err != nil { return err } @@ -136,7 +142,7 @@ func (cfg *MetalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { existingPeer := existingPeer found := false for _, peer := range cfg.Peers { - if fmt.Sprintf("peer-%d", peer.ASN) == existingPeer.Name { + if fmt.Sprintf("peer-%d", peer.Peer.ASN) == existingPeer.Name { found = true break } @@ -156,17 +162,17 @@ func (cfg *MetalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { Kind: "BGPPeer", }, ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("peer-%d", peer.ASN), - Namespace: metallbNamespace, + Name: fmt.Sprintf("peer-%d", peer.Peer.ASN), + Namespace: cfg.namespace, }, } res, err := controllerutil.CreateOrUpdate(ctx, c, bgpPeer, func() error { bgpPeer.Spec = metallbv1beta2.BGPPeerSpec{ - MyASN: uint32(peer.MyASN), - ASN: uint32(peer.ASN), + MyASN: uint32(peer.Peer.MyASN), + ASN: uint32(peer.Peer.ASN), HoldTime: metav1.Duration{Duration: 90 * time.Second}, KeepaliveTime: metav1.Duration{Duration: 0 * time.Second}, - Address: peer.Address, + Address: peer.Peer.Address, NodeSelectors: peer.NodeSelectors, } return nil @@ -181,7 +187,7 @@ func (cfg *MetalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { // IPAddressPools addressPoolList := metallbv1beta1.IPAddressPoolList{} - err = c.List(ctx, &addressPoolList, client.InNamespace(metallbNamespace)) + err = c.List(ctx, &addressPoolList, client.InNamespace(cfg.namespace)) if err != nil { return err } @@ -210,7 +216,7 @@ func (cfg *MetalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { }, ObjectMeta: metav1.ObjectMeta{ Name: pool.Name, - Namespace: metallbNamespace, + Namespace: cfg.namespace, }, } res, err := controllerutil.CreateOrUpdate(ctx, c, ipAddressPool, func() error { @@ -231,7 +237,7 @@ func (cfg *MetalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { // BGPAdvertisements for _, pool := range cfg.AddressPools { bgpAdvertisementList := metallbv1beta1.BGPAdvertisementList{} - err = c.List(ctx, &bgpAdvertisementList, client.InNamespace(metallbNamespace)) + err = c.List(ctx, &bgpAdvertisementList, client.InNamespace(cfg.namespace)) if err != nil { return err } @@ -259,7 +265,7 @@ func (cfg *MetalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { }, ObjectMeta: metav1.ObjectMeta{ Name: pool.Name, - Namespace: metallbNamespace, + Namespace: cfg.namespace, }, } res, err := controllerutil.CreateOrUpdate(ctx, c, bgpAdvertisement, func() error { diff --git a/pkg/controllers/loadbalancer/metallb_test.go b/pkg/controllers/loadbalancer/metallb/config_test.go similarity index 99% rename from pkg/controllers/loadbalancer/metallb_test.go rename to pkg/controllers/loadbalancer/metallb/config_test.go index e649c03..8471688 100644 --- a/pkg/controllers/loadbalancer/metallb_test.go +++ b/pkg/controllers/loadbalancer/metallb/config_test.go @@ -1,4 +1,4 @@ -package loadbalancer +package metallb import ( "fmt" diff --git a/pkg/controllers/loadbalancer/metallb/peer.go b/pkg/controllers/loadbalancer/metallb/peer.go new file mode 100644 index 0000000..6902d15 --- /dev/null +++ b/pkg/controllers/loadbalancer/metallb/peer.go @@ -0,0 +1,51 @@ +package metallb + +import ( + "fmt" + + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Peer struct { + Peer *loadbalancer.Peer `json:"peer,omitempty" yaml:"peer,omitempty"` + NodeSelectors []metav1.LabelSelector `json:"node-selectors,omitempty" yaml:"node-selectors,omitempty"` +} + +func newPeer(node v1.Node, asn int64) (*Peer, error) { + hostname := node.GetName() + + matchExpression := metav1.LabelSelectorRequirement{ + Key: "kubernetes.io/hostname", + Operator: "In", + Values: []string{ + hostname, + }, + } + + peer, err := loadbalancer.NewPeer(node, asn) + if err != nil { + return nil, err + } + + return &Peer{ + Peer: peer, + NodeSelectors: []metav1.LabelSelector{ + { + MatchExpressions: []metav1.LabelSelectorRequirement{ + matchExpression, + }, + }, + }, + }, nil +} + +func NodeAddress(node v1.Node) (string, error) { + for _, a := range node.Status.Addresses { + if a.Type == v1.NodeInternalIP { + return a.Address, nil + } + } + return "", fmt.Errorf("unable to determine node address") +} diff --git a/pkg/controllers/loadbalancer/peer.go b/pkg/controllers/loadbalancer/peer.go index 19628bb..36bbad7 100644 --- a/pkg/controllers/loadbalancer/peer.go +++ b/pkg/controllers/loadbalancer/peer.go @@ -4,27 +4,15 @@ import ( "fmt" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type Peer struct { - MyASN int64 `json:"my-asn" yaml:"my-asn"` - ASN int64 `json:"peer-asn" yaml:"peer-asn"` - Address string `json:"peer-address" yaml:"peer-address"` - NodeSelectors []metav1.LabelSelector `json:"node-selectors,omitempty" yaml:"node-selectors,omitempty"` + MyASN int64 `json:"my-asn" yaml:"my-asn"` + ASN int64 `json:"peer-asn" yaml:"peer-asn"` + Address string `json:"peer-address" yaml:"peer-address"` } -func newPeer(node v1.Node, asn int64) (*Peer, error) { - hostname := node.GetName() - - matchExpression := metav1.LabelSelectorRequirement{ - Key: "kubernetes.io/hostname", - Operator: "In", - Values: []string{ - hostname, - }, - } - +func NewPeer(node v1.Node, asn int64) (*Peer, error) { address, err := NodeAddress(node) if err != nil { return nil, err @@ -33,13 +21,6 @@ func newPeer(node v1.Node, asn int64) (*Peer, error) { MyASN: asn, ASN: asn, Address: address, - NodeSelectors: []metav1.LabelSelector{ - { - MatchExpressions: []metav1.LabelSelectorRequirement{ - matchExpression, - }, - }, - }, }, nil } diff --git a/pkg/resources/constants/constants.go b/pkg/resources/constants/constants.go index b817569..32866a0 100644 --- a/pkg/resources/constants/constants.go +++ b/pkg/resources/constants/constants.go @@ -19,4 +19,6 @@ const ( MetalLBSpecificAddressPool = "metallb.universe.tf/address-pool" IPPrefix = "metallb-" + + Loadbalancer = "LOADBALANCER" ) From ef78441b3cdce1ff5cdd484a282f0dee87a67ea3 Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Thu, 20 Jun 2024 13:35:13 +0200 Subject: [PATCH 02/17] use metallb as default --- metal/cloud.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/cloud.go b/metal/cloud.go index 7b25564..9a18ff0 100644 --- a/metal/cloud.go +++ b/metal/cloud.go @@ -103,7 +103,7 @@ func NewCloud(_ io.Reader) (cloudprovider.Interface, error) { case "cilium": config = cilium.NewCiliumConfig() default: - return nil, fmt.Errorf("loadbalancer type must be on of 'metallb' or 'cilium'") + config = metallb.NewMetalLBConfig() } loadBalancerController := loadbalancer.New(partitionID, projectID, clusterID, defaultExternalNetworkID, additionalNetworks, config) From 26341c9637754f3752e00b94bcdfeb6d77360a1a Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Mon, 24 Jun 2024 15:25:18 +0200 Subject: [PATCH 03/17] remove cilium namespace --- metal/cloud.go | 2 +- pkg/controllers/loadbalancer/cilium/config.go | 43 ++++++++----------- .../loadbalancer/cilium/config_test.go | 2 +- pkg/controllers/loadbalancer/config.go | 2 +- pkg/controllers/loadbalancer/loadbalancer.go | 12 +++--- 5 files changed, 27 insertions(+), 34 deletions(-) diff --git a/metal/cloud.go b/metal/cloud.go index 9a18ff0..e69d335 100644 --- a/metal/cloud.go +++ b/metal/cloud.go @@ -96,7 +96,7 @@ func NewCloud(_ io.Reader) (cloudprovider.Interface, error) { instancesController := instances.New(defaultExternalNetworkID) zonesController := zones.New() - var config loadbalancer.Config + var config loadbalancer.LoadBalancer switch loadbalancerType { case "metallb": config = metallb.NewMetalLBConfig() diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index 7dc2611..7d0a383 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -23,25 +23,20 @@ import ( "sigs.k8s.io/yaml" ) -const ( - ciliumNamespace = "kube-system" -) - -type CiliumConfig struct { +type Cilium struct { Peers []*Peer `json:"peers,omitempty" yaml:"peers,omitempty"` AddressPools []*loadbalancer.AddressPool `json:"address-pools,omitempty" yaml:"address-pools,omitempty"` - namespace string } -func NewCiliumConfig() *CiliumConfig { - return &CiliumConfig{namespace: ciliumNamespace} +func NewCiliumConfig() *Cilium { + return &Cilium{} } -func (cfg *CiliumConfig) Namespace() string { - return cfg.namespace +func (cfg *Cilium) Namespace() string { + return "" } -func (cfg *CiliumConfig) CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { +func (cfg *Cilium) CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { err := cfg.computeAddressPools(ips, nws) if err != nil { return err @@ -53,7 +48,7 @@ func (cfg *CiliumConfig) CalculateConfig(ips []*models.V1IPResponse, nws sets.Se return nil } -func (cfg *CiliumConfig) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { +func (cfg *Cilium) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { for _, ip := range ips { if !nws.Has(*ip.Networkid) { klog.Infof("skipping ip %q: not part of cluster networks", *ip.Ipaddress) @@ -65,7 +60,7 @@ func (cfg *CiliumConfig) computeAddressPools(ips []*models.V1IPResponse, nws set return nil } -func (cfg *CiliumConfig) computePeers(nodes []v1.Node) error { +func (cfg *Cilium) computePeers(nodes []v1.Node) error { cfg.Peers = []*Peer{} // we want an empty array of peers and not nil if there are no nodes for _, n := range nodes { labels := n.GetLabels() @@ -89,7 +84,7 @@ func (cfg *CiliumConfig) computePeers(nodes []v1.Node) error { return nil } -func (cfg *CiliumConfig) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { +func (cfg *Cilium) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { for _, pool := range cfg.AddressPools { if pool.Name == poolName { return pool @@ -102,7 +97,7 @@ func (cfg *CiliumConfig) getOrCreateAddressPool(poolName string) *loadbalancer.A return pool } -func (cfg *CiliumConfig) addIPToPool(network string, ip models.V1IPResponse) { +func (cfg *Cilium) addIPToPool(network string, ip models.V1IPResponse) { t := ip.Type poolType := models.V1IPBaseTypeEphemeral if t != nil && *t == models.V1IPBaseTypeStatic { @@ -113,7 +108,7 @@ func (cfg *CiliumConfig) addIPToPool(network string, ip models.V1IPResponse) { pool.AppendIP(*ip.Ipaddress) } -func (cfg *CiliumConfig) ToYAML() (string, error) { +func (cfg *Cilium) ToYAML() (string, error) { bb, err := yaml.Marshal(cfg) if err != nil { return "", err @@ -121,7 +116,7 @@ func (cfg *CiliumConfig) ToYAML() (string, error) { return string(bb), nil } -func (cfg *CiliumConfig) WriteCRs(ctx context.Context, c client.Client) error { +func (cfg *Cilium) WriteCRs(ctx context.Context, c client.Client) error { err := cfg.writeCiliumBGPPeeringPolicies(ctx, c) if err != nil { return fmt.Errorf("failed to write ciliumbgppeeringpolicy resources %w", err) @@ -135,9 +130,9 @@ func (cfg *CiliumConfig) WriteCRs(ctx context.Context, c client.Client) error { return nil } -func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c client.Client) error { +func (cfg *Cilium) writeCiliumBGPPeeringPolicies(ctx context.Context, c client.Client) error { existingPolicies := ciliumv2alpha1.CiliumBGPPeeringPolicyList{} - err := c.List(ctx, &existingPolicies, client.InNamespace(cfg.namespace)) + err := c.List(ctx, &existingPolicies) if err != nil { return err } @@ -165,8 +160,7 @@ func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c cl Kind: "CiliumBGPPeeringPolicy", }, ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%d", peer.Peer.ASN), - Namespace: cfg.namespace, + Name: fmt.Sprintf("%d", peer.Peer.ASN), }, } res, err := controllerutil.CreateOrUpdate(ctx, c, bgpPeeringPolicy, func() error { @@ -199,9 +193,9 @@ func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c cl return nil } -func (cfg *CiliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c client.Client) error { +func (cfg *Cilium) writeCiliumLoadBalancerIPPools(ctx context.Context, c client.Client) error { existingPools := ciliumv2alpha1.CiliumLoadBalancerIPPoolList{} - err := c.List(ctx, &existingPools, client.InNamespace(cfg.namespace)) + err := c.List(ctx, &existingPools) if err != nil { return err } @@ -229,8 +223,7 @@ func (cfg *CiliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c c Kind: "CiliumLoadBalancerIpPool", }, ObjectMeta: metav1.ObjectMeta{ - Name: pool.Name, - Namespace: cfg.namespace, + Name: pool.Name, }, } res, err := controllerutil.CreateOrUpdate(ctx, c, ipPool, func() error { diff --git a/pkg/controllers/loadbalancer/cilium/config_test.go b/pkg/controllers/loadbalancer/cilium/config_test.go index 19e1796..1ae6ecc 100644 --- a/pkg/controllers/loadbalancer/cilium/config_test.go +++ b/pkg/controllers/loadbalancer/cilium/config_test.go @@ -298,7 +298,7 @@ func TestCiliumConfig_CalculateConfig(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - cfg := &CiliumConfig{} + cfg := &Cilium{} err := cfg.CalculateConfig(tt.ips, tt.nws, tt.nodes) if diff := cmp.Diff(err, tt.wantErr); diff != "" { diff --git a/pkg/controllers/loadbalancer/config.go b/pkg/controllers/loadbalancer/config.go index e4f95d3..e181460 100644 --- a/pkg/controllers/loadbalancer/config.go +++ b/pkg/controllers/loadbalancer/config.go @@ -9,7 +9,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type Config interface { +type LoadBalancer interface { Namespace() string CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error WriteCRs(ctx context.Context, c client.Client) error diff --git a/pkg/controllers/loadbalancer/loadbalancer.go b/pkg/controllers/loadbalancer/loadbalancer.go index 6e28575..6c09592 100644 --- a/pkg/controllers/loadbalancer/loadbalancer.go +++ b/pkg/controllers/loadbalancer/loadbalancer.go @@ -29,7 +29,7 @@ import ( ) type LoadBalancerController struct { - Config + LoadBalancer MetalService *metal.MetalService partitionID string projectID string @@ -44,9 +44,9 @@ type LoadBalancerController struct { } // New returns a new load balancer controller that satisfies the kubernetes cloud provider load balancer interface -func New(partitionID, projectID, clusterID, defaultExternalNetworkID string, additionalNetworks []string, config Config) *LoadBalancerController { +func New(partitionID, projectID, clusterID, defaultExternalNetworkID string, additionalNetworks []string, config LoadBalancer) *LoadBalancerController { return &LoadBalancerController{ - Config: config, + LoadBalancer: config, partitionID: partitionID, projectID: projectID, clusterID: clusterID, @@ -342,7 +342,7 @@ func (l *LoadBalancerController) updateLoadBalancerConfig(ctx context.Context, n return fmt.Errorf("could not find ips of this project's cluster: %w", err) } - err = l.Config.CalculateConfig(ips, l.additionalNetworks, nodes) + err = l.LoadBalancer.CalculateConfig(ips, l.additionalNetworks, nodes) if err != nil { return err } @@ -350,13 +350,13 @@ func (l *LoadBalancerController) updateLoadBalancerConfig(ctx context.Context, n // TODO: in a future release this can be removed err = l.K8sClient.Delete(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ Name: "config", - Namespace: l.Config.Namespace(), + Namespace: l.LoadBalancer.Namespace(), }}) if client.IgnoreNotFound(err) != nil { return fmt.Errorf("unable to cleanup deprecated metallb configmap: %w", err) } - err = l.Config.WriteCRs(ctx, l.K8sClient) + err = l.LoadBalancer.WriteCRs(ctx, l.K8sClient) if err != nil { return err } From cf15988fda7b9627bf7b071f7e37865d9b198c9a Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Tue, 25 Jun 2024 08:36:08 +0200 Subject: [PATCH 04/17] add cilium to scheme --- metal/cloud.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metal/cloud.go b/metal/cloud.go index e69d335..7fdcdc3 100644 --- a/metal/cloud.go +++ b/metal/cloud.go @@ -23,6 +23,7 @@ import ( "k8s.io/kubectl/pkg/scheme" "sigs.k8s.io/controller-runtime/pkg/client" + ciliumv2alpha1 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1" metallbv1beta1 "go.universe.tf/metallb/api/v1beta1" metallbv1beta2 "go.universe.tf/metallb/api/v1beta2" ) @@ -137,6 +138,10 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, if err != nil { klog.Fatalf("unable to add metallb v1beta2 to scheme: %v", err) } + err = ciliumv2alpha1.AddToScheme(scheme.Scheme) + if err != nil { + klog.Fatalf("unable to add cilium v2alpha1 to scheme: %v", err) + } k8sClient, err := client.New(k8sRestConfig, client.Options{Scheme: scheme.Scheme}) if err != nil { klog.Fatalf("unable to create k8s client: %v", err) From 53e9ed2fce625ddb39ee67f1c796e19a77f403c0 Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Tue, 25 Jun 2024 08:36:15 +0200 Subject: [PATCH 05/17] config renaming --- metal/cloud.go | 2 +- pkg/controllers/loadbalancer/cilium/config.go | 26 +++++++++---------- .../loadbalancer/cilium/config_test.go | 2 +- pkg/controllers/loadbalancer/config.go | 2 +- pkg/controllers/loadbalancer/loadbalancer.go | 12 ++++----- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/metal/cloud.go b/metal/cloud.go index 7fdcdc3..ddd43f7 100644 --- a/metal/cloud.go +++ b/metal/cloud.go @@ -97,7 +97,7 @@ func NewCloud(_ io.Reader) (cloudprovider.Interface, error) { instancesController := instances.New(defaultExternalNetworkID) zonesController := zones.New() - var config loadbalancer.LoadBalancer + var config loadbalancer.LoadBalancerConfig switch loadbalancerType { case "metallb": config = metallb.NewMetalLBConfig() diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index 7d0a383..2c27937 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -23,20 +23,20 @@ import ( "sigs.k8s.io/yaml" ) -type Cilium struct { +type CiliumConfig struct { Peers []*Peer `json:"peers,omitempty" yaml:"peers,omitempty"` AddressPools []*loadbalancer.AddressPool `json:"address-pools,omitempty" yaml:"address-pools,omitempty"` } -func NewCiliumConfig() *Cilium { - return &Cilium{} +func NewCiliumConfig() *CiliumConfig { + return &CiliumConfig{} } -func (cfg *Cilium) Namespace() string { +func (cfg *CiliumConfig) Namespace() string { return "" } -func (cfg *Cilium) CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { +func (cfg *CiliumConfig) CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { err := cfg.computeAddressPools(ips, nws) if err != nil { return err @@ -48,7 +48,7 @@ func (cfg *Cilium) CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[stri return nil } -func (cfg *Cilium) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { +func (cfg *CiliumConfig) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { for _, ip := range ips { if !nws.Has(*ip.Networkid) { klog.Infof("skipping ip %q: not part of cluster networks", *ip.Ipaddress) @@ -60,7 +60,7 @@ func (cfg *Cilium) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[ return nil } -func (cfg *Cilium) computePeers(nodes []v1.Node) error { +func (cfg *CiliumConfig) computePeers(nodes []v1.Node) error { cfg.Peers = []*Peer{} // we want an empty array of peers and not nil if there are no nodes for _, n := range nodes { labels := n.GetLabels() @@ -84,7 +84,7 @@ func (cfg *Cilium) computePeers(nodes []v1.Node) error { return nil } -func (cfg *Cilium) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { +func (cfg *CiliumConfig) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { for _, pool := range cfg.AddressPools { if pool.Name == poolName { return pool @@ -97,7 +97,7 @@ func (cfg *Cilium) getOrCreateAddressPool(poolName string) *loadbalancer.Address return pool } -func (cfg *Cilium) addIPToPool(network string, ip models.V1IPResponse) { +func (cfg *CiliumConfig) addIPToPool(network string, ip models.V1IPResponse) { t := ip.Type poolType := models.V1IPBaseTypeEphemeral if t != nil && *t == models.V1IPBaseTypeStatic { @@ -108,7 +108,7 @@ func (cfg *Cilium) addIPToPool(network string, ip models.V1IPResponse) { pool.AppendIP(*ip.Ipaddress) } -func (cfg *Cilium) ToYAML() (string, error) { +func (cfg *CiliumConfig) ToYAML() (string, error) { bb, err := yaml.Marshal(cfg) if err != nil { return "", err @@ -116,7 +116,7 @@ func (cfg *Cilium) ToYAML() (string, error) { return string(bb), nil } -func (cfg *Cilium) WriteCRs(ctx context.Context, c client.Client) error { +func (cfg *CiliumConfig) WriteCRs(ctx context.Context, c client.Client) error { err := cfg.writeCiliumBGPPeeringPolicies(ctx, c) if err != nil { return fmt.Errorf("failed to write ciliumbgppeeringpolicy resources %w", err) @@ -130,7 +130,7 @@ func (cfg *Cilium) WriteCRs(ctx context.Context, c client.Client) error { return nil } -func (cfg *Cilium) writeCiliumBGPPeeringPolicies(ctx context.Context, c client.Client) error { +func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c client.Client) error { existingPolicies := ciliumv2alpha1.CiliumBGPPeeringPolicyList{} err := c.List(ctx, &existingPolicies) if err != nil { @@ -193,7 +193,7 @@ func (cfg *Cilium) writeCiliumBGPPeeringPolicies(ctx context.Context, c client.C return nil } -func (cfg *Cilium) writeCiliumLoadBalancerIPPools(ctx context.Context, c client.Client) error { +func (cfg *CiliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c client.Client) error { existingPools := ciliumv2alpha1.CiliumLoadBalancerIPPoolList{} err := c.List(ctx, &existingPools) if err != nil { diff --git a/pkg/controllers/loadbalancer/cilium/config_test.go b/pkg/controllers/loadbalancer/cilium/config_test.go index 1ae6ecc..19e1796 100644 --- a/pkg/controllers/loadbalancer/cilium/config_test.go +++ b/pkg/controllers/loadbalancer/cilium/config_test.go @@ -298,7 +298,7 @@ func TestCiliumConfig_CalculateConfig(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - cfg := &Cilium{} + cfg := &CiliumConfig{} err := cfg.CalculateConfig(tt.ips, tt.nws, tt.nodes) if diff := cmp.Diff(err, tt.wantErr); diff != "" { diff --git a/pkg/controllers/loadbalancer/config.go b/pkg/controllers/loadbalancer/config.go index e181460..4c79a42 100644 --- a/pkg/controllers/loadbalancer/config.go +++ b/pkg/controllers/loadbalancer/config.go @@ -9,7 +9,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type LoadBalancer interface { +type LoadBalancerConfig interface { Namespace() string CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error WriteCRs(ctx context.Context, c client.Client) error diff --git a/pkg/controllers/loadbalancer/loadbalancer.go b/pkg/controllers/loadbalancer/loadbalancer.go index 6c09592..5831d43 100644 --- a/pkg/controllers/loadbalancer/loadbalancer.go +++ b/pkg/controllers/loadbalancer/loadbalancer.go @@ -29,7 +29,7 @@ import ( ) type LoadBalancerController struct { - LoadBalancer + LoadBalancerConfig MetalService *metal.MetalService partitionID string projectID string @@ -44,9 +44,9 @@ type LoadBalancerController struct { } // New returns a new load balancer controller that satisfies the kubernetes cloud provider load balancer interface -func New(partitionID, projectID, clusterID, defaultExternalNetworkID string, additionalNetworks []string, config LoadBalancer) *LoadBalancerController { +func New(partitionID, projectID, clusterID, defaultExternalNetworkID string, additionalNetworks []string, config LoadBalancerConfig) *LoadBalancerController { return &LoadBalancerController{ - LoadBalancer: config, + LoadBalancerConfig: config, partitionID: partitionID, projectID: projectID, clusterID: clusterID, @@ -342,7 +342,7 @@ func (l *LoadBalancerController) updateLoadBalancerConfig(ctx context.Context, n return fmt.Errorf("could not find ips of this project's cluster: %w", err) } - err = l.LoadBalancer.CalculateConfig(ips, l.additionalNetworks, nodes) + err = l.LoadBalancerConfig.CalculateConfig(ips, l.additionalNetworks, nodes) if err != nil { return err } @@ -350,13 +350,13 @@ func (l *LoadBalancerController) updateLoadBalancerConfig(ctx context.Context, n // TODO: in a future release this can be removed err = l.K8sClient.Delete(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ Name: "config", - Namespace: l.LoadBalancer.Namespace(), + Namespace: l.LoadBalancerConfig.Namespace(), }}) if client.IgnoreNotFound(err) != nil { return fmt.Errorf("unable to cleanup deprecated metallb configmap: %w", err) } - err = l.LoadBalancer.WriteCRs(ctx, l.K8sClient) + err = l.LoadBalancerConfig.WriteCRs(ctx, l.K8sClient) if err != nil { return err } From 1776c0abcaa5d637726c0329f53b529c6ef63afe Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Tue, 25 Jun 2024 09:25:13 +0200 Subject: [PATCH 06/17] rename metallb to load balancer --- pkg/controllers/housekeeping/housekeeper.go | 34 ++++++++++---------- pkg/controllers/housekeeping/loadbalancer.go | 34 ++++++++++++++++++++ pkg/controllers/housekeeping/metallb.go | 34 -------------------- pkg/controllers/loadbalancer/loadbalancer.go | 10 +++--- 4 files changed, 56 insertions(+), 56 deletions(-) create mode 100644 pkg/controllers/housekeeping/loadbalancer.go delete mode 100644 pkg/controllers/housekeeping/metallb.go diff --git a/pkg/controllers/housekeeping/housekeeper.go b/pkg/controllers/housekeeping/housekeeper.go index 0a1a0c4..6d35c11 100644 --- a/pkg/controllers/housekeeping/housekeeper.go +++ b/pkg/controllers/housekeeping/housekeeper.go @@ -16,19 +16,19 @@ import ( "github.com/metal-stack/metal-ccm/pkg/resources/metal" ) -// Housekeeper periodically updates nodes, loadbalancers and metallb +// Housekeeper periodically updates nodes and loadbalancers type Housekeeper struct { - client metalgo.Client - stop <-chan struct{} - k8sClient clientset.Interface - ticker *tickerSyncer - lbController *loadbalancer.LoadBalancerController - lastTagSync time.Time - lastMetalLBConfigSync time.Time - metalAPIErrors int32 - ms *metal.MetalService - sshPublicKey string - clusterID string + client metalgo.Client + stop <-chan struct{} + k8sClient clientset.Interface + ticker *tickerSyncer + lbController *loadbalancer.LoadBalancerController + lastTagSync time.Time + lastLoadBalancerConfigSync time.Time + metalAPIErrors int32 + ms *metal.MetalService + sshPublicKey string + clusterID string } // New returns a new house keeper @@ -48,7 +48,7 @@ func New(metalClient metalgo.Client, stop <-chan struct{}, lbController *loadbal // Run runs the housekeeper... func (h *Housekeeper) Run() { h.startTagSynching() - h.startMetalLBConfigSynching() + h.startLoadBalancerConfigSynching() h.startSSHKeysSynching() h.watchNodes() h.runHealthCheck() @@ -85,20 +85,20 @@ func (h *Housekeeper) watchNodes() { return } if oldTunnelAddress == newTunnelAddress { - // node was not modified and ip address has not changed, not updating metallb config + // node was not modified and ip address has not changed, not updating load balancer config return } - klog.Info("node was modified and ip address has changed, updating metallb config") + klog.Info("node was modified and ip address has changed, updating load balancer config") nodes, err := kubernetes.GetNodes(h.k8sClient) if err != nil { klog.Errorf("error listing nodes: %v", err) return } - err = h.lbController.UpdateMetalLBConfig(context.Background(), nodes) + err = h.lbController.UpdateLoadBalancerConfig(context.Background(), nodes) if err != nil { - klog.Errorf("error updating metallb config: %v", err) + klog.Errorf("error updating load balancer config: %v", err) } }, }, diff --git a/pkg/controllers/housekeeping/loadbalancer.go b/pkg/controllers/housekeeping/loadbalancer.go new file mode 100644 index 0000000..acb8fd3 --- /dev/null +++ b/pkg/controllers/housekeeping/loadbalancer.go @@ -0,0 +1,34 @@ +package housekeeping + +import ( + "context" + "fmt" + "time" + + "github.com/metal-stack/metal-ccm/pkg/resources/kubernetes" +) + +const ( + syncLoadBalancerInterval = 1 * time.Minute + syncLoadBalancerMinimalInternval = 5 * time.Second +) + +func (h *Housekeeper) startLoadBalancerConfigSynching() { + go h.ticker.Start("load balancer syncher", syncLoadBalancerInterval, h.stop, h.updateLoadBalancerConfig) +} + +func (h *Housekeeper) updateLoadBalancerConfig() error { + if time.Since(h.lastLoadBalancerConfigSync) < syncLoadBalancerMinimalInternval { + return nil + } + nodes, err := kubernetes.GetNodes(h.k8sClient) + if err != nil { + return fmt.Errorf("error listing nodes: %w", err) + } + err = h.lbController.UpdateLoadBalancerConfig(context.Background(), nodes) + if err != nil { + return fmt.Errorf("error updating load balancer config: %w", err) + } + h.lastLoadBalancerConfigSync = time.Now() + return nil +} diff --git a/pkg/controllers/housekeeping/metallb.go b/pkg/controllers/housekeeping/metallb.go deleted file mode 100644 index a7ddcd0..0000000 --- a/pkg/controllers/housekeeping/metallb.go +++ /dev/null @@ -1,34 +0,0 @@ -package housekeeping - -import ( - "context" - "fmt" - "time" - - "github.com/metal-stack/metal-ccm/pkg/resources/kubernetes" -) - -const ( - syncMetalLBInterval = 1 * time.Minute - syncMetalLBMinimalInternval = 5 * time.Second -) - -func (h *Housekeeper) startMetalLBConfigSynching() { - go h.ticker.Start("metallb syncher", syncMetalLBInterval, h.stop, h.updateMetalLBConfig) -} - -func (h *Housekeeper) updateMetalLBConfig() error { - if time.Since(h.lastMetalLBConfigSync) < syncMetalLBMinimalInternval { - return nil - } - nodes, err := kubernetes.GetNodes(h.k8sClient) - if err != nil { - return fmt.Errorf("error listing nodes: %w", err) - } - err = h.lbController.UpdateMetalLBConfig(context.Background(), nodes) - if err != nil { - return fmt.Errorf("error updating metallb config: %w", err) - } - h.lastMetalLBConfigSync = time.Now() - return nil -} diff --git a/pkg/controllers/loadbalancer/loadbalancer.go b/pkg/controllers/loadbalancer/loadbalancer.go index 5831d43..a47d624 100644 --- a/pkg/controllers/loadbalancer/loadbalancer.go +++ b/pkg/controllers/loadbalancer/loadbalancer.go @@ -176,7 +176,7 @@ func (l *LoadBalancerController) EnsureLoadBalancer(ctx context.Context, cluster ingressStatus = append(ingressStatus, v1.LoadBalancerIngress{IP: ip}) - err = l.UpdateMetalLBConfig(ctx, ns) + err = l.UpdateLoadBalancerConfig(ctx, ns) if err != nil { return nil, rollback(err) } @@ -194,7 +194,7 @@ func (l *LoadBalancerController) UpdateLoadBalancer(ctx context.Context, cluster for i := range nodes { ns = append(ns, *nodes[i]) } - return l.UpdateMetalLBConfig(ctx, ns) + return l.UpdateLoadBalancerConfig(ctx, ns) } // EnsureLoadBalancerDeleted deletes the cluster load balancer if it @@ -273,8 +273,8 @@ func (l *LoadBalancerController) removeServiceTag(ip models.V1IPResponse, servic return newTags, last } -// UpdateMetalLBConfig the metallb config for given nodes -func (l *LoadBalancerController) UpdateMetalLBConfig(ctx context.Context, nodes []v1.Node) error { +// UpdateLoadBalancerConfig updates the load balancer config for the given nodes +func (l *LoadBalancerController) UpdateLoadBalancerConfig(ctx context.Context, nodes []v1.Node) error { l.configWriteMutex.Lock() defer l.configWriteMutex.Unlock() @@ -283,7 +283,7 @@ func (l *LoadBalancerController) UpdateMetalLBConfig(ctx context.Context, nodes return err } - klog.Info("metallb config updated successfully") + klog.Info("load balancer config updated successfully") return nil } From 61ba202886632df4fbeff4599930f0fe80cb0c8d Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Tue, 25 Jun 2024 10:00:48 +0200 Subject: [PATCH 07/17] only remove lb configmap if namespace is set --- pkg/controllers/loadbalancer/loadbalancer.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/loadbalancer/loadbalancer.go b/pkg/controllers/loadbalancer/loadbalancer.go index a47d624..ee8b506 100644 --- a/pkg/controllers/loadbalancer/loadbalancer.go +++ b/pkg/controllers/loadbalancer/loadbalancer.go @@ -348,12 +348,14 @@ func (l *LoadBalancerController) updateLoadBalancerConfig(ctx context.Context, n } // TODO: in a future release this can be removed - err = l.K8sClient.Delete(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Name: "config", - Namespace: l.LoadBalancerConfig.Namespace(), - }}) - if client.IgnoreNotFound(err) != nil { - return fmt.Errorf("unable to cleanup deprecated metallb configmap: %w", err) + if l.LoadBalancerConfig.Namespace() != "" { + err = l.K8sClient.Delete(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "config", + Namespace: l.LoadBalancerConfig.Namespace(), + }}) + if client.IgnoreNotFound(err) != nil { + return fmt.Errorf("unable to cleanup deprecated metallb configmap: %w", err) + } } err = l.LoadBalancerConfig.WriteCRs(ctx, l.K8sClient) From 96931627666a0e4ad0f426432aa83a1ca8070b0e Mon Sep 17 00:00:00 2001 From: Robert Volkmann Date: Wed, 3 Jul 2024 14:58:28 +0200 Subject: [PATCH 08/17] Deactivate graceful restart to send exportPodCidr See: https://github.com/cilium/cilium/issues/32886#issuecomment-2156914018 --- pkg/controllers/loadbalancer/cilium/config.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index 2c27937..79877f0 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -6,10 +6,11 @@ import ( "strconv" "strings" - "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" "github.com/metal-stack/metal-lib/pkg/pointer" "github.com/metal-stack/metal-lib/pkg/tag" + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -172,9 +173,8 @@ func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c cl ExportPodCIDR: pointer.Pointer(true), Neighbors: []ciliumv2alpha1.CiliumBGPNeighbor{ { - PeerAddress: "127.0.0.1/32", - PeerASN: peer.Peer.ASN, - GracefulRestart: &ciliumv2alpha1.CiliumBGPNeighborGracefulRestart{Enabled: true}, + PeerAddress: "127.0.0.1/32", + PeerASN: peer.Peer.ASN, }, }, }, From 838c01a4725dc15c2e9a09c4dab9dc063bd2b5fc Mon Sep 17 00:00:00 2001 From: Robert Volkmann Date: Wed, 3 Jul 2024 16:53:08 +0200 Subject: [PATCH 09/17] Revert "Deactivate graceful restart to send exportPodCidr" This reverts commit 96931627666a0e4ad0f426432aa83a1ca8070b0e. --- pkg/controllers/loadbalancer/cilium/config.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index 79877f0..2c27937 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -6,11 +6,10 @@ import ( "strconv" "strings" + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" "github.com/metal-stack/metal-lib/pkg/pointer" "github.com/metal-stack/metal-lib/pkg/tag" - "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -173,8 +172,9 @@ func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c cl ExportPodCIDR: pointer.Pointer(true), Neighbors: []ciliumv2alpha1.CiliumBGPNeighbor{ { - PeerAddress: "127.0.0.1/32", - PeerASN: peer.Peer.ASN, + PeerAddress: "127.0.0.1/32", + PeerASN: peer.Peer.ASN, + GracefulRestart: &ciliumv2alpha1.CiliumBGPNeighborGracefulRestart{Enabled: true}, }, }, }, From 790d6e9d3765b039fcef4c574b328418c84537d5 Mon Sep 17 00:00:00 2001 From: Robert Volkmann Date: Wed, 3 Jul 2024 16:54:58 +0200 Subject: [PATCH 10/17] Replace deprecated Cidrs with Blocks --- pkg/controllers/loadbalancer/cilium/config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index 2c27937..d2e27b2 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -6,10 +6,11 @@ import ( "strconv" "strings" - "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" "github.com/metal-stack/metal-lib/pkg/pointer" "github.com/metal-stack/metal-lib/pkg/tag" + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -235,7 +236,7 @@ func (cfg *CiliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c c cidrs = append(cidrs, ipPoolBlock) } ipPool.Spec = ciliumv2alpha1.CiliumLoadBalancerIPPoolSpec{ - Cidrs: cidrs, + Blocks: cidrs, } return nil }) From c73e25bc668c91ca2b276e281261831f0cbf50c6 Mon Sep 17 00:00:00 2001 From: Robert Volkmann Date: Wed, 3 Jul 2024 17:13:46 +0200 Subject: [PATCH 11/17] BGP control plane should announce all services --- pkg/controllers/loadbalancer/cilium/config.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index d2e27b2..ceec707 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -19,6 +19,7 @@ import ( "github.com/metal-stack/metal-go/api/models" ciliumv2alpha1 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1" + slimv1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/yaml" @@ -178,6 +179,16 @@ func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c cl GracefulRestart: &ciliumv2alpha1.CiliumBGPNeighborGracefulRestart{Enabled: true}, }, }, + // A NotIn match expression with a dummy key and value have to be used to announce ALL services. + ServiceSelector: pointer.Pointer(slimv1.LabelSelector{ + MatchExpressions: []slimv1.LabelSelectorRequirement{ + { + Key: "io.cilium/bgp-control-plane", + Operator: slimv1.LabelSelectorOpNotIn, + Values: []string{"ignore"}, + }, + }, + }), }, }, } From 887924642602c93a8816de04b9ffe58678758c32 Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Thu, 4 Jul 2024 11:51:37 +0200 Subject: [PATCH 12/17] add cilium specific virtual router annotation to nodes --- metal/cloud.go | 26 +++--- pkg/controllers/loadbalancer/cilium/config.go | 80 ++++++++++++++----- .../loadbalancer/cilium/config_test.go | 4 +- pkg/controllers/loadbalancer/config.go | 2 +- pkg/controllers/loadbalancer/loadbalancer.go | 5 +- .../loadbalancer/metallb/config.go | 26 +++--- .../loadbalancer/metallb/config_test.go | 4 +- pkg/resources/kubernetes/node.go | 18 +++++ 8 files changed, 111 insertions(+), 54 deletions(-) diff --git a/metal/cloud.go b/metal/cloud.go index ddd43f7..81e0fca 100644 --- a/metal/cloud.go +++ b/metal/cloud.go @@ -47,7 +47,6 @@ func NewCloud(_ io.Reader) (cloudprovider.Interface, error) { partitionID := os.Getenv(constants.MetalPartitionIDEnvVar) clusterID := os.Getenv(constants.MetalClusterIDEnvVar) defaultExternalNetworkID := os.Getenv(constants.MetalDefaultExternalNetworkEnvVar) - loadbalancerType := os.Getenv(constants.Loadbalancer) var ( additionalNetworksString = os.Getenv(constants.MetalAdditionalNetworks) @@ -96,18 +95,7 @@ func NewCloud(_ io.Reader) (cloudprovider.Interface, error) { instancesController := instances.New(defaultExternalNetworkID) zonesController := zones.New() - - var config loadbalancer.LoadBalancerConfig - switch loadbalancerType { - case "metallb": - config = metallb.NewMetalLBConfig() - case "cilium": - config = cilium.NewCiliumConfig() - default: - config = metallb.NewMetalLBConfig() - } - - loadBalancerController := loadbalancer.New(partitionID, projectID, clusterID, defaultExternalNetworkID, additionalNetworks, config) + loadBalancerController := loadbalancer.New(partitionID, projectID, clusterID, defaultExternalNetworkID, additionalNetworks) klog.Info("initialized cloud controller manager") return &cloud{ @@ -123,6 +111,7 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, projectID := os.Getenv(constants.MetalProjectIDEnvVar) sshPublicKey := os.Getenv(constants.MetalSSHPublicKey) clusterID := os.Getenv(constants.MetalClusterIDEnvVar) + loadbalancerType := os.Getenv(constants.Loadbalancer) k8sClientSet := clientBuilder.ClientOrDie("cloud-controller-manager") k8sRestConfig, err := clientBuilder.Config("cloud-controller-manager") @@ -147,6 +136,16 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, klog.Fatalf("unable to create k8s client: %v", err) } + var config loadbalancer.LoadBalancerConfig + switch loadbalancerType { + case "metallb": + config = metallb.NewMetalLBConfig() + case "cilium": + config = cilium.NewCiliumConfig(k8sClientSet) + default: + config = metallb.NewMetalLBConfig() + } + housekeeper := housekeeping.New(metalclient, stop, c.loadBalancer, k8sClientSet, projectID, sshPublicKey, clusterID) ms := metal.New(metalclient, k8sClientSet, projectID) @@ -154,6 +153,7 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, c.loadBalancer.K8sClientSet = k8sClientSet c.loadBalancer.K8sClient = k8sClient c.loadBalancer.MetalService = ms + c.loadBalancer.LoadBalancerConfig = config c.zones.MetalService = ms go housekeeper.Run() diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index ceec707..8041a5a 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -5,15 +5,19 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/metal-stack/metal-lib/pkg/pointer" "github.com/metal-stack/metal-lib/pkg/tag" "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" + "github.com/metal-stack/metal-ccm/pkg/resources/kubernetes" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "github.com/metal-stack/metal-go/api/models" @@ -25,20 +29,21 @@ import ( "sigs.k8s.io/yaml" ) -type CiliumConfig struct { +type ciliumConfig struct { Peers []*Peer `json:"peers,omitempty" yaml:"peers,omitempty"` AddressPools []*loadbalancer.AddressPool `json:"address-pools,omitempty" yaml:"address-pools,omitempty"` + k8sClient clientset.Interface } -func NewCiliumConfig() *CiliumConfig { - return &CiliumConfig{} +func NewCiliumConfig(k8sClient clientset.Interface) *ciliumConfig { + return &ciliumConfig{k8sClient: k8sClient} } -func (cfg *CiliumConfig) Namespace() string { +func (cfg *ciliumConfig) Namespace() string { return "" } -func (cfg *CiliumConfig) CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { +func (cfg *ciliumConfig) PrepareConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { err := cfg.computeAddressPools(ips, nws) if err != nil { return err @@ -50,7 +55,7 @@ func (cfg *CiliumConfig) CalculateConfig(ips []*models.V1IPResponse, nws sets.Se return nil } -func (cfg *CiliumConfig) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { +func (cfg *ciliumConfig) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { for _, ip := range ips { if !nws.Has(*ip.Networkid) { klog.Infof("skipping ip %q: not part of cluster networks", *ip.Ipaddress) @@ -62,17 +67,12 @@ func (cfg *CiliumConfig) computeAddressPools(ips []*models.V1IPResponse, nws set return nil } -func (cfg *CiliumConfig) computePeers(nodes []v1.Node) error { +func (cfg *ciliumConfig) computePeers(nodes []v1.Node) error { cfg.Peers = []*Peer{} // we want an empty array of peers and not nil if there are no nodes for _, n := range nodes { - labels := n.GetLabels() - asnString, ok := labels[tag.MachineNetworkPrimaryASN] - if !ok { - return fmt.Errorf("node %q misses label: %s", n.GetName(), tag.MachineNetworkPrimaryASN) - } - asn, err := strconv.ParseInt(asnString, 10, 64) + asn, err := getASNFromNodeLabels(n) if err != nil { - return fmt.Errorf("unable to parse valid integer from asn annotation: %w", err) + return err } peer, err := newPeer(n, asn) @@ -86,7 +86,7 @@ func (cfg *CiliumConfig) computePeers(nodes []v1.Node) error { return nil } -func (cfg *CiliumConfig) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { +func (cfg *ciliumConfig) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { for _, pool := range cfg.AddressPools { if pool.Name == poolName { return pool @@ -99,7 +99,7 @@ func (cfg *CiliumConfig) getOrCreateAddressPool(poolName string) *loadbalancer.A return pool } -func (cfg *CiliumConfig) addIPToPool(network string, ip models.V1IPResponse) { +func (cfg *ciliumConfig) addIPToPool(network string, ip models.V1IPResponse) { t := ip.Type poolType := models.V1IPBaseTypeEphemeral if t != nil && *t == models.V1IPBaseTypeStatic { @@ -110,7 +110,7 @@ func (cfg *CiliumConfig) addIPToPool(network string, ip models.V1IPResponse) { pool.AppendIP(*ip.Ipaddress) } -func (cfg *CiliumConfig) ToYAML() (string, error) { +func (cfg *ciliumConfig) ToYAML() (string, error) { bb, err := yaml.Marshal(cfg) if err != nil { return "", err @@ -118,7 +118,7 @@ func (cfg *CiliumConfig) ToYAML() (string, error) { return string(bb), nil } -func (cfg *CiliumConfig) WriteCRs(ctx context.Context, c client.Client) error { +func (cfg *ciliumConfig) WriteCRs(ctx context.Context, c client.Client) error { err := cfg.writeCiliumBGPPeeringPolicies(ctx, c) if err != nil { return fmt.Errorf("failed to write ciliumbgppeeringpolicy resources %w", err) @@ -132,7 +132,7 @@ func (cfg *CiliumConfig) WriteCRs(ctx context.Context, c client.Client) error { return nil } -func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c client.Client) error { +func (cfg *ciliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c client.Client) error { existingPolicies := ciliumv2alpha1.CiliumBGPPeeringPolicyList{} err := c.List(ctx, &existingPolicies) if err != nil { @@ -205,7 +205,7 @@ func (cfg *CiliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c cl return nil } -func (cfg *CiliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c client.Client) error { +func (cfg *ciliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c client.Client) error { existingPools := ciliumv2alpha1.CiliumLoadBalancerIPPoolList{} err := c.List(ctx, &existingPools) if err != nil { @@ -261,3 +261,43 @@ func (cfg *CiliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c c return nil } + +func (cfg *ciliumConfig) writeNodeAnnotations() error { + nodes, err := kubernetes.GetNodes(cfg.k8sClient) + if err != nil { + return fmt.Errorf("failed to write node annotations: %w", err) + } + backoff := wait.Backoff{ + Steps: 20, + Duration: 50 * time.Millisecond, + Jitter: 1.0, + } + for _, n := range nodes { + asn, err := getASNFromNodeLabels(n) + if err != nil { + return fmt.Errorf("failed to write node annotations for node %s: %w", n.Name, err) + } + annotations := map[string]string{ + fmt.Sprintf("cilium.io/bgp-virtual-router.%d", asn): "router-id=127.0.0.1", + } + err = kubernetes.UpdateNodeAnnotationsWithBackoff(cfg.k8sClient, n.Name, annotations, backoff) + if err != nil { + return fmt.Errorf("failed to write node annotations for node %s: %w", n.Name, err) + } + } + + return nil +} + +func getASNFromNodeLabels(node v1.Node) (int64, error) { + labels := node.GetLabels() + asnString, ok := labels[tag.MachineNetworkPrimaryASN] + if !ok { + return 0, fmt.Errorf("node %q misses label: %s", node.GetName(), tag.MachineNetworkPrimaryASN) + } + asn, err := strconv.ParseInt(asnString, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse valid integer from asn annotation: %w", err) + } + return asn, nil +} diff --git a/pkg/controllers/loadbalancer/cilium/config_test.go b/pkg/controllers/loadbalancer/cilium/config_test.go index 19e1796..7aaf9b3 100644 --- a/pkg/controllers/loadbalancer/cilium/config_test.go +++ b/pkg/controllers/loadbalancer/cilium/config_test.go @@ -298,9 +298,9 @@ func TestCiliumConfig_CalculateConfig(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - cfg := &CiliumConfig{} + cfg := &ciliumConfig{} - err := cfg.CalculateConfig(tt.ips, tt.nws, tt.nodes) + err := cfg.PrepareConfig(tt.ips, tt.nws, tt.nodes) if diff := cmp.Diff(err, tt.wantErr); diff != "" { t.Errorf("CiliumConfig.CalculateConfig() error = %v", diff) return diff --git a/pkg/controllers/loadbalancer/config.go b/pkg/controllers/loadbalancer/config.go index 4c79a42..f50d207 100644 --- a/pkg/controllers/loadbalancer/config.go +++ b/pkg/controllers/loadbalancer/config.go @@ -11,6 +11,6 @@ import ( type LoadBalancerConfig interface { Namespace() string - CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error + PrepareConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error WriteCRs(ctx context.Context, c client.Client) error } diff --git a/pkg/controllers/loadbalancer/loadbalancer.go b/pkg/controllers/loadbalancer/loadbalancer.go index ee8b506..84d0b19 100644 --- a/pkg/controllers/loadbalancer/loadbalancer.go +++ b/pkg/controllers/loadbalancer/loadbalancer.go @@ -44,9 +44,8 @@ type LoadBalancerController struct { } // New returns a new load balancer controller that satisfies the kubernetes cloud provider load balancer interface -func New(partitionID, projectID, clusterID, defaultExternalNetworkID string, additionalNetworks []string, config LoadBalancerConfig) *LoadBalancerController { +func New(partitionID, projectID, clusterID, defaultExternalNetworkID string, additionalNetworks []string) *LoadBalancerController { return &LoadBalancerController{ - LoadBalancerConfig: config, partitionID: partitionID, projectID: projectID, clusterID: clusterID, @@ -342,7 +341,7 @@ func (l *LoadBalancerController) updateLoadBalancerConfig(ctx context.Context, n return fmt.Errorf("could not find ips of this project's cluster: %w", err) } - err = l.LoadBalancerConfig.CalculateConfig(ips, l.additionalNetworks, nodes) + err = l.LoadBalancerConfig.PrepareConfig(ips, l.additionalNetworks, nodes) if err != nil { return err } diff --git a/pkg/controllers/loadbalancer/metallb/config.go b/pkg/controllers/loadbalancer/metallb/config.go index 7c711ac..e960bc3 100644 --- a/pkg/controllers/loadbalancer/metallb/config.go +++ b/pkg/controllers/loadbalancer/metallb/config.go @@ -29,23 +29,23 @@ const ( metallbNamespace = "metallb-system" ) -// MetalLBConfig is a struct containing a config for metallb -type MetalLBConfig struct { +// metalLBConfig is a struct containing a config for metallb +type metalLBConfig struct { Peers []*Peer `json:"peers,omitempty" yaml:"peers,omitempty"` AddressPools []*loadbalancer.AddressPool `json:"address-pools,omitempty" yaml:"address-pools,omitempty"` namespace string } -func NewMetalLBConfig() *MetalLBConfig { - return &MetalLBConfig{namespace: metallbNamespace} +func NewMetalLBConfig() *metalLBConfig { + return &metalLBConfig{namespace: metallbNamespace} } -func (cfg *MetalLBConfig) Namespace() string { +func (cfg *metalLBConfig) Namespace() string { return cfg.namespace } // CalculateConfig computes the metallb config from given parameter input. -func (cfg *MetalLBConfig) CalculateConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { +func (cfg *metalLBConfig) PrepareConfig(ips []*models.V1IPResponse, nws sets.Set[string], nodes []v1.Node) error { err := cfg.computeAddressPools(ips, nws) if err != nil { return err @@ -57,7 +57,7 @@ func (cfg *MetalLBConfig) CalculateConfig(ips []*models.V1IPResponse, nws sets.S return nil } -func (cfg *MetalLBConfig) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { +func (cfg *metalLBConfig) computeAddressPools(ips []*models.V1IPResponse, nws sets.Set[string]) error { for _, ip := range ips { if !nws.Has(*ip.Networkid) { klog.Infof("skipping ip %q: not part of cluster networks", *ip.Ipaddress) @@ -69,7 +69,7 @@ func (cfg *MetalLBConfig) computeAddressPools(ips []*models.V1IPResponse, nws se return nil } -func (cfg *MetalLBConfig) computePeers(nodes []v1.Node) error { +func (cfg *metalLBConfig) computePeers(nodes []v1.Node) error { cfg.Peers = []*Peer{} // we want an empty array of peers and not nil if there are no nodes for _, n := range nodes { labels := n.GetLabels() @@ -95,7 +95,7 @@ func (cfg *MetalLBConfig) computePeers(nodes []v1.Node) error { // getOrCreateAddressPool returns the address pool of the given network. // It will be created if it does not exist yet. -func (cfg *MetalLBConfig) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { +func (cfg *metalLBConfig) getOrCreateAddressPool(poolName string) *loadbalancer.AddressPool { for _, pool := range cfg.AddressPools { if pool.Name == poolName { return pool @@ -109,7 +109,7 @@ func (cfg *MetalLBConfig) getOrCreateAddressPool(poolName string) *loadbalancer. } // announceIPs appends the given IPs to the network address pools. -func (cfg *MetalLBConfig) addIPToPool(network string, ip models.V1IPResponse) { +func (cfg *metalLBConfig) addIPToPool(network string, ip models.V1IPResponse) { t := ip.Type poolType := models.V1IPBaseTypeEphemeral if t != nil && *t == models.V1IPBaseTypeStatic { @@ -121,7 +121,7 @@ func (cfg *MetalLBConfig) addIPToPool(network string, ip models.V1IPResponse) { } // ToYAML returns this config in YAML format. -func (cfg *MetalLBConfig) ToYAML() (string, error) { +func (cfg *metalLBConfig) ToYAML() (string, error) { bb, err := yaml.Marshal(cfg) if err != nil { return "", err @@ -129,8 +129,8 @@ func (cfg *MetalLBConfig) ToYAML() (string, error) { return string(bb), nil } -// Write inserts or updates the Metal-LB custom resources. -func (cfg *MetalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { +// WriteCRs inserts or updates the Metal-LB custom resources. +func (cfg *metalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { // BGPPeers bgpPeerList := metallbv1beta2.BGPPeerList{} diff --git a/pkg/controllers/loadbalancer/metallb/config_test.go b/pkg/controllers/loadbalancer/metallb/config_test.go index 8471688..de6c018 100644 --- a/pkg/controllers/loadbalancer/metallb/config_test.go +++ b/pkg/controllers/loadbalancer/metallb/config_test.go @@ -298,9 +298,9 @@ func TestMetalLBConfig_CalculateConfig(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - cfg := &MetalLBConfig{} + cfg := &metalLBConfig{} - err := cfg.CalculateConfig(tt.ips, tt.nws, tt.nodes) + err := cfg.PrepareConfig(tt.ips, tt.nws, tt.nodes) if diff := cmp.Diff(err, tt.wantErr); diff != "" { t.Errorf("MetalLBConfig.CalculateConfig() error = %v", diff) return diff --git a/pkg/resources/kubernetes/node.go b/pkg/resources/kubernetes/node.go index cb8fac1..f0b33e0 100644 --- a/pkg/resources/kubernetes/node.go +++ b/pkg/resources/kubernetes/node.go @@ -41,6 +41,24 @@ func UpdateNodeLabelsWithBackoff(client clientset.Interface, nodeName string, la }) } +// UpdateNodeAnnotationsWithBackoff updates labels on a given node with a given backoff retry. +func UpdateNodeAnnotationsWithBackoff(client clientset.Interface, nodeName string, annotations map[string]string, backoff wait.Backoff) error { + return retry.RetryOnConflict(backoff, func() error { + + node, err := client.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + for key, value := range annotations { + node.Annotations[key] = value + } + + _, err = client.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) + return err + }) +} + // NodeNamesOfNodes returns the node names of the nodes func NodeNamesOfNodes(nodes []v1.Node) string { var nn []string From afcde7365f9201a5a25937de857f6d8d907865ca Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Thu, 4 Jul 2024 11:55:28 +0200 Subject: [PATCH 13/17] forgot to call write node annotations --- pkg/controllers/loadbalancer/cilium/config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index 8041a5a..86d556a 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -129,6 +129,11 @@ func (cfg *ciliumConfig) WriteCRs(ctx context.Context, c client.Client) error { return fmt.Errorf("failed to write ciliumloadbalancerippool resources %w", err) } + err = cfg.writeNodeAnnotations() + if err != nil { + return fmt.Errorf("failed to write node annotations %w", err) + } + return nil } From 4402bebfbac2e419f95ecda51029db10071e29cc Mon Sep 17 00:00:00 2001 From: Ilja Rotar Date: Thu, 4 Jul 2024 13:36:16 +0200 Subject: [PATCH 14/17] add context to new k8s client func --- pkg/controllers/housekeeping/loadbalancer.go | 2 +- pkg/controllers/loadbalancer/cilium/config.go | 8 ++++---- pkg/resources/kubernetes/node.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/controllers/housekeeping/loadbalancer.go b/pkg/controllers/housekeeping/loadbalancer.go index acb8fd3..c589b53 100644 --- a/pkg/controllers/housekeeping/loadbalancer.go +++ b/pkg/controllers/housekeeping/loadbalancer.go @@ -21,7 +21,7 @@ func (h *Housekeeper) updateLoadBalancerConfig() error { if time.Since(h.lastLoadBalancerConfigSync) < syncLoadBalancerMinimalInternval { return nil } - nodes, err := kubernetes.GetNodes(h.k8sClient) + nodes, err := kubernetes.GetNodes(context.Background(), h.k8sClient) if err != nil { return fmt.Errorf("error listing nodes: %w", err) } diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index 86d556a..3877499 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -129,7 +129,7 @@ func (cfg *ciliumConfig) WriteCRs(ctx context.Context, c client.Client) error { return fmt.Errorf("failed to write ciliumloadbalancerippool resources %w", err) } - err = cfg.writeNodeAnnotations() + err = cfg.writeNodeAnnotations(ctx) if err != nil { return fmt.Errorf("failed to write node annotations %w", err) } @@ -267,8 +267,8 @@ func (cfg *ciliumConfig) writeCiliumLoadBalancerIPPools(ctx context.Context, c c return nil } -func (cfg *ciliumConfig) writeNodeAnnotations() error { - nodes, err := kubernetes.GetNodes(cfg.k8sClient) +func (cfg *ciliumConfig) writeNodeAnnotations(ctx context.Context) error { + nodes, err := kubernetes.GetNodes(ctx, cfg.k8sClient) if err != nil { return fmt.Errorf("failed to write node annotations: %w", err) } @@ -285,7 +285,7 @@ func (cfg *ciliumConfig) writeNodeAnnotations() error { annotations := map[string]string{ fmt.Sprintf("cilium.io/bgp-virtual-router.%d", asn): "router-id=127.0.0.1", } - err = kubernetes.UpdateNodeAnnotationsWithBackoff(cfg.k8sClient, n.Name, annotations, backoff) + err = kubernetes.UpdateNodeAnnotationsWithBackoff(ctx, cfg.k8sClient, n.Name, annotations, backoff) if err != nil { return fmt.Errorf("failed to write node annotations for node %s: %w", n.Name, err) } diff --git a/pkg/resources/kubernetes/node.go b/pkg/resources/kubernetes/node.go index 8d8746f..3e0a1be 100644 --- a/pkg/resources/kubernetes/node.go +++ b/pkg/resources/kubernetes/node.go @@ -42,10 +42,10 @@ func UpdateNodeLabelsWithBackoff(ctx context.Context, client clientset.Interface } // UpdateNodeAnnotationsWithBackoff updates labels on a given node with a given backoff retry. -func UpdateNodeAnnotationsWithBackoff(client clientset.Interface, nodeName string, annotations map[string]string, backoff wait.Backoff) error { +func UpdateNodeAnnotationsWithBackoff(ctx context.Context, client clientset.Interface, nodeName string, annotations map[string]string, backoff wait.Backoff) error { return retry.RetryOnConflict(backoff, func() error { - node, err := client.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + node, err := client.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) if err != nil { return err } @@ -54,7 +54,7 @@ func UpdateNodeAnnotationsWithBackoff(client clientset.Interface, nodeName strin node.Annotations[key] = value } - _, err = client.CoreV1().Nodes().Update(context.Background(), node, metav1.UpdateOptions{}) + _, err = client.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) return err }) } From 4baa11c70a7b1ce0b6c5c8d23cf6f018b65a26fd Mon Sep 17 00:00:00 2001 From: Robert Volkmann Date: Thu, 5 Sep 2024 19:38:26 +0200 Subject: [PATCH 15/17] DROP: Checkout pull request HEAD commit instead of merge commit --- .github/workflows/docker.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index f5b6368..267d9e4 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -30,6 +30,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} - name: Set up Go 1.22 uses: actions/setup-go@v5 From 1263958ccd45642baa33651ee756adc032034ab6 Mon Sep 17 00:00:00 2001 From: Robert Volkmann Date: Fri, 6 Sep 2024 10:42:00 +0200 Subject: [PATCH 16/17] DROP: Allow building container image on merge conflicts --- .github/workflows/docker.yaml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 267d9e4..30e4759 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -1,15 +1,7 @@ --- name: Docker Build Action on: - pull_request: - branches: - - master - release: - types: - - published push: - branches: - - master env: REGISTRY: ghcr.io @@ -44,15 +36,9 @@ jobs: with: args: --build-tags integration -p bugs -p unused --timeout=5m - - name: Make tag - run: | - [ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo "tag=${GITHUB_HEAD_REF##*/}" >> $GITHUB_ENV || true - [ "${GITHUB_EVENT_NAME}" == 'release' ] && echo "tag=${GITHUB_REF##*/}" >> $GITHUB_ENV || true - [ "${GITHUB_EVENT_NAME}" == 'push' ] && echo "tag=latest" >> $GITHUB_ENV || true - - name: Build and push image uses: docker/build-push-action@v5 with: context: . push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.tag }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cilium-loadbalancer From c8bc44aae7ee3803f4beb246ac82f7dd972bf7f8 Mon Sep 17 00:00:00 2001 From: Robert Volkmann Date: Fri, 6 Sep 2024 15:42:19 +0200 Subject: [PATCH 17/17] Fix linter warnings regarding casting to unit32 --- pkg/controllers/loadbalancer/cilium/config.go | 4 ++-- pkg/controllers/loadbalancer/cilium/peer.go | 7 +++++-- pkg/controllers/loadbalancer/metallb/config.go | 4 ++-- pkg/controllers/loadbalancer/metallb/peer.go | 7 +++++-- pkg/controllers/loadbalancer/peer.go | 6 +++--- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pkg/controllers/loadbalancer/cilium/config.go b/pkg/controllers/loadbalancer/cilium/config.go index 3877499..a22288d 100644 --- a/pkg/controllers/loadbalancer/cilium/config.go +++ b/pkg/controllers/loadbalancer/cilium/config.go @@ -175,12 +175,12 @@ func (cfg *ciliumConfig) writeCiliumBGPPeeringPolicies(ctx context.Context, c cl NodeSelector: &peer.NodeSelector, VirtualRouters: []ciliumv2alpha1.CiliumBGPVirtualRouter{ { - LocalASN: peer.Peer.MyASN, + LocalASN: int64(peer.Peer.MyASN), ExportPodCIDR: pointer.Pointer(true), Neighbors: []ciliumv2alpha1.CiliumBGPNeighbor{ { PeerAddress: "127.0.0.1/32", - PeerASN: peer.Peer.ASN, + PeerASN: int64(peer.Peer.ASN), GracefulRestart: &ciliumv2alpha1.CiliumBGPNeighborGracefulRestart{Enabled: true}, }, }, diff --git a/pkg/controllers/loadbalancer/cilium/peer.go b/pkg/controllers/loadbalancer/cilium/peer.go index f92556a..079661c 100644 --- a/pkg/controllers/loadbalancer/cilium/peer.go +++ b/pkg/controllers/loadbalancer/cilium/peer.go @@ -2,8 +2,9 @@ package cilium import ( slimv1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1" - "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" v1 "k8s.io/api/core/v1" + + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" ) type Peer struct { @@ -22,7 +23,9 @@ func newPeer(node v1.Node, asn int64) (*Peer, error) { }, } - peer, err := loadbalancer.NewPeer(node, asn) + // we can safely cast the asn to an uint32 because its max value is defined as such + // see: https://en.wikipedia.org/wiki/Autonomous_system_(Internet) + peer, err := loadbalancer.NewPeer(node, uint32(asn)) // nolint:gosec if err != nil { return nil, err } diff --git a/pkg/controllers/loadbalancer/metallb/config.go b/pkg/controllers/loadbalancer/metallb/config.go index e960bc3..fc1c063 100644 --- a/pkg/controllers/loadbalancer/metallb/config.go +++ b/pkg/controllers/loadbalancer/metallb/config.go @@ -168,8 +168,8 @@ func (cfg *metalLBConfig) WriteCRs(ctx context.Context, c client.Client) error { } res, err := controllerutil.CreateOrUpdate(ctx, c, bgpPeer, func() error { bgpPeer.Spec = metallbv1beta2.BGPPeerSpec{ - MyASN: uint32(peer.Peer.MyASN), - ASN: uint32(peer.Peer.ASN), + MyASN: peer.Peer.MyASN, + ASN: peer.Peer.ASN, HoldTime: metav1.Duration{Duration: 90 * time.Second}, KeepaliveTime: metav1.Duration{Duration: 0 * time.Second}, Address: peer.Peer.Address, diff --git a/pkg/controllers/loadbalancer/metallb/peer.go b/pkg/controllers/loadbalancer/metallb/peer.go index 6902d15..714f52c 100644 --- a/pkg/controllers/loadbalancer/metallb/peer.go +++ b/pkg/controllers/loadbalancer/metallb/peer.go @@ -3,9 +3,10 @@ package metallb import ( "fmt" - "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/metal-stack/metal-ccm/pkg/controllers/loadbalancer" ) type Peer struct { @@ -24,7 +25,9 @@ func newPeer(node v1.Node, asn int64) (*Peer, error) { }, } - peer, err := loadbalancer.NewPeer(node, asn) + // we can safely cast the asn to an uint32 because its max value is defined as such + // see: https://en.wikipedia.org/wiki/Autonomous_system_(Internet) + peer, err := loadbalancer.NewPeer(node, uint32(asn)) // nolint:gosec if err != nil { return nil, err } diff --git a/pkg/controllers/loadbalancer/peer.go b/pkg/controllers/loadbalancer/peer.go index 36bbad7..2ca4a7b 100644 --- a/pkg/controllers/loadbalancer/peer.go +++ b/pkg/controllers/loadbalancer/peer.go @@ -7,12 +7,12 @@ import ( ) type Peer struct { - MyASN int64 `json:"my-asn" yaml:"my-asn"` - ASN int64 `json:"peer-asn" yaml:"peer-asn"` + MyASN uint32 `json:"my-asn" yaml:"my-asn"` + ASN uint32 `json:"peer-asn" yaml:"peer-asn"` Address string `json:"peer-address" yaml:"peer-address"` } -func NewPeer(node v1.Node, asn int64) (*Peer, error) { +func NewPeer(node v1.Node, asn uint32) (*Peer, error) { address, err := NodeAddress(node) if err != nil { return nil, err