First draft of forbidden mode for isolated clusters (#172)

Co-authored-by: Valentin Knabel <dev@vknabel.com>
Co-authored-by: Ulrich Schreiner <ulrich.schreiner@gmail.com>
Co-authored-by: Markus Wennrich <markus.wennrich@f-i-ts.de>
Co-authored-by: Gerrit91 <info@gerritschwerthelm.de>

.github/workflows/build.yaml

@@ -23,14 +23,14 @@ jobs:
 
     steps:
     - name: Log in to the container registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: ${{ env.REGISTRY }}
         username: ${{ secrets.DOCKER_REGISTRY_USER }}
         password: ${{ secrets.DOCKER_REGISTRY_TOKEN }}
 
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - uses: google-github-actions/auth@v1
       with:
@@ -40,9 +40,10 @@ jobs:
       uses: google-github-actions/setup-gcloud@v0
 
     - name: Set up Go 1.21
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: '1.21'
+        cache: false
 
     - name: Lint
       uses: golangci/golangci-lint-action@v3
@@ -64,7 +65,7 @@ jobs:
         make
 
     - name: Push Docker image
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v5
       with:
         context: .
         push: true
@@ -89,12 +90,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Go 1.21
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: '1.21'
+        cache: false
 
     - name: Run tests
       run: |

api/v1/clusterwidenetworkpolicy_types.go

@@ -27,6 +27,8 @@ const (
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:shortName=cwnp
 // +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.state"
+// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message"
 type ClusterwideNetworkPolicy struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -67,12 +69,26 @@ type PolicySpec struct {
 
 type FQDNState map[string][]IPSet
 
+// PolicyDeploymentState describes the state of a CWNP deployment
+type PolicyDeploymentState string
+
+const (
+	// PolicyDeploymentStateDeployed the CWNP was deployed to a native nftable rule
+	PolicyDeploymentStateDeployed = PolicyDeploymentState("deployed")
+	// PolicyDeploymentStateIgnored the CWNP was not deployed to a native nftable rule because it is outside of allowed networks
+	PolicyDeploymentStateIgnored = PolicyDeploymentState("ignored")
+)
+
 // PolicyStatus defines the observed state for CWNP resource
 type PolicyStatus struct {
 	// FQDNState stores mapping from FQDN rules to nftables sets used for a firewall rule.
 	// Key is either MatchName or MatchPattern
 	// +optional
 	FQDNState FQDNState `json:"fqdn_state,omitempty"`
+	// State of the CWNP, can be either deployed or ignored
+	State PolicyDeploymentState `json:"state,omitempty"`
+	// Message describes why the state changed
+	Message string `json:"message,omitempty"`
 }
 
 // IngressRule describes a particular set of traffic that is allowed to the cluster.

config/crd/bases/metal-stack.io_clusterwidenetworkpolicies.yaml

@@ -16,7 +16,14 @@ spec:
     singular: clusterwidenetworkpolicy
   scope: Namespaced
   versions:
-  - name: v1
+  - additionalPrinterColumns:
+    - jsonPath: .status.state
+      name: Status
+      type: string
+    - jsonPath: .status.message
+      name: Message
+      type: string
+    name: v1
     schema:
       openAPIV3Schema:
         description: ClusterwideNetworkPolicy contains the desired state for a cluster
@@ -251,6 +258,12 @@ spec:
                 description: FQDNState stores mapping from FQDN rules to nftables
                   sets used for a firewall rule. Key is either MatchName or MatchPattern
                 type: object
+              message:
+                description: Message describes why the state changed
+                type: string
+              state:
+                description: State of the CWNP, can be either deployed or ignored
+                type: string
             type: object
         type: object
     served: true

controllers/clusterwidenetworkpolicy_controller.go

@@ -5,13 +5,17 @@ import (
 	"fmt"
 	"time"
 
+	"go4.org/netipx"
+
 	"github.com/metal-stack/firewall-controller/v2/pkg/dns"
+	"github.com/metal-stack/firewall-controller/v2/pkg/helper"
 	"github.com/metal-stack/firewall-controller/v2/pkg/nftables"
 
 	"github.com/go-logr/logr"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -33,7 +37,8 @@ type ClusterwideNetworkPolicyReconciler struct {
 	FirewallName  string
 	SeedNamespace string
 
-	Log logr.Logger
+	Log      logr.Logger
+	Recorder record.EventRecorder
 
 	Interval time.Duration
 	DnsProxy *dns.DNSProxy
@@ -91,7 +96,14 @@ func (r *ClusterwideNetworkPolicyReconciler) Reconcile(ctx context.Context, _ ct
 	if err := r.ShootClient.List(ctx, &services); err != nil {
 		return ctrl.Result{}, err
 	}
-	nftablesFirewall := nftables.NewFirewall(f, &cwnps, &services, r.DnsProxy, r.Log)
+
+	validCwnps, err := r.allowedCWNPs(ctx, cwnps.Items, f.Spec.AllowedNetworks)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	cwnps.Items = validCwnps
+
+	nftablesFirewall := nftables.NewFirewall(f, &cwnps, &services, r.DnsProxy, r.Log, r.Recorder)
 	if err := r.manageDNSProxy(ctx, f, cwnps, nftablesFirewall); err != nil {
 		return ctrl.Result{}, err
 	}
@@ -103,8 +115,8 @@ func (r *ClusterwideNetworkPolicyReconciler) Reconcile(ctx context.Context, _ ct
 	if updated {
 		for _, i := range cwnps.Items {
 			o := i
-			if err := r.ShootClient.Status().Update(ctx, &o); err != nil {
-				return ctrl.Result{}, fmt.Errorf("failed to updated CWNP status: %w", err)
+			if err := r.updateCWNPState(ctx, o, firewallv1.PolicyDeploymentStateDeployed, ""); err != nil {
+				return ctrl.Result{}, err
 			}
 		}
 	}
@@ -181,3 +193,82 @@ func (r *ClusterwideNetworkPolicyReconciler) getReconciliationTicker(scheduleCha
 		}
 	}
 }
+
+func (r *ClusterwideNetworkPolicyReconciler) allowedCWNPs(ctx context.Context, cwnps []firewallv1.ClusterwideNetworkPolicy, allowedNetworks firewallv2.AllowedNetworks) ([]firewallv1.ClusterwideNetworkPolicy, error) {
+	if len(allowedNetworks.Egress) == 0 && len(allowedNetworks.Ingress) == 0 {
+		return cwnps, nil
+	}
+
+	validCWNPs := make([]firewallv1.ClusterwideNetworkPolicy, 0, len(cwnps))
+
+	egressSet, err := helper.BuildNetworksIPSet(allowedNetworks.Egress)
+	if err != nil {
+		return nil, err
+	}
+
+	ingressSet, err := helper.BuildNetworksIPSet(allowedNetworks.Ingress)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, cwnp := range cwnps {
+		cwnp := cwnp
+		oke, err := r.validateCWNPEgressTargetPrefix(cwnp, egressSet)
+		if err != nil {
+			return nil, err
+		}
+		oki, err := r.validateCWNPIngressTargetPrefix(cwnp, ingressSet)
+		if err != nil {
+			return nil, err
+		}
+
+		if !oki || !oke {
+			// at least one of ingress and/or egress is not in the allowed network set
+			if err := r.updateCWNPState(ctx, cwnp, firewallv1.PolicyDeploymentStateIgnored, "ingress/egress does not match allowed networks"); err != nil {
+				return nil, err
+			}
+			continue
+		}
+
+		validCWNPs = append(validCWNPs, cwnp)
+	}
+
+	return validCWNPs, nil
+}
+
+func (r *ClusterwideNetworkPolicyReconciler) updateCWNPState(ctx context.Context, cwnp firewallv1.ClusterwideNetworkPolicy, state firewallv1.PolicyDeploymentState, msg string) error {
+	// do nothing if message and state already have the desired values
+	if cwnp.Status.Message == msg && cwnp.Status.State == state {
+		return nil
+	}
+
+	cwnp.Status.Message = msg
+	cwnp.Status.State = state
+
+	if err := r.ShootClient.Status().Update(ctx, &cwnp); err != nil {
+		return fmt.Errorf("failed to update status of CWNP %q to %q: %w", cwnp.Name, state, err)
+	}
+	return nil
+}
+
+func (r *ClusterwideNetworkPolicyReconciler) validateCWNPEgressTargetPrefix(cwnp firewallv1.ClusterwideNetworkPolicy, ipSet *netipx.IPSet) (bool, error) {
+	for _, egress := range cwnp.Spec.Egress {
+		for _, to := range egress.To {
+			if ok, err := helper.ValidateCIDR(&cwnp, to.CIDR, ipSet, r.Recorder); !ok {
+				return false, err
+			}
+		}
+	}
+	return true, nil
+}
+
+func (r *ClusterwideNetworkPolicyReconciler) validateCWNPIngressTargetPrefix(cwnp firewallv1.ClusterwideNetworkPolicy, ipSet *netipx.IPSet) (bool, error) {
+	for _, ingress := range cwnp.Spec.Ingress {
+		for _, from := range ingress.From {
+			if ok, err := helper.ValidateCIDR(&cwnp, from.CIDR, ipSet, r.Recorder); !ok {
+				return false, err
+			}
+		}
+	}
+	return true, nil
+}

controllers/firewall_controller.go

@@ -86,7 +86,8 @@ func (r *FirewallReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		if apierrors.IsNotFound(err) {
 			r.Log.Info("flushing k8s firewall rules")
 
-			defaultFw := nftables.NewDefaultFirewall()
+			defaultFw := nftables.NewFirewall(&firewallv2.Firewall{}, &firewallv1.ClusterwideNetworkPolicyList{}, &corev1.ServiceList{}, nil, logr.Discard(), r.Recorder)
+
 			flushErr := defaultFw.Flush()
 			if flushErr != nil {
 				r.Log.Error(flushErr, "error flushing k8s firewall rules")

go.mod

@@ -10,14 +10,15 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c
 	github.com/ks2211/go-suricata v0.0.0-20200823200910-986ce1470707
-	github.com/metal-stack/firewall-controller-manager v0.3.0
+	github.com/metal-stack/firewall-controller-manager v0.3.2
 	github.com/metal-stack/metal-go v0.26.2
 	github.com/metal-stack/metal-lib v0.14.3
 	github.com/metal-stack/metal-networker v0.41.0
 	github.com/metal-stack/v v1.0.3
-	github.com/miekg/dns v1.1.56
-	github.com/txn2/txeh v1.5.3
+	github.com/miekg/dns v1.1.58
+	github.com/txn2/txeh v1.5.5
 	github.com/vishvananda/netlink v1.2.1-beta.2
+	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
 	k8s.io/api v0.26.3
 	k8s.io/apiextensions-apiserver v0.26.3
 	k8s.io/apimachinery v0.28.2
@@ -83,7 +84,7 @@ require (
 	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.16.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect

go.sum

@@ -172,8 +172,8 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/
 github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
 github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI=
 github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI=
-github.com/metal-stack/firewall-controller-manager v0.3.0 h1:sBCL7iiG17ZO/1TREv2RYiMdX5VddSc92snR8OKcAF8=
-github.com/metal-stack/firewall-controller-manager v0.3.0/go.mod h1:KjLZv/BatucZM9DQtBLN04wBGjvxEJRV1C+xDkCWwIE=
+github.com/metal-stack/firewall-controller-manager v0.3.2 h1:SXamSfDzEZgXGLf4EAobVnBDtTq/QauXcggiifA5gHg=
+github.com/metal-stack/firewall-controller-manager v0.3.2/go.mod h1:KjLZv/BatucZM9DQtBLN04wBGjvxEJRV1C+xDkCWwIE=
 github.com/metal-stack/metal-go v0.26.2 h1:KZRV1wtCsj0dMo4GpW2+XemmAkPZAYFjbGe7QhhcH1k=
 github.com/metal-stack/metal-go v0.26.2/go.mod h1:olJ3Az7RBh39Q5WFCJOQBd7cJi0xgGYwMTEIFvkDQQY=
 github.com/metal-stack/metal-hammer v0.12.0 h1:t6t73RGmDU1IFkHC7dJxu7xDIZZvwmqmu9/0xZVF/L0=
@@ -184,8 +184,8 @@ github.com/metal-stack/metal-networker v0.41.0 h1:eefp8nzhF6eBDoGjFR8m+chkGUO5QF
 github.com/metal-stack/metal-networker v0.41.0/go.mod h1:jdHKFIbPBNHnvies0Tb8DlnPfbKV/HuikxPZH3kC6uA=
 github.com/metal-stack/v v1.0.3 h1:Sh2oBlnxrCUD+mVpzfC8HiqL045YWkxs0gpTvkjppqs=
 github.com/metal-stack/v v1.0.3/go.mod h1:YTahEu7/ishwpYKnp/VaW/7nf8+PInogkfGwLcGPdXg=
-github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
-github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
+github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
+github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -259,8 +259,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/txn2/txeh v1.5.3 h1:ZMgc3r+5/AFtE/ayCoICpvxj7xl/CYsZjnIGhozV/Kc=
-github.com/txn2/txeh v1.5.3/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4=
+github.com/txn2/txeh v1.5.5 h1:UN4e/lCK5HGw/gGAi2GCVrNKg0GTCUWs7gs5riaZlz4=
+github.com/txn2/txeh v1.5.5/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4=
 github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
 github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
@@ -286,6 +286,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
 go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -382,8 +384,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
-golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -241,6 +241,7 @@ func main() {
 		SeedClient:    seedMgr.GetClient(),
 		ShootClient:   shootMgr.GetClient(),
 		Log:           ctrl.Log.WithName("controllers").WithName("ClusterwideNetworkPolicy"),
+		Recorder:      shootMgr.GetEventRecorderFor("FirewallController"),
 		FirewallName:  firewallName,
 		SeedNamespace: seedNamespace,
 	}).SetupWithManager(shootMgr); err != nil {