feat: Deduplicate exceptions workflow (#332)

* feat: Add exception dedup workflow * chore: Dedup exceptions * perf: Added JSON parallel processing
yonahd · Jul 25, 2024 · 93e7d80 · 93e7d80
1 parent 01ce4a6
commit 93e7d80
Show file tree

Hide file tree

Showing 7 changed files with 47 additions and 117 deletions.
diff --git a/.github/workflows/exceptions-json-sort.yml b/.github/workflows/exceptions-json-sort.yml
diff --git a/.github/workflows/validate-exception-jsons.yml b/.github/workflows/validate-exception-jsons.yml
@@ -0,0 +1,22 @@
+name: Validate Exception JSONs
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "pkg/kor/exceptions/**"
+
+jobs:
+  sort_json:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Validate Exception JSON Sorting
+        run: make validate-exception-sorting
+
+      - name: Validate Exception Deduplication
+        run: make validate-exception-duplications
diff --git a/Makefile b/Makefile
@@ -17,7 +17,7 @@ test:
 
 sort-exception-files:
 	@echo "Sorting exception files..."
-	@find $(EXCEPTIONS_DIR) -name '$(EXCEPTIONS_FILE_PATTERN)' -exec sh -c ' \
+	@find $(EXCEPTIONS_DIR) -name '$(EXCEPTIONS_FILE_PATTERN)' | xargs -I{} -P 4 sh -c ' \
 		jq "with_entries(.value |= sort_by(.Namespace, .ResourceName))" {} > {}.tmp && mv {}.tmp {} \
 	' \;
 
@@ -36,4 +36,27 @@ validate-exception-sorting:
 	done; \
 	if [ "$$PRINT_ERR" = 0 ]; then \
 		echo "Run the following command to sort all files recursively: make sort-exception-files"; \
-	fi; \
+	fi; \
+
+dedup-exception-files:
+	@echo "Deduplicating exception files..."
+	@find $(EXCEPTIONS_DIR) -type f -name '$(EXCEPTIONS_FILE_PATTERN)' | xargs -I{} -P 4 sh -c ' \
+		jq '\''keys[0] as $$key | { ($$key): (.[$$key] | group_by(.Namespace, .ResourceName) | map(.[0])) }'\'' "$$1" > "$$1.tmp" && mv "$$1.tmp" "$$1" \
+	' sh {} \;
+
+
+validate-exception-duplications:
+	@PRINT_ERR=1; \
+	for file in $(wildcard $(EXCEPTIONS_DIR)/*/$(EXCEPTIONS_FILE_PATTERN)); do \
+		DUPLICATES=$$(jq 'keys[0] as $$key | .[$$key] | group_by(.Namespace, .ResourceName) | map(select(length > 1))' "$$file"); \
+		if [ "$$DUPLICATES" != "[]" ]; then \
+			if [ "$$PRINT_ERR" = 1 ]; then \
+				echo "The following JSON files contain duplications:"; \
+				PRINT_ERR=0; \
+			fi; \
+			echo "\t$$file"; \
+		fi; \
+	done; \
+	if [ "$$PRINT_ERR" = 0 ]; then \
+		echo "Run the following command to deduplicate all files recursively: make dedup-exception-files"; \
+	fi; \
diff --git a/pkg/kor/exceptions/clusterroles/clusterroles.json b/pkg/kor/exceptions/clusterroles/clusterroles.json
@@ -132,10 +132,6 @@
       "Namespace": "",
       "ResourceName": "system:aggregated-metrics-reader"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "system:aggregated-metrics-reader"
-    },
     {
       "Namespace": "",
       "ResourceName": "system:auth-delegator"
@@ -204,10 +200,6 @@
       "Namespace": "",
       "ResourceName": "system:node-bootstrapper"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "system:node-bootstrapper"
-    },
     {
       "Namespace": "",
       "ResourceName": "system:node-problem-detector"

diff --git a/pkg/kor/exceptions/configmaps/configmaps.json b/pkg/kor/exceptions/configmaps/configmaps.json
@@ -46,10 +46,6 @@
       "Namespace": "kube-system",
       "ResourceName": "cluster-dns"
     },
-    {
-      "Namespace": "kube-system",
-      "ResourceName": "cluster-dns"
-    },
     {
       "Namespace": "kube-system",
       "ResourceName": "cluster-kubestore"

diff --git a/pkg/kor/exceptions/crds/crds.json b/pkg/kor/exceptions/crds/crds.json
@@ -124,10 +124,6 @@
       "Namespace": "",
       "ResourceName": "etcdsnapshotfiles.k3s.cattle.io"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "etcdsnapshotfiles.k3s.cattle.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "firmwareschemas.metal3.io"
@@ -152,10 +148,6 @@
       "Namespace": "",
       "ResourceName": "helmchartconfigs.helm.cattle.io"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "helmchartconfigs.helm.cattle.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "hostfirmwaresettings.metal3.io"
@@ -180,14 +172,6 @@
       "Namespace": "",
       "ResourceName": "ingressroutes.traefik.containo.us"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "ingressroutes.traefik.containo.us"
-    },
-    {
-      "Namespace": "",
-      "ResourceName": "ingressroutetcps.traefik.containo.us"
-    },
     {
       "Namespace": "",
       "ResourceName": "ingressroutetcps.traefik.containo.us"
@@ -196,22 +180,10 @@
       "Namespace": "",
       "ResourceName": "ingressroutetcps.traefik.io"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "ingressroutetcps.traefik.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "ingressrouteudps.traefik.containo.us"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "ingressrouteudps.traefik.containo.us"
-    },
-    {
-      "Namespace": "",
-      "ResourceName": "ingressrouteudps.traefik.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "ingressrouteudps.traefik.io"
@@ -260,14 +232,6 @@
       "Namespace": "",
       "ResourceName": "middlewares.traefik.containo.us"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "middlewares.traefik.containo.us"
-    },
-    {
-      "Namespace": "",
-      "ResourceName": "middlewares.traefik.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "middlewares.traefik.io"
@@ -276,14 +240,6 @@
       "Namespace": "",
       "ResourceName": "middlewaretcps.traefik.containo.us"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "middlewaretcps.traefik.containo.us"
-    },
-    {
-      "Namespace": "",
-      "ResourceName": "middlewaretcps.traefik.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "middlewaretcps.traefik.io"
@@ -356,14 +312,6 @@
       "Namespace": "",
       "ResourceName": "serverstransports.traefik.containo.us"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "serverstransports.traefik.containo.us"
-    },
-    {
-      "Namespace": "",
-      "ResourceName": "serverstransports.traefik.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "serverstransports.traefik.io"
@@ -372,10 +320,6 @@
       "Namespace": "",
       "ResourceName": "serverstransporttcps.traefik.io"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "serverstransporttcps.traefik.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "serviceattachments.networking.gke.io"
@@ -400,22 +344,10 @@
       "Namespace": "",
       "ResourceName": "tlsoptions.traefik.containo.us"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "tlsoptions.traefik.containo.us"
-    },
     {
       "Namespace": "",
       "ResourceName": "tlsoptions.traefik.io"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "tlsoptions.traefik.io"
-    },
-    {
-      "Namespace": "",
-      "ResourceName": "tlsstores.traefik.containo.us"
-    },
     {
       "Namespace": "",
       "ResourceName": "tlsstores.traefik.containo.us"
@@ -424,14 +356,6 @@
       "Namespace": "",
       "ResourceName": "tlsstores.traefik.io"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "tlsstores.traefik.io"
-    },
-    {
-      "Namespace": "",
-      "ResourceName": "traefikservices.traefik.containo.us"
-    },
     {
       "Namespace": "",
       "ResourceName": "traefikservices.traefik.containo.us"
@@ -440,10 +364,6 @@
       "Namespace": "",
       "ResourceName": "traefikservices.traefik.io"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "traefikservices.traefik.io"
-    },
     {
       "Namespace": "",
       "ResourceName": "updateinfos.nodemanagement.gke.io"

diff --git a/pkg/kor/exceptions/storageclasses/storageclasses.json b/pkg/kor/exceptions/storageclasses/storageclasses.json
@@ -24,10 +24,6 @@
       "Namespace": "",
       "ResourceName": "local-path"
     },
-    {
-      "Namespace": "",
-      "ResourceName": "local-path"
-    },
     {
       "Namespace": "",
       "ResourceName": "managed"