JQ filtering in client

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+yarn run lint-staged

client/__tests__/__snapshots__/filter.test.ts.snap

@@ -0,0 +1,114 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`json path filter returns only selected fields 1`] = `
+{
+  "data": {
+    "items": [
+      {
+        "metadata": {
+          "name": "admin",
+        },
+      },
+      {
+        "metadata": {
+          "name": "aws-node",
+        },
+      },
+    ],
+    "kind": "ClusterRoleList",
+  },
+  "status": 200,
+}
+`;
+
+exports[`json path filter returns original input data on error 1`] = `
+{
+  "data": {
+    "apiVersion": "rbac.authorization.k8s.io/v1",
+    "items": [
+      {
+        "metadata": {
+          "managedFields": [
+            {
+              "manager": "clusterrole-aggregation-controller",
+              "operation": "Apply",
+            },
+            {
+              "manager": "kube-apiserver",
+              "operation": "Update",
+            },
+          ],
+          "name": "admin",
+          "resourceVersion": "345",
+          "uid": "4251609f-b3ca-4875-8ec6-76b40ab62748",
+        },
+      },
+      {
+        "metadata": {
+          "managedFields": [
+            {
+              "manager": "kubectl-client-side-apply",
+              "operation": "Update",
+            },
+          ],
+          "name": "aws-node",
+          "resourceVersion": "273",
+          "uid": "63a3bed8-a750-440d-bca9-b5f41c8786c2",
+        },
+      },
+    ],
+    "kind": "ClusterRoleList",
+    "metadata": {
+      "resourceVersion": "4423772",
+    },
+  },
+  "status": 200,
+  "statusText": "OK",
+}
+`;
+
+exports[`json path filter returns original input data when no filter supplied 1`] = `
+{
+  "data": {
+    "apiVersion": "rbac.authorization.k8s.io/v1",
+    "items": [
+      {
+        "metadata": {
+          "managedFields": [
+            {
+              "manager": "clusterrole-aggregation-controller",
+              "operation": "Apply",
+            },
+            {
+              "manager": "kube-apiserver",
+              "operation": "Update",
+            },
+          ],
+          "name": "admin",
+          "resourceVersion": "345",
+          "uid": "4251609f-b3ca-4875-8ec6-76b40ab62748",
+        },
+      },
+      {
+        "metadata": {
+          "managedFields": [
+            {
+              "manager": "kubectl-client-side-apply",
+              "operation": "Update",
+            },
+          ],
+          "name": "aws-node",
+          "resourceVersion": "273",
+          "uid": "63a3bed8-a750-440d-bca9-b5f41c8786c2",
+        },
+      },
+    ],
+    "kind": "ClusterRoleList",
+    "metadata": {
+      "resourceVersion": "4423772",
+    },
+  },
+  "status": 200,
+  "statusText": "OK",
+}
+`;

client/__tests__/filter.test.ts

@@ -0,0 +1,70 @@
+import { jpFilter as jqFilter } from "../filter";
+
+const exampleResponse = {
+  status: 200,
+  statusText: "OK",
+  data: {
+    kind: "ClusterRoleList",
+    apiVersion: "rbac.authorization.k8s.io/v1",
+    metadata: {
+      resourceVersion: "4423772",
+    },
+    items: [
+      {
+        metadata: {
+          name: "admin",
+          uid: "4251609f-b3ca-4875-8ec6-76b40ab62748",
+          resourceVersion: "345",
+          managedFields: [
+            {
+              manager: "clusterrole-aggregation-controller",
+              operation: "Apply",
+            },
+            {
+              manager: "kube-apiserver",
+              operation: "Update",
+            },
+          ],
+        },
+      },
+      {
+        metadata: {
+          name: "aws-node",
+          uid: "63a3bed8-a750-440d-bca9-b5f41c8786c2",
+          resourceVersion: "273",
+          managedFields: [
+            {
+              manager: "kubectl-client-side-apply",
+              operation: "Update",
+            },
+          ],
+        },
+      },
+    ],
+  },
+};
+
+describe("json path filter", () => {
+  it("returns only selected fields", async () => {
+    const result = await jqFilter(
+      exampleResponse,
+      "{ status: .status, data: { kind: .data.kind, items: [.data.items[] | { metadata: { name: .metadata.name } }] } }"
+    );
+
+    expect(result).toMatchSnapshot();
+  });
+
+  describe("returns original input data", () => {
+    it("on error", async () => {
+      const result = await jqFilter(exampleResponse, "{ { }");
+
+      expect(result).toMatchSnapshot();
+    });
+
+    it("when no filter supplied", async () => {
+      const result = await jqFilter(exampleResponse, undefined);
+
+      expect(result).toMatchSnapshot();
+    });
+  });
+});

client/filter.ts

@@ -0,0 +1,28 @@
+import { isArray } from "lodash";
+
+import { createLogger } from "../log";
+
+const logger = createLogger({ name: "filter" });
+
+// jq-node doesn't support `import` syntax
+const jq = require("node-jq");
+
+export const jpFilter = async (
+  data: any,
+  jqHeader: string | string[] | undefined
+): Promise<any> => {
+  const query = isArray(jqHeader) ? jqHeader[0] : jqHeader;
+  if (!query) {
+    return data;
+  }
+  try {
+    // jq.run() returns a Promise
+    return await jq.run(query, data, { input: "json", output: "json" });
+  } catch (error: any) {
+    logger.error(
+      { error, jpSelectQuery: query },
+      "Error running jq query, ignoring filters"
+    );
+    return data;
+  }
+};

client/index.ts

@@ -4,15 +4,16 @@ import {
   JSONRPCServer,
   JSONRPCServerAndClient,
 } from "json-rpc-2.0";
-import { omit } from "lodash";
+import { isArray, omit } from "lodash";
 import { Logger } from "pino";
 import WebSocket from "ws";
 
 import { DEFAULT_FORWARDED_REQUEST_TIMEOUT_MILLIS } from "../common/constants";
 import { createLogger } from "../log";
-import { ForwardedRequest, ForwardedResponse } from "../types";
+import { ForwardedRequest, ForwardedResponse, JQ_HEADER } from "../types";
 import { deferral } from "../util/deferral";
 import { Backoff } from "./backoff";
+import { jpFilter as jqFilter } from "./filter";
 import { jwt } from "./jwks";
 
 /**
@@ -62,11 +63,7 @@ export class JsonRpcClient {
     });
 
     axios.interceptors.response.use((response) => {
-      // Do not log response object, it's lengthy and difficult to filter out the authorization header
-      this.#logger.debug(
-        { response: omit(response, "request") },
-        "Axios response"
-      );
+      this.#logger.debug({ response }, "Axios response");
       return response;
     });
   }
@@ -134,7 +131,15 @@ export class JsonRpcClient {
     this.#webSocket = clientSocket;
 
     client.addMethod("call", async (request: ForwardedRequest) => {
-      this.#logger.debug({ request }, "forwarded request");
+      this.#logger.debug(
+        {
+          request: {
+            ...request,
+            headers: omit(request.headers, "authorization"),
+          },
+        },
+        "forwarded request"
+      );
       // The headers are modified:
       // 1. The Content-Length header may not be accurate for the forwarded request. By removing it, axios can recalculate the correct length.
       // 2. The Host header should be switched out to the host this client is targeting.
@@ -153,11 +158,15 @@ export class JsonRpcClient {
           request.options?.timeoutMillis ||
           DEFAULT_FORWARDED_REQUEST_TIMEOUT_MILLIS,
       });
+      const jpSelectHeader = request.headers[JQ_HEADER];
+      this.#logger.debug({ response }, "forwarded response before filters");
+      const data = await jqFilter(response.data, jpSelectHeader);
+      this.#logger.debug({ data }, "forwarded response data after filters");
       return {
         headers: response.headers,
         status: response.status,
         statusText: response.statusText,
-        data: response.data,
+        data,
       } as ForwardedResponse;
     });
 

log/index.ts

@@ -22,6 +22,7 @@ export const createLogger = <T extends LoggerOptions>(
     // Redact the authorization header that may contain a secret token
     redact: [
       "response.config.headers.authorization", // Axios intercepted response object
+      "response.request.headers.authorization", // Axios intercepted response object contains the request as well
       "request.headers.authorization", // Axios intercepted request object + forwarded request object (JSON RPC request)
     ],
     formatters: {

package.json

@@ -21,7 +21,7 @@
     "eslint-plugin-import": "^2.25.2",
     "eslint-plugin-n": "^15.0.0",
     "eslint-plugin-promise": "^6.0.0",
-    "husky": "^8.0.2",
+    "husky": "^8.0.3",
     "jest": "29.2.1",
     "jest-extended": "3.2.4",
     "jest-mock": "29.2.1",
@@ -46,6 +46,7 @@
     "json-rpc-2.0": "^1.6.0",
     "json-stream-stringify": "^3.1.0",
     "lodash": "^4.17.21",
+    "node-jq": "^4.2.2",
     "path-to-regexp": "^6.2.1",
     "pino": "^8.14.1",
     "pino-http": "^8.3.3",
@@ -62,7 +63,7 @@
     "start:prod:server": "NODE_PATH=dist node dist/cli server",
     "start:dev:client": "NODE_ENV=development nodemon cli client",
     "start:prod:client": "NODE_PATH=dist node dist/cli client",
-    "prepare": "cd .. && husky install ts/.husky",
+    "prepare": "husky install",
     "format": "prettier --write '**/*.{html,js,ts,json,less,md}'",
     "format:quick": "pretty-quick --branch origin/main"
   },

prettier.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  importOrder: ["^(cli|testing)([/]|$)", "^[.][.]?([/]|$)"],
+  importOrderSeparation: true,
+  importOrderSortSpecifiers: true,
+};

types/index.ts

@@ -2,6 +2,8 @@ import { Request } from "express";
 import core from "express-serve-static-core";
 import { IncomingHttpHeaders } from "node:http";
 
+export const JQ_HEADER = "braekhus-jq-response-filter";
+
 export type IncomingRequest = Request<
   core.ParamsDictionary,
   any,