From d9ef7dd46c2cd584fe2145d933cc48d7752f9803 Mon Sep 17 00:00:00 2001 From: Dan Livings Date: Wed, 25 Sep 2024 14:38:03 +0100 Subject: [PATCH 1/8] Add `better-sqlite3` dependency This will allow Towtruck to use SQLite as a relational database for data storage instead of JSON files. --- package-lock.json | 449 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 450 insertions(+) diff --git a/package-lock.json b/package-lock.json index e04b69d..412e509 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "@octokit/app": "^15.1.0", + "better-sqlite3": "^11.3.0", "date-fns": "^4.1.0", "nunjucks": "^3.2.4", "undici": "^6.19.8" @@ -632,12 +633,43 @@ "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/before-after-hook": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-3.0.2.tgz", "integrity": "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==", "license": "Apache-2.0" }, + "node_modules/better-sqlite3": { + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.3.0.tgz", + "integrity": "sha512-iHt9j8NPYF3oKCNOO5ZI4JwThjt3Z6J6XrcwG85VNMVzv1ByqrHWv5VILEbCMFWDsoHhXvQ7oC8vgRXFAKgl9w==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -651,6 +683,26 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -675,6 +727,30 @@ "node": ">=8" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -740,6 +816,12 @@ "node": ">= 6" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -819,6 +901,30 @@ } } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -826,6 +932,24 @@ "dev": true, "license": "MIT" }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -1006,6 +1130,15 @@ "node": ">=0.10.0" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -1050,6 +1183,12 @@ "node": ">=16.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -1101,6 +1240,12 @@ "dev": true, "license": "ISC" }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1116,6 +1261,12 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1152,6 +1303,26 @@ "node": ">=8" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -1189,6 +1360,18 @@ "node": ">=0.8.19" } }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -1339,6 +1522,18 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "license": "ISC" }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -1352,6 +1547,21 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -1359,6 +1569,12 @@ "dev": true, "license": "MIT" }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "license": "MIT" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -1366,6 +1582,18 @@ "dev": true, "license": "MIT" }, + "node_modules/node-abi": { + "version": "3.68.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.68.0.tgz", + "integrity": "sha512-7vbj10trelExNjFSBm5kTvZXXa7pZyKWx9RCKIyqe6I9Ev3IzGpQoqBP3a+cOdxY+pWj6VkP28n/2wWysBHD/A==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -1401,6 +1629,15 @@ } } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -1544,6 +1781,32 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/prebuild-install": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", + "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -1554,6 +1817,16 @@ "node": ">= 0.8.0" } }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -1585,6 +1858,44 @@ ], "license": "MIT" }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -1643,6 +1954,38 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -1666,6 +2009,60 @@ "node": ">=8" } }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -1705,6 +2102,34 @@ "node": ">=8" } }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -1725,6 +2150,18 @@ "node": ">=8.0" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -1776,6 +2213,12 @@ "punycode": "^2.1.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -1802,6 +2245,12 @@ "node": ">=0.10.0" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 8250334..c9b77b8 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "license": "MIT", "dependencies": { "@octokit/app": "^15.1.0", + "better-sqlite3": "^11.3.0", "date-fns": "^4.1.0", "nunjucks": "^3.2.4", "undici": "^6.19.8" From be46483f2eef6966ec13e87aa03c24448f420f87 Mon Sep 17 00:00:00 2001 From: Dan Livings Date: Thu, 26 Sep 2024 12:49:28 +0100 Subject: [PATCH 2/8] Implement basic key-value database storage mechanism This introduces the `TowtruckDatabase` class, which serves as a wrapper around the `Database` class in `better-sqlite3` and provides utility methods to setup the database schema and interact with it in particular ways. The key-value approach serves as an intermediate step between the current flat file database and a proper relational database - it allows for improved querying of the data while maintaining flexibility of the shape of the data. --- db/index.js | 113 +++++++++++++ db/index.test.js | 418 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 531 insertions(+) create mode 100644 db/index.js create mode 100644 db/index.test.js diff --git a/db/index.js b/db/index.js new file mode 100644 index 0000000..5f8f4db --- /dev/null +++ b/db/index.js @@ -0,0 +1,113 @@ +import Database from "better-sqlite3"; + +const dbSchemaSql = [ + `CREATE TABLE IF NOT EXISTS towtruck_data ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scope TEXT NOT NULL, + name TEXT NOT NULL, + key TEXT NOT NULL, + value JSONB, + UNIQUE (scope, name, key) + );` +]; + +const initialiseDatabase = (filename, options) => { + const db = new Database(filename, options); + db.pragma('journal_mode = WAL'); + + const dbSchemaStatements = dbSchemaSql.map((sql) => db.prepare(sql)); + + db.transaction(() => { + dbSchemaStatements.forEach((statement) => statement.run()); + })(); + + return db; +} + +export class TowtruckDatabase { + #db; + + #saveStatement; + #getStatement; + #getAllForNameStatement; + #getAllForScopeStatement; + + constructor(filename = "./data/towtruck.db", options) { + this.#db = initialiseDatabase(filename, options); + + this.#saveStatement = this.#db.prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?) ON CONFLICT DO UPDATE SET value = excluded.value;"); + this.#getStatement = this.#db.prepare("SELECT value FROM towtruck_data WHERE scope = ? AND name = ? AND key = ?;"); + this.#getAllForNameStatement = this.#db.prepare("SELECT key, value FROM towtruck_data WHERE scope = ? AND name = ?;"); + this.#getAllForScopeStatement = this.#db.prepare("SELECT name, key, value FROM towtruck_data WHERE scope = ?;"); + } + + #save(scope, name, key, data) { + return this.#saveStatement.run(scope, name, key, JSON.stringify(data)); + } + + #get(scope, name, key) { + const json = this.#getStatement.get(scope, name, key)?.value; + + if (json) return JSON.parse(json); + } + + #getAllForName(scope, name) { + const rows = this.#getAllForNameStatement.all(scope, name); + + const result = {}; + rows.forEach((row) => result[row.key] = JSON.parse(row.value)); + + return result; + } + + #getAllForScope(scope) { + const rows = this.#getAllForScopeStatement.all(scope); + + const result = {}; + rows.forEach((row) => { + if (!result[row.name]) { + result[row.name] = {}; + } + + result[row.name][row.key] = JSON.parse(row.value); + }); + + return result; + } + + saveToRepository(name, key, data) { + return this.#save("repository", name, key, data); + } + + getFromRepository(name, key) { + return this.#get("repository", name, key); + } + + getAllFromRepository(name) { + return this.#getAllForName("repository", name); + } + + getAllRepositories() { + return this.#getAllForScope("repository"); + } + + saveToDependency(name, key, data) { + return this.#save("dependency", name, key, data); + } + + getFromDependency(name, key) { + return this.#get("dependency", name, key); + } + + getAllFromDependency(name) { + return this.#getAllForName("dependency", name); + } + + getAllDependencies() { + return this.#getAllForScope("dependency"); + } + + transaction(fn) { + return this.#db.transaction(fn); + } +}; diff --git a/db/index.test.js b/db/index.test.js new file mode 100644 index 0000000..d6bb602 --- /dev/null +++ b/db/index.test.js @@ -0,0 +1,418 @@ +import { afterEach, before, describe, it } from "node:test"; +import expect from "node:assert"; +import { TowtruckDatabase } from "./index.js"; +import { mkdir, unlink } from "node:fs/promises"; +import Database from "better-sqlite3"; +import { dirname } from "node:path"; + +const testDbPath = "./data/test.db"; + +const deleteFile = async (path) => { + try { + await unlink(path); + } catch (e) { + if (e.code === "ENOENT") { + return; + } + + throw e; + } +} + +describe("TowtruckDatabase", () => { + before(async () => await mkdir(dirname(testDbPath), { recursive: true })); + + afterEach(async () => { + await Promise.all([ + deleteFile(testDbPath), + deleteFile(`${testDbPath}-wal`), + deleteFile(`${testDbPath}-shm`), + ]); + }); + + describe("constructor", () => { + it("ensures database is set up correctly", () => { + const log = []; + new TowtruckDatabase(testDbPath, { verbose: (str) => log.push(str)}); + + expect(log.find(str => str === "PRAGMA journal_mode = WAL"), "Journal mode should be set to WAL."); + expect(log.find(str => str.includes("CREATE TABLE IF NOT EXISTS towtruck_data")), "A `towtruck_data` table schema should be defined."); + }); + }); + + describe("saveToRepository", () => { + it("inserts the expected data into the table", () => { + const db = new TowtruckDatabase(testDbPath); + + const data = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + const expected = { + id: 1, + scope: "repository", + name: "test-repo", + key: "some-data", + value: JSON.stringify(data), + }; + + const result = db.saveToRepository("test-repo", "some-data", data); + const actual = new Database(testDbPath).prepare("SELECT * FROM towtruck_data WHERE ROWID = ?;").get(result.lastInsertRowid); + + expect.deepStrictEqual(actual, expected); + }); + + it("updates the row if the data already exists", () => { + const db = new TowtruckDatabase(testDbPath); + + const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);"); + const original = insertStatement.run("repository", "test-repo", "some-data", JSON.stringify({})); + + const data = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + const expected = { + id: 1, + scope: "repository", + name: "test-repo", + key: "some-data", + value: JSON.stringify(data), + }; + + db.saveToRepository("test-repo", "some-data", data); + const actual = new Database(testDbPath).prepare("SELECT * FROM towtruck_data WHERE ROWID = ?;").get(original.lastInsertRowid); + + expect.deepStrictEqual(actual, expected); + }); + }); + + describe("getFromRepository", () => { + it("retrieves the expected data from the table", () => { + const db = new TowtruckDatabase(testDbPath); + + const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);"); + + const expected = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + insertStatement.run("repository", "test-repo", "some-data", JSON.stringify(expected)); + insertStatement.run("repository", "test-repo", "some-other-data", JSON.stringify({})); + insertStatement.run("repository", "another-repo", "some-data", JSON.stringify({})); + insertStatement.run("dependency", "test-dependency", "some-data", JSON.stringify({})); + + const actual = db.getFromRepository("test-repo", "some-data"); + + expect.deepStrictEqual(actual, expected); + }); + + it("returns undefined when the given key doesn't exist", () => { + const db = new TowtruckDatabase(testDbPath); + + const actual = db.getFromRepository("test-repo", "some-data"); + + expect.strictEqual(actual, undefined); + }); + }); + + describe("getAllFromRepository", () => { + it("retrieves the expected data from the table", () => { + const db = new TowtruckDatabase(testDbPath); + + const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);"); + + const someData = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + const someOtherData = { + foo: "bar", + baz: false, + quux: 0.123456789, + }; + + const expected = { + "some-data": someData, + "some-other-data": someOtherData, + }; + + insertStatement.run("repository", "test-repo", "some-data", JSON.stringify(someData)); + insertStatement.run("repository", "test-repo", "some-other-data", JSON.stringify(someOtherData)); + insertStatement.run("repository", "another-repo", "some-data", JSON.stringify({})); + insertStatement.run("dependency", "test-dependency", "some-data", JSON.stringify({})); + + const actual = db.getAllFromRepository("test-repo"); + + expect.deepStrictEqual(actual, expected); + }); + + it("returns an empty object when the given repository doesn't exist", () => { + const db = new TowtruckDatabase(testDbPath); + + const actual = db.getAllFromRepository("test-repo"); + + expect.deepStrictEqual(actual, {}); + }); + }); + + describe("getAllRepositories", () => { + it("retrieves the expected data from the table", () => { + const db = new TowtruckDatabase(testDbPath); + + const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);"); + + const testRepoSomeData = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + const testRepoSomeOtherData = { + foo: "bar", + baz: false, + quux: 0.123456789, + }; + + const anotherRepoSomeData = [1, "foo", true, null]; + + const expected = { + "test-repo": { + "some-data": testRepoSomeData, + "some-other-data": testRepoSomeOtherData, + }, + "another-repo": { + "some-data": anotherRepoSomeData, + }, + }; + + insertStatement.run("repository", "test-repo", "some-data", JSON.stringify(testRepoSomeData)); + insertStatement.run("repository", "test-repo", "some-other-data", JSON.stringify(testRepoSomeOtherData)); + insertStatement.run("repository", "another-repo", "some-data", JSON.stringify(anotherRepoSomeData)); + insertStatement.run("dependency", "test-dependency", "some-data", JSON.stringify({})); + + const actual = db.getAllRepositories(); + + expect.deepStrictEqual(actual, expected); + }); + + it("returns an empty object when no repositories exist", () => { + const db = new TowtruckDatabase(testDbPath); + + const actual = db.getAllRepositories(); + + expect.deepStrictEqual(actual, {}); + }); + }); + + describe("saveToDependency", () => { + it("inserts the expected data into the table", () => { + const db = new TowtruckDatabase(testDbPath); + + const data = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + const expected = { + id: 1, + scope: "dependency", + name: "test-dep", + key: "some-data", + value: JSON.stringify(data), + }; + + const result = db.saveToDependency("test-dep", "some-data", data); + const actual = new Database(testDbPath).prepare("SELECT * FROM towtruck_data WHERE ROWID = ?;").get(result.lastInsertRowid); + + expect.deepStrictEqual(actual, expected); + }); + + it("updates the row if the data already exists", () => { + const db = new TowtruckDatabase(testDbPath); + + const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);"); + const original = insertStatement.run("dependency", "test-dep", "some-data", JSON.stringify({})); + + const data = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + const expected = { + id: 1, + scope: "dependency", + name: "test-dep", + key: "some-data", + value: JSON.stringify(data), + }; + + db.saveToDependency("test-dep", "some-data", data); + const actual = new Database(testDbPath).prepare("SELECT * FROM towtruck_data WHERE ROWID = ?;").get(original.lastInsertRowid); + + expect.deepStrictEqual(actual, expected); + }); + }); + + describe("getFromDependency", () => { + it("retrieves the expected data from the table", () => { + const db = new TowtruckDatabase(testDbPath); + + const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);"); + + const expected = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + insertStatement.run("dependency", "test-dep", "some-data", JSON.stringify(expected)); + insertStatement.run("dependency", "test-dep", "some-other-data", JSON.stringify({})); + insertStatement.run("dependency", "another-dep", "some-data", JSON.stringify({})); + insertStatement.run("repository", "test-repo", "some-data", JSON.stringify({})); + + const actual = db.getFromDependency("test-dep", "some-data"); + + expect.deepStrictEqual(actual, expected); + }); + + it("returns undefined when the given key doesn't exist", () => { + const db = new TowtruckDatabase(testDbPath); + + const actual = db.getFromDependency("test-dep", "some-data"); + + expect.strictEqual(actual, undefined); + }); + }); + + describe("getAllFromDependency", () => { + it("retrieves the expected data from the table", () => { + const db = new TowtruckDatabase(testDbPath); + + const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);"); + + const someData = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + const someOtherData = { + foo: "bar", + baz: false, + quux: 0.123456789, + }; + + const expected = { + "some-data": someData, + "some-other-data": someOtherData, + }; + + insertStatement.run("dependency", "test-dep", "some-data", JSON.stringify(someData)); + insertStatement.run("dependency", "test-dep", "some-other-data", JSON.stringify(someOtherData)); + insertStatement.run("dependency", "another-dep", "some-data", JSON.stringify({})); + insertStatement.run("repository", "test-repository", "some-data", JSON.stringify({})); + + const actual = db.getAllFromDependency("test-dep"); + + expect.deepStrictEqual(actual, expected); + }); + + it("returns an empty object when the given dependency doesn't exist", () => { + const db = new TowtruckDatabase(testDbPath); + + const actual = db.getAllFromDependency("test-dep"); + + expect.deepStrictEqual(actual, {}); + }); + }); + + describe("getAllDependencies", () => { + it("retrieves the expected data from the table", () => { + const db = new TowtruckDatabase(testDbPath); + + const insertStatement = new Database(testDbPath).prepare("INSERT INTO towtruck_data (scope, name, key, value) VALUES (?, ?, ?, ?);"); + + const testDepSomeData = { + array: [1, 2, 3], + text: "Text", + object: { + boolean: true, + missing: null, + }, + }; + + const testDepSomeOtherData = { + foo: "bar", + baz: false, + quux: 0.123456789, + }; + + const anotherDepSomeData = [1, "foo", true, null]; + + const expected = { + "test-dep": { + "some-data": testDepSomeData, + "some-other-data": testDepSomeOtherData, + }, + "another-dep": { + "some-data": anotherDepSomeData, + }, + }; + + insertStatement.run("dependency", "test-dep", "some-data", JSON.stringify(testDepSomeData)); + insertStatement.run("dependency", "test-dep", "some-other-data", JSON.stringify(testDepSomeOtherData)); + insertStatement.run("dependency", "another-dep", "some-data", JSON.stringify(anotherDepSomeData)); + insertStatement.run("repository", "test-repository", "some-data", JSON.stringify({})); + + const actual = db.getAllDependencies(); + + expect.deepStrictEqual(actual, expected); + }); + + it("returns an empty object when no dependencies exist", () => { + const db = new TowtruckDatabase(testDbPath); + + const actual = db.getAllDependencies(); + + expect.deepStrictEqual(actual, {}); + }); + }); +}); From 58f02bd7968e362c2c64fd5b4bdd235c7a808a90 Mon Sep 17 00:00:00 2001 From: Dan Livings Date: Thu, 26 Sep 2024 12:54:23 +0100 Subject: [PATCH 3/8] Add 'SQLite3 Editor' VS Code extension to recommendations --- .vscode/extensions.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index d995987..09fcadb 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,3 +1,7 @@ { - "recommendations": ["esbenp.prettier-vscode", "ms-playwright.playwright"] + "recommendations": [ + "esbenp.prettier-vscode", + "ms-playwright.playwright", + "yy0931.vscode-sqlite3-editor" + ] } From f4524804736e2fb58bd68b7f935f7498db94830c Mon Sep 17 00:00:00 2001 From: Dan Livings Date: Thu, 26 Sep 2024 16:47:27 +0100 Subject: [PATCH 4/8] Migrate seeding scripts to use new SQLite database interface This uses the SQLite database to store both dependency lifetimes and repo information using the key-value table in a way that introduces minimal changes to the structure of the persisted data, which aids the migration both for seeding and for retrieving and presenting the data. --- .../fetchAllDependencyEolInfo.js | 28 +++++++---------- utils/githubApi/fetchAllRepos.js | 31 ++++++------------- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js b/utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js index c96c318..dfc0eab 100644 --- a/utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js +++ b/utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js @@ -1,7 +1,5 @@ import { EndOfLifeDateApiClient } from "./index.js"; -import { readFromJsonFile } from "../index.js"; -import { writeFile, mkdir } from "fs/promises"; -import path from "path"; +import { TowtruckDatabase } from "../../db/index.js"; /** * @typedef {Object} DependencyLifetimes @@ -16,12 +14,12 @@ import path from "path"; * @returns {Promise} */ const fetchAllDependencyLifetimes = async () => { - const pathToRepos = "./data/repos.json"; - const persistedRepoData = await readFromJsonFile(pathToRepos); + const db = new TowtruckDatabase(); + const persistedRepoData = db.getAllRepositories(); const dependencySet = new Set(); - persistedRepoData.repos - .flatMap((repo) => repo.dependencies) + Object.entries(persistedRepoData) + .flatMap(([, repo]) => repo.main.dependencies) .forEach((dependency) => dependencySet.add(dependency.name)); const apiClient = new EndOfLifeDateApiClient(); @@ -48,21 +46,17 @@ const fetchAllDependencyLifetimes = async () => { */ const saveAllDependencyLifetimes = async () => { console.info("Fetching all dependency EOL info..."); - const lifetimes = await fetchAllDependencyLifetimes(); + const allLifetimes = await fetchAllDependencyLifetimes(); try { - const dir = path.dirname("./data/lifetimes.json"); - await mkdir(dir, { recursive: true }); + const db = new TowtruckDatabase(); console.info("Saving all dependency EOL info..."); - const toSave = { - lifetimes, - }; - - await writeFile("./data/lifetimes.json", JSON.stringify(toSave), { - encoding: "utf-8", - flag: "w", + const saveAllLifetimes = db.transaction((lifetimes) => { + lifetimes.forEach((item) => db.saveToDependency(item.dependency, "lifetimes", item.lifetimes)) }); + + saveAllLifetimes(allLifetimes); } catch (error) { console.error("Error saving all dependency EOL info", error); } diff --git a/utils/githubApi/fetchAllRepos.js b/utils/githubApi/fetchAllRepos.js index 9171e44..6b0cb94 100644 --- a/utils/githubApi/fetchAllRepos.js +++ b/utils/githubApi/fetchAllRepos.js @@ -1,11 +1,10 @@ import { OctokitApp } from "../../octokitApp.js"; -import { writeFile, mkdir } from "fs/promises"; import { mapRepoFromApiForStorage } from "../index.js"; -import path from "path"; import { getDependenciesForRepo } from "../renovate/dependencyDashboard.js"; import { getOpenPRsForRepo } from "./fetchOpenPrs.js"; import { getOpenIssuesForRepo } from "./fetchOpenIssues.js"; import { getDependabotAlertsForRepo } from "./fetchDependabotAlerts.js"; +import { TowtruckDatabase } from "../../db/index.js"; /** * @typedef {import('../index.js').StoredRepo} StoredRepo @@ -52,35 +51,25 @@ const fetchAllRepos = async () => { return repos; }; -/** - * Fetches installation data for the app from the GitHub API - * @returns {Object} - */ -const installationOctokit = await OctokitApp.app.octokit.request( - "GET /app/installations" -); - /** * Saves all repos to a JSON file */ const saveAllRepos = async () => { console.info("Fetching all repos..."); - const repos = await fetchAllRepos(); + const allRepos = await fetchAllRepos(); try { - const dir = path.dirname("./data/repos.json"); - await mkdir(dir, { recursive: true }); + const db = new TowtruckDatabase(); console.info("Saving all repos..."); - const toSave = { - org: installationOctokit.data[0].account.login, - repos, - }; - - await writeFile("./data/repos.json", JSON.stringify(toSave), { - encoding: "utf-8", - flag: "w", + const saveAllRepos = db.transaction((repos) => { + repos.forEach((repo) => { + db.saveToRepository(repo.name, "main", repo); + db.saveToRepository(repo.name, "owner", repo.owner); + }); }); + + saveAllRepos(allRepos); } catch (error) { console.error("Error saving all repos", error); } From d46db7ac3fa28e173ca5fac2344aa2e004521535 Mon Sep 17 00:00:00 2001 From: Dan Livings Date: Fri, 27 Sep 2024 11:55:28 +0100 Subject: [PATCH 5/8] Update view model logic to use data persisted in SQLite --- index.js | 10 +-- utils/index.js | 46 ++++------ utils/index.test.js | 204 +++++++++++++++++++++++--------------------- 3 files changed, 128 insertions(+), 132 deletions(-) diff --git a/index.js b/index.js index 2312a57..b83861d 100644 --- a/index.js +++ b/index.js @@ -1,8 +1,9 @@ import { createServer } from "http"; import nunjucks from "nunjucks"; -import { readFromJsonFile, mapRepoFromStorageToUi } from "./utils/index.js"; +import { mapRepoFromStorageToUi } from "./utils/index.js"; import { getQueryParams } from "./utils/queryParams.js"; import { sortByType } from "./utils/sorting.js"; +import { TowtruckDatabase } from "./db/index.js"; nunjucks.configure({ autoescape: true, @@ -17,10 +18,9 @@ const httpServer = createServer(async (request, response) => { return response.end(); } - const [ persistedRepoData, persistedLifetimeData ] = await Promise.all([ - readFromJsonFile("./data/repos.json"), - readFromJsonFile("./data/lifetimes.json"), - ]); + const db = new TowtruckDatabase(); + const persistedRepoData = db.getAllRepositories(); + const persistedLifetimeData = db.getAllDependencies(); const reposForUi = mapRepoFromStorageToUi(persistedRepoData, persistedLifetimeData); diff --git a/utils/index.js b/utils/index.js index e46770e..9dab8f4 100644 --- a/utils/index.js +++ b/utils/index.js @@ -1,4 +1,3 @@ -import { readFile } from "fs/promises"; import { differenceInYears, formatDistance, formatDistanceToNow, startOfToday } from "date-fns"; import { getDependencyEndOfLifeDate, getDependencyState } from "./endOfLifeDateApi/index.js"; @@ -64,7 +63,7 @@ const defaultIcon = "bx-question-mark"; * @returns {UiDependency} */ export const mapDependencyFromStorageToUi = (dependency, persistedLifetimes) => { - const lifetimes = persistedLifetimes.lifetimes.find((item) => item.dependency === dependency.name); + const lifetimes = Object.entries(persistedLifetimes).find(([name]) => name === dependency.name)?.[1]; const state = lifetimes === undefined ? "unknown" : getDependencyState(dependency, lifetimes.lifetimes); const latestVersion = lifetimes?.lifetimes[0]?.latest; @@ -139,49 +138,37 @@ export const hashToTailwindColor = (str) => { * @returns {RepoData} */ export const mapRepoFromStorageToUi = (persistedData, persistedLifetimes) => { - const mappedRepos = persistedData.repos.map((repo) => { - const newDate = new Date(repo.updatedAt).toLocaleDateString(); - const dependencies = repo.dependencies.map((dependency) => mapDependencyFromStorageToUi(dependency, persistedLifetimes)); + const mappedRepos = Object.entries(persistedData).map(([, repo]) => { + const newDate = new Date(repo.main.updatedAt).toLocaleDateString(); + const dependencies = repo.main.dependencies.map((dependency) => mapDependencyFromStorageToUi(dependency, persistedLifetimes)); - const mostRecentPrOpenedAt = repo.mostRecentPrOpenedAt && formatDistanceToNow(repo.mostRecentPrOpenedAt, { addSuffix: true }); - const oldestOpenPrOpenedAt = repo.oldestOpenPrOpenedAt && formatDistanceToNow(repo.oldestOpenPrOpenedAt, { addSuffix: true }); - const mostRecentIssueOpenedAt = repo.mostRecentIssueOpenedAt && formatDistanceToNow(repo.mostRecentIssueOpenedAt, { addSuffix: true }); - const oldestOpenIssueOpenedAt = repo.oldestOpenIssueOpenedAt && formatDistanceToNow(repo.oldestOpenIssueOpenedAt, { addSuffix: true }); + const mostRecentPrOpenedAt = repo.main.mostRecentPrOpenedAt && formatDistanceToNow(repo.main.mostRecentPrOpenedAt, { addSuffix: true }); + const oldestOpenPrOpenedAt = repo.main.oldestOpenPrOpenedAt && formatDistanceToNow(repo.main.oldestOpenPrOpenedAt, { addSuffix: true }); + const mostRecentIssueOpenedAt = repo.main.mostRecentIssueOpenedAt && formatDistanceToNow(repo.main.mostRecentIssueOpenedAt, { addSuffix: true }); + const oldestOpenIssueOpenedAt = repo.main.oldestOpenIssueOpenedAt && formatDistanceToNow(repo.main.oldestOpenIssueOpenedAt, { addSuffix: true }); - const languageColor = hashToTailwindColor(repo.language); + const languageColor = hashToTailwindColor(repo.main.language); return { - ...repo, + ...repo.main, updatedAt: newDate, - updatedAtISO8601: repo.updatedAt, + updatedAtISO8601: repo.main.updatedAt, dependencies, mostRecentPrOpenedAt, - mostRecentPrOpenedAtISO8601: repo.mostRecentPrOpenedAt, + mostRecentPrOpenedAtISO8601: repo.main.mostRecentPrOpenedAt, oldestOpenPrOpenedAt, - oldestOpenPrOpenedAtISO8601: repo.oldestOpenPrOpenedAt, + oldestOpenPrOpenedAtISO8601: repo.main.oldestOpenPrOpenedAt, mostRecentIssueOpenedAt, - mostRecentIssueOpenedAtISO8601: repo.mostRecentIssueOpenedAt, + mostRecentIssueOpenedAtISO8601: repo.main.mostRecentIssueOpenedAt, oldestOpenIssueOpenedAt, - oldestOpenIssueOpenedAtISO8601: repo.oldestOpenIssueOpenedAt, + oldestOpenIssueOpenedAtISO8601: repo.main.oldestOpenIssueOpenedAt, languageColor, }; }); const totalRepos = mappedRepos.length; - return { ...persistedData, repos: mappedRepos, totalRepos }; -}; - -/** - * Reads data from a JSON file - * @param {string} filePath - The path to the file to read from - * @returns {any} - */ -export const readFromJsonFile = async (filePath) => { - const json = await readFile(filePath, { encoding: "utf-8" }); - const persistedData = JSON.parse(json); - - return persistedData; + return { org: Object.entries(persistedData)[0][1].owner, repos: mappedRepos, totalRepos }; }; /** @@ -226,6 +213,7 @@ export const readFromJsonFile = async (filePath) => { */ export const mapRepoFromApiForStorage = (repo) => ({ name: repo.name, + owner: repo.owner.login, description: repo.description, htmlUrl: repo.html_url, apiUrl: repo.url, diff --git a/utils/index.test.js b/utils/index.test.js index e75ee9b..bec6546 100644 --- a/utils/index.test.js +++ b/utils/index.test.js @@ -5,28 +5,27 @@ import { formatDistanceToNow } from "date-fns"; describe("mapRepoFromStorageToUi", () => { it("converts ISO8601 timestamps to human-readable forms", () => { - const storedRepos = [ - { - name: "repo1", - description: "description1", - updatedAt: "2021-01-01T00:00:00Z", - htmlUrl: "http://url.com/repo1", - apiUrl: "http://api.com/repo1", - pullsUrl: "http://api.com/repo1/pulls", - issuesUrl: "http://api.com/repo1/issues", - language: null, - topics: [], - openIssues: 0, - dependencies: [], - mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", - oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", - mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", - oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", - }, - ]; - const persistedData = { - repos: storedRepos, + repo1: { + owner: "dxw", + main: { + name: "repo1", + description: "description1", + updatedAt: "2021-01-01T00:00:00Z", + htmlUrl: "http://url.com/repo1", + apiUrl: "http://api.com/repo1", + pullsUrl: "http://api.com/repo1/pulls", + issuesUrl: "http://api.com/repo1/issues", + language: null, + topics: [], + openIssues: 0, + dependencies: [], + mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", + oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", + oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", + }, + }, }; const expected = [ @@ -59,45 +58,47 @@ describe("mapRepoFromStorageToUi", () => { }); it("converts the language string to a Tailwind colour", () => { - const storedRepos = [ - { - name: "repo1", - description: "description1", - updatedAt: "2021-01-01T00:00:00Z", - htmlUrl: "http://url.com/repo1", - apiUrl: "http://api.com/repo1", - pullsUrl: "http://api.com/repo1/pulls", - issuesUrl: "http://api.com/repo1/issues", - language: "Ruby", - topics: [], - openIssues: 0, - dependencies: [], - mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", - oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", - mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", - oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", + const persistedData = { + repo1: { + owner: "dxw", + main: { + name: "repo1", + description: "description1", + updatedAt: "2021-01-01T00:00:00Z", + htmlUrl: "http://url.com/repo1", + apiUrl: "http://api.com/repo1", + pullsUrl: "http://api.com/repo1/pulls", + issuesUrl: "http://api.com/repo1/issues", + language: "Ruby", + topics: [], + openIssues: 0, + dependencies: [], + mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", + oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", + oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", + }, }, - { - name: "repo2", - description: "description2", - updatedAt: "2021-01-01T00:00:00Z", - htmlUrl: "http://url.com/repo2", - apiUrl: "http://api.com/repo2", - pullsUrl: "http://api.com/repo2/pulls", - issuesUrl: "http://api.com/repo2/issues", - language: "TypeScript", - topics: [], - openIssues: 0, - dependencies: [], - mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", - oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", - mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", - oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", + repo2: { + owner: "dxw", + main: { + name: "repo2", + description: "description2", + updatedAt: "2021-01-01T00:00:00Z", + htmlUrl: "http://url.com/repo2", + apiUrl: "http://api.com/repo2", + pullsUrl: "http://api.com/repo2/pulls", + issuesUrl: "http://api.com/repo2/issues", + language: "TypeScript", + topics: [], + openIssues: 0, + dependencies: [], + mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", + oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", + oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", + }, }, - ]; - - const persistedData = { - repos: storedRepos, }; const expected = [ @@ -153,49 +154,55 @@ describe("mapRepoFromStorageToUi", () => { }); it("returns a count of the number of repos", () => { - const storedRepos = [ - { - name: "repo1", - description: "description1", - updatedAt: "2021-01-01T00:00:00Z", - htmlUrl: "http://url.com/repo1", - apiUrl: "http://api.com/repo1", - pullsUrl: "http://api.com/repo1/pulls", - issuesUrl: "http://api.com/repo1/issues", - language: null, - topics: [], - openIssues: 0, - dependencies: [], + const persistedData = { + repo1: { + owner: "dxw", + main: { + name: "repo1", + description: "description1", + updatedAt: "2021-01-01T00:00:00Z", + htmlUrl: "http://url.com/repo1", + apiUrl: "http://api.com/repo1", + pullsUrl: "http://api.com/repo1/pulls", + issuesUrl: "http://api.com/repo1/issues", + language: null, + topics: [], + openIssues: 0, + dependencies: [], + }, }, - { - name: "repo2", - description: "description2", - updatedAt: "2021-01-01T00:00:00Z", - htmlUrl: "http://url.com/repo2", - apiUrl: "http://api.com/repo2", - pullsUrl: "http://api.com/repo2/pulls", - issuesUrl: "http://api.com/repo2/issues", - language: null, - topics: [], - openIssues: 0, - dependencies: [], + repo2: { + owner: "dxw", + main: { + name: "repo2", + description: "description2", + updatedAt: "2021-01-01T00:00:00Z", + htmlUrl: "http://url.com/repo2", + apiUrl: "http://api.com/repo2", + pullsUrl: "http://api.com/repo2/pulls", + issuesUrl: "http://api.com/repo2/issues", + language: null, + topics: [], + openIssues: 0, + dependencies: [], + }, }, - { - name: "repo3", - description: "description3", - updatedAt: "2021-01-01T00:00:00Z", - htmlUrl: "http://url.com/repo3", - apiUrl: "http://api.com/repo3", - pullsUrl: "http://api.com/repo3/pulls", - issuesUrl: "http://api.com/repo3/issues", - language: null, - topics: [], - openIssues: 0, - dependencies: [], + repo3: { + owner: "dxw", + main: { + name: "repo3", + description: "description3", + updatedAt: "2021-01-01T00:00:00Z", + htmlUrl: "http://url.com/repo3", + apiUrl: "http://api.com/repo3", + pullsUrl: "http://api.com/repo3/pulls", + issuesUrl: "http://api.com/repo3/issues", + language: null, + topics: [], + openIssues: 0, + dependencies: [], + }, }, - ]; - const persistedData = { - repos: storedRepos, }; expect.deepEqual(mapRepoFromStorageToUi(persistedData).totalRepos, 3); @@ -353,6 +360,7 @@ describe("mapRepoFromStorageToUi", () => { const repoToSave = { name: "security-alert-notifier", + owner: "dxw", description: "Icinga plugin to fetch security vulnerabilities for a GitHub organization.", htmlUrl: "https://github.com/dxw/security-alert-notifier", From c834800b89e861b9e0b1e88c662a0ae46a3d1e80 Mon Sep 17 00:00:00 2001 From: Dan Livings Date: Fri, 27 Sep 2024 13:44:53 +0100 Subject: [PATCH 6/8] Update Playwright setup to copy example SQLite DB to data folder --- .gitignore | 4 +++ e2es/seedTestData.js | 3 +-- e2es/testData/lifetimes.json | 3 --- e2es/testData/repos.json | 50 ----------------------------------- e2es/testData/towtruck.db | Bin 0 -> 16384 bytes 5 files changed, 5 insertions(+), 55 deletions(-) delete mode 100644 e2es/testData/lifetimes.json delete mode 100644 e2es/testData/repos.json create mode 100644 e2es/testData/towtruck.db diff --git a/.gitignore b/.gitignore index d7bc316..b39fd6b 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,7 @@ towtruck.private-key.pem /playwright-report/ /blob-report/ /playwright/.cache/ + +# SQLite +*.db-wal +*.db-shm diff --git a/e2es/seedTestData.js b/e2es/seedTestData.js index b5eff69..553808d 100755 --- a/e2es/seedTestData.js +++ b/e2es/seedTestData.js @@ -8,8 +8,7 @@ export const createDestinationDirectory = async () => { export const seedTestData = async () => { console.info("Seeding test data..."); await Promise.all([ - copyFile("./e2es/testData/repos.json", "./data/repos.json"), - copyFile("./e2es/testData/lifetimes.json", "./data/lifetimes.json"), + copyFile("./e2es/testData/towtruck.db", "./data/towtruck.db"), ]); }; diff --git a/e2es/testData/lifetimes.json b/e2es/testData/lifetimes.json deleted file mode 100644 index a991ff5..0000000 --- a/e2es/testData/lifetimes.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "lifetimes": [] -} diff --git a/e2es/testData/repos.json b/e2es/testData/repos.json deleted file mode 100644 index 750b656..0000000 --- a/e2es/testData/repos.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "org": "dxw", - "repos": [ - { - "name": "optionparser", - "description": "Command-line option parser for PHP (forked with a `v1.0.0`)", - "htmlUrl": "https://github.com/dxw/optionparser", - "apiUrl": "https://api.github.com/repos/dxw/optionparser", - "pullsUrl": "https://api.github.com/repos/dxw/optionparser/pulls{/number}", - "issuesUrl": "https://api.github.com/repos/dxw/optionparser/issues{/number}", - "updatedAt": "2024-07-08T12:39:14Z", - "language": "PHP", - "topics": ["composer", "govpress", "packagist", "php"], - "openIssues": 10, - "dependencies": [], - "openPrCount": 0, - "openBotPrCount": 0 - }, - { - "name": "govuk-blogs", - "description": "This is the theme in use for the blogs hosted at blog.gov.uk.", - "htmlUrl": "https://github.com/dxw/govuk-blogs", - "apiUrl": "https://api.github.com/repos/dxw/govuk-blogs", - "pullsUrl": "https://api.github.com/repos/dxw/govuk-blogs/pulls{/number}", - "issuesUrl": "https://api.github.com/repos/dxw/govuk-blogs/issues{/number}", - "updatedAt": "2024-09-04T09:02:48Z", - "language": "PHP", - "topics": ["govpress", "wordpress-theme"], - "openIssues": 1, - "dependencies": [], - "openPrCount": 20, - "openBotPrCount": 10 - }, - { - "name": "php-missing", - "description": "Some functions missing from PHP's stdlib", - "htmlUrl": "https://github.com/dxw/php-missing", - "apiUrl": "https://api.github.com/repos/dxw/php-missing", - "pullsUrl": "https://api.github.com/repos/dxw/php-missing/pulls{/number}", - "issuesUrl": "https://api.github.com/repos/dxw/php-missing/issues{/number}", - "updatedAt": "2024-07-18T18:12:43Z", - "language": "PHP", - "topics": ["composer", "govpress", "packagist", "php"], - "openIssues": 7, - "dependencies": [], - "openPrCount": 200, - "openBotPrCount": 3 - } - ] -} diff --git a/e2es/testData/towtruck.db b/e2es/testData/towtruck.db new file mode 100644 index 0000000000000000000000000000000000000000..4bdda52ecce83756b5d4485ebad57286c15e9b33 GIT binary patch literal 16384 zcmeI3&u`;I6vv&U&6Y}C!(kP1V(_Y@)i#NfR%#pVfwYPs(5Bhe(Ncb>>%`N`^CY!r=FLlcGjz#?o3pS%?KD`L2(@ho^_K)i| ztxmMs-a(xtY}Xv!rW@e7{!Kr1E0@owzLiBx&W79zx`Q)A=lWP7Jr-}QiL5OXNTxv= zO|AYx{g||l8;7;yQ}S{Bl+@bVQKJdohxMkmAuRnab1Bj4pJ=3c1i5{15H*CunpQuc zPo`C0>Q+F>M<++ky~w}aY&>Y!$(rb3gNU;v;JgmquJ5L^xx;))G980Hz6PQPoQd+; z*cqLb$BG${TDZTLRv>4WxFr8B|0MsgLY9Ap0wyGY1dsp{Kmter3EVJ&XGhsv(z|!> zd^hl@%Y2hF?}^)Y3%2R|rqi=^)477>jS?<)lqykTZA!Q*QVi;MJ=5hTbHer$25YSI z!0Cn+KDiD<&OK(6)`zXUPke4zW+!s)bK7crR_NI0-1V!aQqSc5pi}HJyJS2bmL~es zT{GbUHN}{hIFISRZeUsdYj~H0<5kHC><;ywMq>v+0e!7#k!K=Y;KHJq8Z{omsHl~# zg1TKO?`Y+nYPnL~+Wa)?)zY0_p!dYJ0G?4LXRg`x!SOQ%xWQ$lp0Yud9=mis>iglb zt9J)_&-A${^j+l(arI!O>@>uQfcN(G1%)~T*8C z1YVql#gWfhdwTL5LP-ueFRk+PVWd0gEC%}+=x-Z)iT`^>g=$0%5) zLrG*!qFCWu*l@xkYj7|CVGUtkB070=SuUzY_0jsQ%9lX7@_qTo8@}CRI3$1skN^@u0!RP} zAOR$R1dsp{Kmz}20`DxsL+sSl{d)LVD4T4LZPdf|MY0GFmSX<#9eQZ600wt+F@xj- zflSgM)`QsMr^7CCZ;XZ*??PCF@BdTspAvj9Aps Date: Fri, 27 Sep 2024 13:03:46 +0100 Subject: [PATCH 7/8] Fix GitHub Actions workflows to use correct node version The NODE_VERSION variable was not being correctly set, so the `setup-node` action was defaulting to the latest LTS version (20.17.0). This was causing `better-sqlite3` problems as it relies on native code which does not target that version. --- .github/workflows/playwright.yml | 8 ++++++-- .github/workflows/test.yml | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 2e33704..42dc9e5 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -10,9 +10,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - name: Read .nvmrc + run: echo "NODE_VERSION=$(cat .nvmrc)" >> $GITHUB_OUTPUT + id: nvm + - name: Use Node.js ${{ steps.nvm.outputs.NODE_VERSION }} + uses: actions/setup-node@v4 with: - node-version: lts/* + node-version: ${{ steps.nvm.outputs.NODE_VERSION }} - name: Install dependencies run: script/setup - name: Install Playwright Browsers diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d53bfad..8d15738 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,15 +12,15 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v4 - name: Read .nvmrc - run: echo "{NODE_VERSION}={$(cat .nvmrc)}" >> $GITHUB_OUTPUT + run: echo "NODE_VERSION=$(cat .nvmrc)" >> $GITHUB_OUTPUT id: nvm - name: Use Node.js ${{ steps.nvm.outputs.NODE_VERSION }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: node-version: ${{ steps.nvm.outputs.NODE_VERSION }} - - name: Checkout repository - uses: actions/checkout@v4 - name: Install dependencies run: | ./script/setup From 95df3c110e7b69fbfe6e4d920d120ab5fb56d6bb Mon Sep 17 00:00:00 2001 From: Dan Livings Date: Tue, 1 Oct 2024 14:39:07 +0100 Subject: [PATCH 8/8] Extract data from each GitHub API call into separate DB keyed value This will make it easier to update individual pieces of data in the future, such as via a webhook. --- e2es/testData/towtruck.db | Bin 16384 -> 16384 bytes .../fetchAllDependencyEolInfo.js | 2 +- utils/githubApi/fetchAllRepos.js | 15 +++-- utils/index.js | 21 ++++--- utils/index.test.js | 54 +++++++++++++++--- 5 files changed, 68 insertions(+), 24 deletions(-) diff --git a/e2es/testData/towtruck.db b/e2es/testData/towtruck.db index 4bdda52ecce83756b5d4485ebad57286c15e9b33..7c112129059dba5c3cfd147bcebe87942f45d10b 100644 GIT binary patch literal 16384 zcmeHOUvJw~6nBy)ZL=j_yLHhX&@QREwygQr7TQKU&{Yu>y0&XmiLpPO8{Z{X9NV+6 z+l^5~#>6JR0}}6(koW+=0}>yA7Z5z~#0!Xb9y!-;=W9FN(lN1$UCUPEd(XM&oZtQV zoO^9`{nC;`FuJLmEs3Bh;e-&43U47K2tprRJ#g(G5%>|j-|suO7 z@GTo_wcu^f!dhj%T0zzM3riIgkR8*c7M=jj@pdE9n;9LAd@4F)5`Bl5cCEQ7OGNVB zM1AA=y3kk>!J)`#ak*M~yRwE>*A_3%uU$s(RW76Xjq1wcGH73{ELSHSNvozC7*#9p zSJCncTpLSEZU=R%Q+pHN>D0U}sWwLMuCFX#aP>Eq7cXs8(3oRj0y$A4P#y=f<8#qO z=Hh5XP&66e-UCFkiSFEQ`Xbsa`EFxCHM+kSQt^~97k%{xZJ$h8#9Qnb2^ z4Xnx1mQLnXY!d4(FeFQKBB?9T;kISVs7QI0RrQ^9yp2tT+zIrwu&mgv13DW@y>UQS z4aQPxk{a$#?zU&rA##^y zWo!{D-$}8QE7CKd2yDnCDW*QhSWI7>XNgC`Ns8Dj9%8SZNPw>+16~X3syqEqs}yWq zxz;{?JcKy&fWEl6+6#Tuv5|WVUKk7&3|m##Xw9|uvIf*LZPi@RZH40h0om!8g6MKe-D<^#_=>p@&UxB8^6K zFDoz)AM0&bPfrK?!OY~ka6g-%Gd%Q=d9axoZb^#f)K9wJqIr<%b5oy;?;m%Ec!1d` z9+%^8hOnS$I>m&VSl3~tc+=Kuw7OVmud1V)rrtuU@2rkm$Re_;Y`OLuq@`|{DphO{ zVp!!up{|gIy_K)&t%7`erx0>kGL(P@wB#8rCyvg#lq=RVXcrvCy9Lc|ZDDiQ_1&Gy zXBlme1peV{1D5vwf;E#X%~ngZWvE%xQ+vfqm9)Ao)t%`CJY83aZYVViP`r`_++bCO z&FloqLS`^bY+3ZTA=R2vU9pIB(=f8vT!!ygJ*qvE+`WG)?l9^>=W-jO1CxCA8Rerx zwV_xDPSU{i0&XZ8vMuLRozv(nolpa6HL+~Q3 zd?;eD>d%20zTvv1#!RjV=>yigbLG->m(=(->7!Ik?(UAn==vnX@Px_W`i=&f3DSCY zBpMWB$nS}ri-6b&I~t3GCzpg`JeayJq+_Yi(jSXg#c$JprVFXB(qD=niNA>_Q$MAy zr>}}{ieIF^KT4Zn?h8kNBft^h2yg^A0vrL307rl$z!5lF2#m(z<#fc``7=8@_Kovk zP>$Ksv2p`rad@B0jK}WjSiuwL!ElZd3>o&m6N*9Eka15oJpm4|w6N&tYGh1h;OKKeDt+W zBG}Iur2TOOv9bDWY*e5J1co!64LCdaSWtkm;0Q#iiA=voK-PpKk%9*%BK{@7$q$YI pM}Q;15#R`L1ULd50geDifFr;W;0SO8{tpNYL}H^6-~Ry&{sXsMHn;!) delta 216 zcmZo@U~Fh$+`uRx%)r3F#!2J7o z2#SGqq_Y5ZsET$cLT$4NHs|DEjZ5a4t`1kSG@ZaaR;h(!% P&_IKKvyJ{?0Y)|e=ubM< diff --git a/utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js b/utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js index dfc0eab..d395a2c 100644 --- a/utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js +++ b/utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js @@ -19,7 +19,7 @@ const fetchAllDependencyLifetimes = async () => { const dependencySet = new Set(); Object.entries(persistedRepoData) - .flatMap(([, repo]) => repo.main.dependencies) + .flatMap(([, repo]) => repo.dependencies) .forEach((dependency) => dependencySet.add(dependency.name)); const apiClient = new EndOfLifeDateApiClient(); diff --git a/utils/githubApi/fetchAllRepos.js b/utils/githubApi/fetchAllRepos.js index 6b0cb94..c920224 100644 --- a/utils/githubApi/fetchAllRepos.js +++ b/utils/githubApi/fetchAllRepos.js @@ -40,9 +40,8 @@ const fetchAllRepos = async () => { repository, octokit, }), - ]).then(([dependencies, prInfo, issueInfo , alerts]) => { - repo.dependencies = dependencies; - repo = { ...repo, ...prInfo, ...issueInfo, ...alerts }; + ]).then(([dependencies, prInfo, issueInfo, alerts]) => { + repo = { repo, dependencies, prInfo, issueInfo, alerts }; }); repos.push(repo); @@ -64,8 +63,14 @@ const saveAllRepos = async () => { console.info("Saving all repos..."); const saveAllRepos = db.transaction((repos) => { repos.forEach((repo) => { - db.saveToRepository(repo.name, "main", repo); - db.saveToRepository(repo.name, "owner", repo.owner); + const name = repo.repo.name; + const owner = repo.repo.owner; + db.saveToRepository(name, "main", repo.repo); + db.saveToRepository(name, "owner", owner); + db.saveToRepository(name, "dependencies", repo.dependencies); + db.saveToRepository(name, "pullRequests", repo.prInfo); + db.saveToRepository(name, "issues", repo.issueInfo); + db.saveToRepository(name, "dependabotAlerts", repo.alerts); }); }); diff --git a/utils/index.js b/utils/index.js index 9dab8f4..f772d1e 100644 --- a/utils/index.js +++ b/utils/index.js @@ -140,28 +140,31 @@ export const hashToTailwindColor = (str) => { export const mapRepoFromStorageToUi = (persistedData, persistedLifetimes) => { const mappedRepos = Object.entries(persistedData).map(([, repo]) => { const newDate = new Date(repo.main.updatedAt).toLocaleDateString(); - const dependencies = repo.main.dependencies.map((dependency) => mapDependencyFromStorageToUi(dependency, persistedLifetimes)); + const dependencies = repo.dependencies.map((dependency) => mapDependencyFromStorageToUi(dependency, persistedLifetimes)); - const mostRecentPrOpenedAt = repo.main.mostRecentPrOpenedAt && formatDistanceToNow(repo.main.mostRecentPrOpenedAt, { addSuffix: true }); - const oldestOpenPrOpenedAt = repo.main.oldestOpenPrOpenedAt && formatDistanceToNow(repo.main.oldestOpenPrOpenedAt, { addSuffix: true }); - const mostRecentIssueOpenedAt = repo.main.mostRecentIssueOpenedAt && formatDistanceToNow(repo.main.mostRecentIssueOpenedAt, { addSuffix: true }); - const oldestOpenIssueOpenedAt = repo.main.oldestOpenIssueOpenedAt && formatDistanceToNow(repo.main.oldestOpenIssueOpenedAt, { addSuffix: true }); + const mostRecentPrOpenedAt = repo.pullRequests.mostRecentPrOpenedAt && formatDistanceToNow(repo.pullRequests.mostRecentPrOpenedAt, { addSuffix: true }); + const oldestOpenPrOpenedAt = repo.pullRequests.oldestOpenPrOpenedAt && formatDistanceToNow(repo.pullRequests.oldestOpenPrOpenedAt, { addSuffix: true }); + const mostRecentIssueOpenedAt = repo.issues.mostRecentIssueOpenedAt && formatDistanceToNow(repo.issues.mostRecentIssueOpenedAt, { addSuffix: true }); + const oldestOpenIssueOpenedAt = repo.issues.oldestOpenIssueOpenedAt && formatDistanceToNow(repo.issues.oldestOpenIssueOpenedAt, { addSuffix: true }); const languageColor = hashToTailwindColor(repo.main.language); return { ...repo.main, + ...repo.dependabotAlerts, + ...repo.pullRequests, + ...repo.issues, updatedAt: newDate, updatedAtISO8601: repo.main.updatedAt, dependencies, mostRecentPrOpenedAt, - mostRecentPrOpenedAtISO8601: repo.main.mostRecentPrOpenedAt, + mostRecentPrOpenedAtISO8601: repo.pullRequests.mostRecentPrOpenedAt, oldestOpenPrOpenedAt, - oldestOpenPrOpenedAtISO8601: repo.main.oldestOpenPrOpenedAt, + oldestOpenPrOpenedAtISO8601: repo.pullRequests.oldestOpenPrOpenedAt, mostRecentIssueOpenedAt, - mostRecentIssueOpenedAtISO8601: repo.main.mostRecentIssueOpenedAt, + mostRecentIssueOpenedAtISO8601: repo.issues.mostRecentIssueOpenedAt, oldestOpenIssueOpenedAt, - oldestOpenIssueOpenedAtISO8601: repo.main.oldestOpenIssueOpenedAt, + oldestOpenIssueOpenedAtISO8601: repo.issues.oldestOpenIssueOpenedAt, languageColor, }; }); diff --git a/utils/index.test.js b/utils/index.test.js index bec6546..474b7d4 100644 --- a/utils/index.test.js +++ b/utils/index.test.js @@ -18,13 +18,17 @@ describe("mapRepoFromStorageToUi", () => { issuesUrl: "http://api.com/repo1/issues", language: null, topics: [], - openIssues: 0, - dependencies: [], + }, + pullRequests: { mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + }, + issues: { + openIssues: 0, mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", }, + dependencies: [], }, }; @@ -71,13 +75,17 @@ describe("mapRepoFromStorageToUi", () => { issuesUrl: "http://api.com/repo1/issues", language: "Ruby", topics: [], - openIssues: 0, - dependencies: [], + }, + pullRequests: { mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + }, + issues: { + openIssues: 0, mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", }, + dependencies: [], }, repo2: { owner: "dxw", @@ -91,13 +99,17 @@ describe("mapRepoFromStorageToUi", () => { issuesUrl: "http://api.com/repo2/issues", language: "TypeScript", topics: [], - openIssues: 0, - dependencies: [], + }, + pullRequests: { mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + }, + issues: { + openIssues: 0, mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", }, + dependencies: [], }, }; @@ -167,9 +179,17 @@ describe("mapRepoFromStorageToUi", () => { issuesUrl: "http://api.com/repo1/issues", language: null, topics: [], + }, + pullRequests: { + mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", + oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + }, + issues: { openIssues: 0, - dependencies: [], + mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", + oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", }, + dependencies: [], }, repo2: { owner: "dxw", @@ -183,9 +203,17 @@ describe("mapRepoFromStorageToUi", () => { issuesUrl: "http://api.com/repo2/issues", language: null, topics: [], + }, + pullRequests: { + mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", + oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + }, + issues: { openIssues: 0, - dependencies: [], + mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", + oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", }, + dependencies: [], }, repo3: { owner: "dxw", @@ -199,9 +227,17 @@ describe("mapRepoFromStorageToUi", () => { issuesUrl: "http://api.com/repo3/issues", language: null, topics: [], + }, + pullRequests: { + mostRecentPrOpenedAt: "2021-01-01T00:00:00Z", + oldestOpenPrOpenedAt: "2022-02-02T00:00:00Z", + }, + issues: { openIssues: 0, - dependencies: [], + mostRecentIssueOpenedAt: "2023-03-03T00:00:00Z", + oldestOpenIssueOpenedAt: "2024-04-04T00:00:00Z", }, + dependencies: [], }, };