From 4990f8f7d89e92302bb6ae87bc5c88339d5a4732 Mon Sep 17 00:00:00 2001 From: Maxwell Mapako Date: Fri, 17 May 2024 20:28:07 +0200 Subject: [PATCH] feat: mongodb driver migration and graceful process shutdown (#124) * chore(deps): add esm.sh import for mongodb * refactor: update deps for assertions and handle process singals - `testing/asserts.ts` has been deprecated and `assert/mod.ts` has been used as a replacement - Adds `SIGTERM` and `SIGINT` process handlers to close mongodb connection and gracefully exit the app process * feat: update mongo drive and remove ua_parser in favour of deno std * chore: auto format Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- Dockerfile | 15 ++- deno.json | 26 ++--- deno.lock | 155 ++++++++++++++++++++++++++++ src/common/core/factory.ts | 40 +++----- src/common/core/request.ts | 27 +++-- src/common/core/setup.ts | 25 ++++- src/common/core/utils.ts | 3 + src/common/middleware/targeting.ts | 5 +- src/common/mongo/collection.ts | 8 +- src/common/mongo/factory.ts | 58 ++++++----- src/common/mongo/index.ts | 2 + src/common/mongo/types.d.ts | 15 +++ src/common/mongo/utils.ts | 11 ++ src/common/types/core.d.ts | 4 +- src/common/types/paging.d.ts | 4 +- src/config/index.ts | 4 +- src/config/local/source.ts | 5 +- src/config/local/types.d.ts | 2 +- src/config/transformer/index.ts | 3 +- src/import_map.json | 15 +-- src/index.ts | 9 +- src/news/index.ts | 35 ++++--- src/news/local/index.ts | 94 +---------------- src/news/local/source.ts | 160 +++++++++++++++++++++++++++++ src/news/local/transformer.ts | 2 +- src/news/local/types.d.ts | 15 ++- src/news/mapper.ts | 24 +++-- src/news/repository/index.ts | 58 ++++++++--- src/news/transformer.ts | 13 ++- src/news/types.d.ts | 8 +- src/series/index.ts | 10 +- src/series/local/index.ts | 48 +-------- src/series/local/source.ts | 82 +++++++++++++++ src/series/local/transformer.ts | 48 ++++++--- src/series/local/types.d.ts | 9 ++ src/series/repository/series.ts | 13 +-- src/series/types.d.ts | 4 + 37 files changed, 736 insertions(+), 323 deletions(-) create mode 100644 src/common/mongo/utils.ts create mode 100644 src/news/local/source.ts create mode 100644 src/series/local/source.ts create mode 100644 src/series/local/types.d.ts diff --git a/Dockerfile b/Dockerfile index b3684e6..26e992e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,15 @@ RUN apt-get install unzip FROM scaffold AS cache RUN deno cache src/index.ts -# Complilation broken in the latest version of deno -# FROM cache AS final -# RUN deno compile --allow-net --allow-env --allow-read --output=server src/index.ts +# Fallback and compilation has broken in some instances +#ENTRYPOINT ["deno", "run", "--allow-net", "--allow-env", "--allow-read", "--allow-sys", "src/index.ts"] -#ENTRYPOINT ["/usr/app/server"] +FROM cache AS build +RUN deno check src/index.ts +RUN deno compile --allow-net --allow-env --allow-read --allow-sys --output=/usr/on-the-edge src/index.ts -ENTRYPOINT ["deno", "run", "--allow-net", "--allow-env", "--allow-read", "src/index.ts"] +FROM build AS final +RUN rm -r /usr/app +WORKDIR /usr + +ENTRYPOINT ["/usr/on-the-edge"] diff --git a/deno.json b/deno.json index b668a45..2152db8 100644 --- a/deno.json +++ b/deno.json @@ -2,29 +2,23 @@ "$schema": "http://json-schema.org/draft-07/schema", "importMap": "./src/import_map.json", "lint": { - "files": { - "include": ["src"], - "exclude": [".github", "README.md"] - }, + "include": ["src"], + "exclude": [".github", "README.md"], "rules": { "tags": ["recommended"] } }, "fmt": { - "files": { - "include": ["src"], - "exclude": [".github", "README.md"] - }, - "options": { - "useTabs": false, - "lineWidth": 80, - "indentWidth": 2, - "singleQuote": true, - "proseWrap": "preserve" - } + "include": ["src"], + "exclude": [".github", "README.md"], + "useTabs": false, + "lineWidth": 80, + "indentWidth": 2, + "singleQuote": true, + "proseWrap": "preserve" }, "tasks": { "test": "deno test --allow-read --allow-env --allow-net", - "server": "deno run --allow-read --allow-env --allow-net ./src/index.ts" + "server": "deno run --allow-read --allow-env --allow-net --allow-sys ./src/index.ts" } } diff --git a/deno.lock b/deno.lock index 688c017..e2c1936 100644 --- a/deno.lock +++ b/deno.lock @@ -24,6 +24,7 @@ "jsr:@std/media-types@0.223": "jsr:@std/media-types@0.223.0", "jsr:@std/path@0.223": "jsr:@std/path@0.223.0", "npm:@types/node": "npm:@types/node@18.16.19", + "npm:mongodb@6.6.1": "npm:mongodb@6.6.1", "npm:path-to-regexp@6.2.1": "npm:path-to-regexp@6.2.1" }, "jsr": { @@ -110,13 +111,79 @@ } }, "npm": { + "@mongodb-js/saslprep@1.1.7": { + "integrity": "sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q==", + "dependencies": { + "sparse-bitfield": "sparse-bitfield@3.0.3" + } + }, "@types/node@18.16.19": { "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==", "dependencies": {} }, + "@types/webidl-conversions@7.0.3": { + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "dependencies": {} + }, + "@types/whatwg-url@11.0.4": { + "integrity": "sha512-lXCmTWSHJvf0TRSO58nm978b8HJ/EdsSsEKLd3ODHFjo+3VGAyyTp4v50nWvwtzBxSMQrVOK7tcuN0zGPLICMw==", + "dependencies": { + "@types/webidl-conversions": "@types/webidl-conversions@7.0.3" + } + }, + "bson@6.7.0": { + "integrity": "sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ==", + "dependencies": {} + }, + "memory-pager@1.5.0": { + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "dependencies": {} + }, + "mongodb-connection-string-url@3.0.1": { + "integrity": "sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==", + "dependencies": { + "@types/whatwg-url": "@types/whatwg-url@11.0.4", + "whatwg-url": "whatwg-url@13.0.0" + } + }, + "mongodb@6.6.1": { + "integrity": "sha512-FvA9ocQzRzzvhin1HHLrZDEm0gWvnksbiciYrU/0GmET/t/DdDiMJroA7rfDrHM3AInwGVYw2fwAU2oNYUyUEw==", + "dependencies": { + "@mongodb-js/saslprep": "@mongodb-js/saslprep@1.1.7", + "bson": "bson@6.7.0", + "mongodb-connection-string-url": "mongodb-connection-string-url@3.0.1" + } + }, "path-to-regexp@6.2.1": { "integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==", "dependencies": {} + }, + "punycode@2.3.1": { + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dependencies": {} + }, + "sparse-bitfield@3.0.3": { + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "dependencies": { + "memory-pager": "memory-pager@1.5.0" + } + }, + "tr46@4.1.1": { + "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", + "dependencies": { + "punycode": "punycode@2.3.1" + } + }, + "webidl-conversions@7.0.0": { + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dependencies": {} + }, + "whatwg-url@13.0.0": { + "integrity": "sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==", + "dependencies": { + "tr46": "tr46@4.1.1", + "webidl-conversions": "webidl-conversions@7.0.0" + } } } }, @@ -951,6 +1018,37 @@ "https://deno.land/std@0.208.0/uuid/v3.ts": "397ad58daec8b5ef6ba7e94fe86c9bc56b194adcbe2f70ec40a1fb005203c870", "https://deno.land/std@0.208.0/uuid/v4.ts": "0f081880c156fd59b9e44e2f84ea0f94a3627e89c224eaf6cc982b53d849f37e", "https://deno.land/std@0.208.0/uuid/v5.ts": "9daaf769e487b512d25adf8e137e05ff2e3392d27f66d5b273ee28030ff7cd58", + "https://deno.land/std@0.224.0/assert/_constants.ts": "a271e8ef5a573f1df8e822a6eb9d09df064ad66a4390f21b3e31f820a38e0975", + "https://deno.land/std@0.224.0/assert/assert.ts": "09d30564c09de846855b7b071e62b5974b001bb72a4b797958fe0660e7849834", + "https://deno.land/std@0.224.0/assert/assert_almost_equals.ts": "9e416114322012c9a21fa68e187637ce2d7df25bcbdbfd957cd639e65d3cf293", + "https://deno.land/std@0.224.0/assert/assert_array_includes.ts": "14c5094471bc8e4a7895fc6aa5a184300d8a1879606574cb1cd715ef36a4a3c7", + "https://deno.land/std@0.224.0/assert/assert_equals.ts": "3bbca947d85b9d374a108687b1a8ba3785a7850436b5a8930d81f34a32cb8c74", + "https://deno.land/std@0.224.0/assert/assert_exists.ts": "43420cf7f956748ae6ed1230646567b3593cb7a36c5a5327269279c870c5ddfd", + "https://deno.land/std@0.224.0/assert/assert_false.ts": "3e9be8e33275db00d952e9acb0cd29481a44fa0a4af6d37239ff58d79e8edeff", + "https://deno.land/std@0.224.0/assert/assert_greater.ts": "5e57b201fd51b64ced36c828e3dfd773412c1a6120c1a5a99066c9b261974e46", + "https://deno.land/std@0.224.0/assert/assert_greater_or_equal.ts": "9870030f997a08361b6f63400273c2fb1856f5db86c0c3852aab2a002e425c5b", + "https://deno.land/std@0.224.0/assert/assert_instance_of.ts": "e22343c1fdcacfaea8f37784ad782683ec1cf599ae9b1b618954e9c22f376f2c", + "https://deno.land/std@0.224.0/assert/assert_is_error.ts": "f856b3bc978a7aa6a601f3fec6603491ab6255118afa6baa84b04426dd3cc491", + "https://deno.land/std@0.224.0/assert/assert_less.ts": "60b61e13a1982865a72726a5fa86c24fad7eb27c3c08b13883fb68882b307f68", + "https://deno.land/std@0.224.0/assert/assert_less_or_equal.ts": "d2c84e17faba4afe085e6c9123a63395accf4f9e00150db899c46e67420e0ec3", + "https://deno.land/std@0.224.0/assert/assert_match.ts": "ace1710dd3b2811c391946954234b5da910c5665aed817943d086d4d4871a8b7", + "https://deno.land/std@0.224.0/assert/assert_not_equals.ts": "78d45dd46133d76ce624b2c6c09392f6110f0df9b73f911d20208a68dee2ef29", + "https://deno.land/std@0.224.0/assert/assert_not_instance_of.ts": "3434a669b4d20cdcc5359779301a0588f941ffdc2ad68803c31eabdb4890cf7a", + "https://deno.land/std@0.224.0/assert/assert_not_match.ts": "df30417240aa2d35b1ea44df7e541991348a063d9ee823430e0b58079a72242a", + "https://deno.land/std@0.224.0/assert/assert_not_strict_equals.ts": "37f73880bd672709373d6dc2c5f148691119bed161f3020fff3548a0496f71b8", + "https://deno.land/std@0.224.0/assert/assert_object_match.ts": "411450fd194fdaabc0089ae68f916b545a49d7b7e6d0026e84a54c9e7eed2693", + "https://deno.land/std@0.224.0/assert/assert_rejects.ts": "4bee1d6d565a5b623146a14668da8f9eb1f026a4f338bbf92b37e43e0aa53c31", + "https://deno.land/std@0.224.0/assert/assert_strict_equals.ts": "b4f45f0fd2e54d9029171876bd0b42dd9ed0efd8f853ab92a3f50127acfa54f5", + "https://deno.land/std@0.224.0/assert/assert_string_includes.ts": "496b9ecad84deab72c8718735373feb6cdaa071eb91a98206f6f3cb4285e71b8", + "https://deno.land/std@0.224.0/assert/assert_throws.ts": "c6508b2879d465898dab2798009299867e67c570d7d34c90a2d235e4553906eb", + "https://deno.land/std@0.224.0/assert/assertion_error.ts": "ba8752bd27ebc51f723702fac2f54d3e94447598f54264a6653d6413738a8917", + "https://deno.land/std@0.224.0/assert/equal.ts": "bddf07bb5fc718e10bb72d5dc2c36c1ce5a8bdd3b647069b6319e07af181ac47", + "https://deno.land/std@0.224.0/assert/fail.ts": "0eba674ffb47dff083f02ced76d5130460bff1a9a68c6514ebe0cdea4abadb68", + "https://deno.land/std@0.224.0/assert/mod.ts": "48b8cb8a619ea0b7958ad7ee9376500fe902284bb36f0e32c598c3dc34cbd6f3", + "https://deno.land/std@0.224.0/assert/unimplemented.ts": "8c55a5793e9147b4f1ef68cd66496b7d5ba7a9e7ca30c6da070c1a58da723d73", + "https://deno.land/std@0.224.0/assert/unreachable.ts": "5ae3dbf63ef988615b93eb08d395dda771c96546565f9e521ed86f6510c29e19", + "https://deno.land/std@0.224.0/async/delay.ts": "f90dd685b97c2f142b8069082993e437b1602b8e2561134827eeb7c12b95c499", + "https://deno.land/std@0.224.0/cli/parse_args.ts": "5250832fb7c544d9111e8a41ad272c016f5a53f975ef84d5a9fe5fcb70566ece", "https://deno.land/std@0.224.0/collections/_utils.ts": "b2ec8ada31b5a72ebb1d99774b849b4c09fe4b3a38d07794bd010bd218a16e0b", "https://deno.land/std@0.224.0/collections/aggregate_groups.ts": "c63a57a16e87537ee71df1174a111e2c4cd05f0d4be83db701c8740e81bd157c", "https://deno.land/std@0.224.0/collections/associate_by.ts": "8c09c6a66769480f3181bd7b582796aed85fa71e2595c1b3d73684b9caae33b1", @@ -1000,7 +1098,64 @@ "https://deno.land/std@0.224.0/dotenv/mod.ts": "0180eaeedaaf88647318811cdaa418cc64dc51fb08354f91f5f480d0a1309f7d", "https://deno.land/std@0.224.0/dotenv/parse.ts": "09977ff88dfd1f24f9973a338f0f91bbdb9307eb5ff6085446e7c423e4c7ba0c", "https://deno.land/std@0.224.0/dotenv/stringify.ts": "275da322c409170160440836342eaa7cf012a1d11a7e700d8ca4e7f2f8aa4615", + "https://deno.land/std@0.224.0/encoding/_util.ts": "beacef316c1255da9bc8e95afb1fa56ed69baef919c88dc06ae6cb7a6103d376", + "https://deno.land/std@0.224.0/encoding/base64.ts": "dd59695391584c8ffc5a296ba82bcdba6dd8a84d41a6a539fbee8e5075286eaf", + "https://deno.land/std@0.224.0/encoding/hex.ts": "6270f25e5d85f99fcf315278670ba012b04b7c94b67715b53f30d03249687c07", + "https://deno.land/std@0.224.0/fmt/bytes.ts": "7b294a4b9cf0297efa55acb55d50610f3e116a0ac772d1df0ae00f0b833ccd4a", + "https://deno.land/std@0.224.0/fmt/colors.ts": "508563c0659dd7198ba4bbf87e97f654af3c34eb56ba790260f252ad8012e1c5", + "https://deno.land/std@0.224.0/http/_negotiation/common.ts": "051a9f6edd1ed69507df89bbc16fc1b13b7654b9b8fd38072ec33ae4c185fc13", + "https://deno.land/std@0.224.0/http/_negotiation/encoding.ts": "fdedea1145c1dea3b3de2d5217e8eb927e764083eebc8c52d09a1ed3d9bb7a93", + "https://deno.land/std@0.224.0/http/_negotiation/language.ts": "300a5c586f844c97f246ab72c948e9fde9a8f45e92ec08e1cc9a9df80259e2a3", + "https://deno.land/std@0.224.0/http/_negotiation/media_type.ts": "87a1ecb22c1b268d0fa23d798e1ea238343505268cb1ff82bd038638de29ce31", + "https://deno.land/std@0.224.0/http/cookie.ts": "a377fa60175ba5f61dd4b8a70b34f2bbfbc70782dfd5faf36d314c42e4306006", + "https://deno.land/std@0.224.0/http/etag.ts": "9ca56531be682f202e4239971931060b688ee5c362688e239eeaca39db9e72cb", + "https://deno.land/std@0.224.0/http/file_server.ts": "2a5392195b8e7713288f274d071711b705bb5b3220294d76cce495d456c61a93", + "https://deno.land/std@0.224.0/http/mod.ts": "b0e06293a8a2f71b041add53d4674d4997bd2b83a4760bddad1b5e552130bcc8", + "https://deno.land/std@0.224.0/http/negotiation.ts": "d06ef2958ca712a7dbe4538eed6a46abfa2b87f8e150b7c89d83a6055dabd7cc", + "https://deno.land/std@0.224.0/http/server.ts": "f9313804bf6467a1704f45f76cb6cd0a3396a3b31c316035e6a4c2035d1ea514", + "https://deno.land/std@0.224.0/http/server_sent_event_stream.ts": "d9c20b46f986d78f60c38dbd91e95c71d73b45f29739a8ef4216dfa5f2e71eb3", "https://deno.land/std@0.224.0/http/status.ts": "ed61b4882af2514a81aefd3245e8df4c47b9a8e54929a903577643d2d1ebf514", + "https://deno.land/std@0.224.0/http/unstable_signed_cookie.ts": "2a5bfbdf6b4aa35ef1464300fe1ba4eb89eb79f535c9cb28401d55fbb7038479", + "https://deno.land/std@0.224.0/http/user_agent.ts": "05f8849c7e27b898793bfc70204f0c72b6be9bee7accbe98e18a1c413bd4ace3", + "https://deno.land/std@0.224.0/internal/diff.ts": "6234a4b493ebe65dc67a18a0eb97ef683626a1166a1906232ce186ae9f65f4e6", + "https://deno.land/std@0.224.0/internal/format.ts": "0a98ee226fd3d43450245b1844b47003419d34d210fa989900861c79820d21c2", + "https://deno.land/std@0.224.0/internal/mod.ts": "534125398c8e7426183e12dc255bb635d94e06d0f93c60a297723abe69d3b22e", + "https://deno.land/std@0.224.0/media_types/_db.ts": "19563a2491cd81b53b9c1c6ffd1a9145c355042d4a854c52f6e1424f73ff3923", + "https://deno.land/std@0.224.0/media_types/_util.ts": "e0b8da0c7d8ad2015cf27ac16ddf0809ac984b2f3ec79f7fa4206659d4f10deb", + "https://deno.land/std@0.224.0/media_types/content_type.ts": "ed3f2e1f243b418ad3f441edc95fd92efbadb0f9bde36219c7564c67f9639513", + "https://deno.land/std@0.224.0/media_types/format_media_type.ts": "ffef4718afa2489530cb94021bb865a466eb02037609f7e82899c017959d288a", + "https://deno.land/std@0.224.0/media_types/get_charset.ts": "277ebfceb205bd34e616fe6764ef03fb277b77f040706272bea8680806ae3f11", + "https://deno.land/std@0.224.0/media_types/parse_media_type.ts": "487f000a38c230ccbac25420a50f600862e06796d0eee19d19631b9e84ee9654", + "https://deno.land/std@0.224.0/media_types/type_by_extension.ts": "bf4e3f5d6b58b624d5daa01cbb8b1e86d9939940a77e7c26e796a075b60ec73b", + "https://deno.land/std@0.224.0/media_types/vendor/mime-db.v1.52.0.ts": "0218d2c7d900e8cd6fa4a866e0c387712af4af9a1bae55d6b2546c73d273a1e6", + "https://deno.land/std@0.224.0/path/_common/assert_path.ts": "dbdd757a465b690b2cc72fc5fb7698c51507dec6bfafce4ca500c46b76ff7bd8", + "https://deno.land/std@0.224.0/path/_common/constants.ts": "dc5f8057159f4b48cd304eb3027e42f1148cf4df1fb4240774d3492b5d12ac0c", + "https://deno.land/std@0.224.0/path/_common/normalize.ts": "684df4aa71a04bbcc346c692c8485594fc8a90b9408dfbc26ff32cf3e0c98cc8", + "https://deno.land/std@0.224.0/path/_common/normalize_string.ts": "33edef773c2a8e242761f731adeb2bd6d683e9c69e4e3d0092985bede74f4ac3", + "https://deno.land/std@0.224.0/path/_common/relative.ts": "faa2753d9b32320ed4ada0733261e3357c186e5705678d9dd08b97527deae607", + "https://deno.land/std@0.224.0/path/_os.ts": "8fb9b90fb6b753bd8c77cfd8a33c2ff6c5f5bc185f50de8ca4ac6a05710b2c15", + "https://deno.land/std@0.224.0/path/constants.ts": "0c206169ca104938ede9da48ac952de288f23343304a1c3cb6ec7625e7325f36", + "https://deno.land/std@0.224.0/path/extname.ts": "593303db8ae8c865cbd9ceec6e55d4b9ac5410c1e276bfd3131916591b954441", + "https://deno.land/std@0.224.0/path/join.ts": "ae2ec5ca44c7e84a235fd532e4a0116bfb1f2368b394db1c4fb75e3c0f26a33a", + "https://deno.land/std@0.224.0/path/posix/_util.ts": "1e3937da30f080bfc99fe45d7ed23c47dd8585c5e473b2d771380d3a6937cf9d", + "https://deno.land/std@0.224.0/path/posix/extname.ts": "e398c1d9d1908d3756a7ed94199fcd169e79466dd88feffd2f47ce0abf9d61d2", + "https://deno.land/std@0.224.0/path/posix/join.ts": "7fc2cb3716aa1b863e990baf30b101d768db479e70b7313b4866a088db016f63", + "https://deno.land/std@0.224.0/path/posix/normalize.ts": "baeb49816a8299f90a0237d214cef46f00ba3e95c0d2ceb74205a6a584b58a91", + "https://deno.land/std@0.224.0/path/posix/relative.ts": "3907d6eda41f0ff723d336125a1ad4349112cd4d48f693859980314d5b9da31c", + "https://deno.land/std@0.224.0/path/posix/resolve.ts": "08b699cfeee10cb6857ccab38fa4b2ec703b0ea33e8e69964f29d02a2d5257cf", + "https://deno.land/std@0.224.0/path/relative.ts": "ab739d727180ed8727e34ed71d976912461d98e2b76de3d3de834c1066667add", + "https://deno.land/std@0.224.0/path/resolve.ts": "a6f977bdb4272e79d8d0ed4333e3d71367cc3926acf15ac271f1d059c8494d8d", + "https://deno.land/std@0.224.0/path/windows/_util.ts": "d5f47363e5293fced22c984550d5e70e98e266cc3f31769e1710511803d04808", + "https://deno.land/std@0.224.0/path/windows/extname.ts": "165a61b00d781257fda1e9606a48c78b06815385e7d703232548dbfc95346bef", + "https://deno.land/std@0.224.0/path/windows/join.ts": "8d03530ab89195185103b7da9dfc6327af13eabdcd44c7c63e42e27808f50ecf", + "https://deno.land/std@0.224.0/path/windows/normalize.ts": "78126170ab917f0ca355a9af9e65ad6bfa5be14d574c5fb09bb1920f52577780", + "https://deno.land/std@0.224.0/path/windows/relative.ts": "3e1abc7977ee6cc0db2730d1f9cb38be87b0ce4806759d271a70e4997fc638d7", + "https://deno.land/std@0.224.0/path/windows/resolve.ts": "8dae1dadfed9d46ff46cc337c9525c0c7d959fb400a6308f34595c45bdca1972", + "https://deno.land/std@0.224.0/streams/byte_slice_stream.ts": "5bbdcadb118390affa9b3d0a0f73ef8e83754f59bb89df349add669dd9369713", + "https://deno.land/std@0.224.0/testing/_test_suite.ts": "f10a8a6338b60c403f07a76f3f46bdc9f1e1a820c0a1decddeb2949f7a8a0546", + "https://deno.land/std@0.224.0/testing/asserts.ts": "d0cdbabadc49cc4247a50732ee0df1403fdcd0f95360294ad448ae8c240f3f5c", + "https://deno.land/std@0.224.0/testing/bdd.ts": "3e4de4ff6d8f348b5574661cef9501b442046a59079e201b849d0e74120d476b", + "https://deno.land/std@0.224.0/version.ts": "f6a28c9704d82d1c095988777e30e6172eb674a6570974a0d27a653be769bbbe", "https://deno.land/x/base64@v0.2.1/base.ts": "47dc8d68f07dc91524bdd6db36eccbe59cf4d935b5fc09f27357a3944bb3ff7b", "https://deno.land/x/base64@v0.2.1/base64url.ts": "18bbf879b31f1f32cca8adaa2b6885ae325c2cec6a66c5817b684ca12c46ad5e", "https://deno.land/x/brotli@0.1.7/mod.ts": "08b913e51488b6e7fa181f2814b9ad087fdb5520041db0368f8156bfa45fd73e", diff --git a/src/common/core/factory.ts b/src/common/core/factory.ts index 3686bfa..1428d23 100644 --- a/src/common/core/factory.ts +++ b/src/common/core/factory.ts @@ -8,40 +8,30 @@ import header from '../middleware/header.ts'; import targeting from '../middleware/targeting.ts'; import { logger } from './logger.ts'; import { between } from 'x/optic'; -import _localSourceFactory from '../mongo/factory.ts'; const app = new Application({ state, contextState: 'prototype', }); -export default (opts: FactoryOptions): Application => { - logger.mark('factory-start'); - const router = opts.router ?? new Router(); +app.use( + timing, + header, + growth, + targeting, + error, +); - app.use( - timing, - header, - growth, - targeting, - error, +app.addEventListener('error', (event) => { + logger.critical( + 'common.core.factory:error: Uncaught application exception', + event.error, ); +}); - app.addEventListener('close', (event) => { - _localSourceFactory.disconnect(); - logger.info( - 'common:core:factory:close: Request application stop by user', - event.type, - ); - }); - - app.addEventListener('error', (event) => { - _localSourceFactory.disconnect(); - logger.critical( - 'common.core.factory:error: Uncaught application exception', - event.error, - ); - }); +export default (opts: FactoryOptions): Application => { + logger.mark('factory-start'); + const router = opts.router ?? new Router(); app.use(router.routes()); app.use(router.allowedMethods()); diff --git a/src/common/core/request.ts b/src/common/core/request.ts index 007358c..3fc77bd 100644 --- a/src/common/core/request.ts +++ b/src/common/core/request.ts @@ -1,17 +1,17 @@ import { between } from 'x/optic'; import { logger } from './logger.ts'; -const sanitize = (url: string): string => { - const urlObject = new URL(url); +const sanitize = (uri: string): { safeUrl: string; host: string } => { + const url = new URL(uri); - const queryParams = urlObject.searchParams; + const queryParams = url.searchParams; for (const key of queryParams.keys()) { if (key.includes('api_key') || key.includes('api_secret')) { queryParams.set(key, '********'); } } - return urlObject.toString(); + return { safeUrl: url.toString(), host: url.host }; }; export const defaults: RequestInit = { @@ -20,6 +20,8 @@ export const defaults: RequestInit = { headers: { 'accept': 'application/json, application/xml, text/plain, */*', 'accept-encoding': 'gzip, deflate, br', + 'connection': 'keep-alive', + 'user-agent': `Deno/${Deno.version.deno}`, }, }; @@ -27,16 +29,21 @@ export const request = async ( url: string, options: RequestInit = defaults, ): Promise => { - const sanitizedUrl = sanitize(url); - logger.debug(`----> ${options.method}: ${sanitizedUrl}`); logger.mark('request-start'); - return await fetch(url, options).then((response) => { - logger.debug(`<---- ${response.status} ${options.method}: ${sanitizedUrl}`); + const { safeUrl, host } = sanitize(url); + logger.debug(`----> HTTP ${options.method}: ${safeUrl}`); + return await fetch(url, { + headers: { + ...options.headers, + host: host, + }, + }).then((response) => { + logger.debug(`<---- HTTP/${response.status} ${options.method}: ${safeUrl}`); logger.mark('request-end'); - logger.measure(between('request-start', 'request-end'), sanitizedUrl); + logger.measure(between('request-start', 'request-end'), host); if (!response.ok) { throw new Error( - `<---- HTTP/${response.status} ${options.method}: ${sanitizedUrl}`, + `<---- HTTP/${response.status} ${options.method}: ${safeUrl}`, ); } const contentType = response.headers.get('Content-Type'); diff --git a/src/common/core/setup.ts b/src/common/core/setup.ts index 1d153bb..5ed9a25 100644 --- a/src/common/core/setup.ts +++ b/src/common/core/setup.ts @@ -38,11 +38,34 @@ const applicationState: State = { authorization: null, contentType: null, acceptEncoding: '', - language: '', }, local: await _localSourceFactory.connect(), }; +const onDispose = (token: number) => { + setTimeout(() => { + Deno.removeSignalListener('SIGINT', onTerminationRequest); + Deno.removeSignalListener('SIGTERM', onTerminationRequest); + clearTimeout(token); + Deno.exit(); + }, 500); +}; + +const onTerminationRequest = (): void => { + logger.debug( + 'common.core.setup:onTerminationRequest: OS dispatched signal', + ); + const token = setTimeout(async () => await _localSourceFactory.disconnect()); + logger.debug( + 'common.core.setup:onTerminationRequest: Attempting to exit Deno process', + ); + + onDispose(token); +}; + +Deno.addSignalListener('SIGINT', onTerminationRequest); +Deno.addSignalListener('SIGTERM', onTerminationRequest); + logger.mark('setup-end'); logger.measure(between('setup-start', 'setup-end')); diff --git a/src/common/core/utils.ts b/src/common/core/utils.ts index d0775b6..f595e55 100644 --- a/src/common/core/utils.ts +++ b/src/common/core/utils.ts @@ -35,4 +35,7 @@ export const pagination = (page: number, count: number) => { return { from, to }; }; +export const isNullOrUndefined = (obj: unknown) => + obj == null || obj == undefined; + export const port = env('PORT'); diff --git a/src/common/middleware/targeting.ts b/src/common/middleware/targeting.ts index 991b8e4..592d346 100644 --- a/src/common/middleware/targeting.ts +++ b/src/common/middleware/targeting.ts @@ -1,6 +1,6 @@ import { between } from 'x/optic'; import { State } from '../types/state.d.ts'; -import { UAParser } from 'esm/ua_parser'; +import { UserAgent } from 'std/http'; import { logger } from '../core/logger.ts'; import type { AppContext } from '../types/core.d.ts'; @@ -22,8 +22,7 @@ const contextAttributes = ( }; } - const parser = new UAParser(agent); - const { browser, cpu, device, engine, os } = parser.getResult(); + const { browser, cpu, device, engine, os } = new UserAgent(agent); return Promise.resolve({ 'browser_name': browser.name, diff --git a/src/common/mongo/collection.ts b/src/common/mongo/collection.ts index 55e6e78..cacd3d7 100644 --- a/src/common/mongo/collection.ts +++ b/src/common/mongo/collection.ts @@ -1,7 +1,7 @@ -import { Document } from 'x/mongo'; +import { Document } from 'npm/mongodb'; import { Local } from '../types/core.d.ts'; -export const getCollection = ( - collection: string, +export const collection = ( + name: string, database: Local, -) => database?.collection(collection); +) => database?.collection(name); diff --git a/src/common/mongo/factory.ts b/src/common/mongo/factory.ts index 38b81bc..04f2ce3 100644 --- a/src/common/mongo/factory.ts +++ b/src/common/mongo/factory.ts @@ -1,46 +1,52 @@ -import { MongoClient } from 'x/mongo'; +import { MongoClient } from 'npm/mongodb'; import { logger } from '../core/logger.ts'; import { between } from 'x/optic'; import { env } from '../core/env.ts'; import { Local } from '../types/core.d.ts'; class LocalSourceFactory { - constructor( - private readonly options: string, - private readonly client: MongoClient, - ) { + constructor(private readonly client: MongoClient) { logger.mark('mongo_connection_start'); + client.on('timeout', () => { + logger.warn( + 'common.mongo.factory:LocalSourceFactory: Connection timed out', + ); + }); } connect = async (): Promise => { - try { - const db = await this.client.connect(this.options); - logger.mark('mongo_connection_end'); - logger.measure( - between('mongo_connection_start', 'mongo_connection_end'), - ); - return db; - } catch (e) { - logger.error('common.mongo.factory:connect:', e); - return undefined; - } + return await this.client.connect() + .then((client) => { + logger.mark('mongo_connection_end'); + logger.measure( + between('mongo_connection_start', 'mongo_connection_end'), + ); + return client.db(); + }) + .catch((e) => { + logger.error('common.mongo.factory:connect:', e); + return undefined; + }); }; - disconnect = () => { + disconnect = async () => { logger.mark('mongo_close_start'); - try { - this.client.close(); - } catch (e) { - logger.error('common.mongo.factory:disconnect:', e); - } - logger.mark('mongo_close_end'); - logger.measure(between('mongo_close_start', 'mongo_close_end')); + await this.client.close(true) + .then(() => { + }).catch((e) => { + logger.error('common.mongo.factory:disconnect:', e); + }).finally(() => { + logger.mark('mongo_close_end'); + logger.measure(between('mongo_close_start', 'mongo_close_end')); + }); }; } const _localSourceFactory = new LocalSourceFactory( - env('MONGO_URL'), - new MongoClient(), + new MongoClient(env('MONGO_URL'), { + connectTimeoutMS: 1000, + monitorCommands: true, + }), ); export default _localSourceFactory; diff --git a/src/common/mongo/index.ts b/src/common/mongo/index.ts index ed44932..056a3cd 100644 --- a/src/common/mongo/index.ts +++ b/src/common/mongo/index.ts @@ -1 +1,3 @@ export * from './collection.ts'; +export * from './utils.ts'; +export * from './types.d.ts'; diff --git a/src/common/mongo/types.d.ts b/src/common/mongo/types.d.ts index e69de29..98b5641 100644 --- a/src/common/mongo/types.d.ts +++ b/src/common/mongo/types.d.ts @@ -0,0 +1,15 @@ +import { Document, OptionalId, SortDirection } from 'npm/mongodb'; + +export type Optional = T | undefined | null; + +export type ProjectionOption = { + [K in keyof OptionalId]?: 0 | 1; +}; + +export type SortOption = { + [K in keyof OptionalId]?: SortDirection; +}; + +export interface EntityCursor { + cursor: string; +} diff --git a/src/common/mongo/utils.ts b/src/common/mongo/utils.ts new file mode 100644 index 0000000..7684c8f --- /dev/null +++ b/src/common/mongo/utils.ts @@ -0,0 +1,11 @@ +import { Document, ObjectId, Sort } from 'npm/mongodb'; +import { ProjectionOption, SortOption } from './types.d.ts'; + +export const projectionOf = ( + projection: ProjectionOption, +) => projection; + +export const sortOf = (option: SortOption): Sort => + option as Sort; + +export const idOf = (id: ObjectId): string => id.toHexString(); diff --git a/src/common/types/core.d.ts b/src/common/types/core.d.ts index cea3f72..2e44394 100644 --- a/src/common/types/core.d.ts +++ b/src/common/types/core.d.ts @@ -1,7 +1,7 @@ import { Context } from 'x/oak'; import { State } from './state.d.ts'; import { GrowthBook } from 'esm/growthbook'; -import { Database } from 'x/mongo'; +import { Db } from 'npm/mongodb'; import { AppFeatures } from '../experiment/types.d.ts'; export type RCF822Date = string; @@ -14,4 +14,4 @@ export type AppContext = Context; export type Features = GrowthBook; -export type Local = Database | undefined; +export type Local = Db | undefined; diff --git a/src/common/types/paging.d.ts b/src/common/types/paging.d.ts index e945548..8e530e2 100644 --- a/src/common/types/paging.d.ts +++ b/src/common/types/paging.d.ts @@ -1,7 +1,7 @@ import { IResponse } from './response.d.ts'; export interface IPaging extends IResponse { - page: number; + first?: string; + last?: string; count: number; - total: number; } diff --git a/src/config/index.ts b/src/config/index.ts index 15e5d65..33799ab 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,10 +1,10 @@ import { AppContext } from '../common/types/core.d.ts'; import { Repository } from './repository/index.ts'; import { LocalSource } from './local/index.ts'; -import { getCollection } from '../common/mongo/index.ts'; +import { collection } from '../common/mongo/index.ts'; export const config = async ({ state, response }: AppContext) => { - const localSource = new LocalSource(getCollection('config', state.local)); + const localSource = new LocalSource(collection('config', state.local)); const repository = new Repository( state.features, localSource, diff --git a/src/config/local/source.ts b/src/config/local/source.ts index c6bb78a..71db7c9 100644 --- a/src/config/local/source.ts +++ b/src/config/local/source.ts @@ -1,13 +1,14 @@ -import { Collection } from 'x/mongo'; +import { Collection } from 'npm/mongodb'; import { logger } from '../../common/core/logger.ts'; import { ConfigDocument } from './types.d.ts'; +import { EntityWithId, Optional } from '../../common/mongo/types.d.ts'; export class LocalSource { constructor( private readonly collection?: Collection, ) {} - getConfig = async (): Promise => { + getConfig = async (): Promise>> => { const config = await this.collection?.findOne() ?.catch((e) => { logger.error( diff --git a/src/config/local/types.d.ts b/src/config/local/types.d.ts index 9c1ce2a..41951b1 100644 --- a/src/config/local/types.d.ts +++ b/src/config/local/types.d.ts @@ -1,4 +1,4 @@ -import { Document } from 'x/mongo'; +import { Document } from 'npm/mongodb'; export interface NavigationConfig extends Document { criteria: string; diff --git a/src/config/transformer/index.ts b/src/config/transformer/index.ts index 5e9d775..9a00001 100644 --- a/src/config/transformer/index.ts +++ b/src/config/transformer/index.ts @@ -2,6 +2,7 @@ import { getPlatformSource, isAnalyticsEnabled, } from '../../common/experiment/index.ts'; +import { EntityWithId, Optional } from '../../common/mongo/types.d.ts'; import { Transform } from '../../common/transformer/types.d.ts'; import { Features } from '../../common/types/core.d.ts'; import { ConfigDocument } from '../local/types.d.ts'; @@ -9,7 +10,7 @@ import { ClientConfiguration } from './types.d.ts'; export const transform: Transform< { - document: ConfigDocument | undefined; + document: Optional>; features: Features; }, ClientConfiguration diff --git a/src/import_map.json b/src/import_map.json index 93e2837..2e2f8fb 100644 --- a/src/import_map.json +++ b/src/import_map.json @@ -1,24 +1,19 @@ { "imports": { - "std/server": "https://deno.land/std@0.224.0/http/mod.ts", - "std/server/status": "https://deno.land/std@0.224.0/http/status.ts", - "std/json": "https://deno.land/std@0.224.0/json/mod.ts", + "std/http": "https://deno.land/std@0.224.0/http/mod.ts", "std/dotenv": "https://deno.land/std@0.224.0/dotenv/mod.ts", "std/collections": "https://deno.land/std@0.224.0/collections/mod.ts", - "std/testing/asserts": "https://deno.land/std@0.224.0/testing/asserts.ts", + "std/testing/asserts": "https://deno.land/std@0.224.0/assert/mod.ts", "std/testing/bdd": "https://deno.land/std@0.224.0/testing/bdd.ts", - "std/testing/mock": "https://deno.land/std@0.224.0/testing/mock.ts", - "std/testing/types": "https://deno.land/std@0.224.0/testing/types.ts", "x/optic": "https://deno.land/x/optic@1.3.11/mod.ts", "x/optic/formatters": "https://deno.land/x/optic@1.3.11/formatters/mod.ts", - "x/optic/regex-filter": "https://deno.land/x/optic@1.3.11/filters/regExpFilter.ts", "x/optic/profiler": "https://deno.land/x/optic@1.3.11/logger/profileMeasure.ts", - "x/mongo": "https://deno.land/x/mongo@v0.33.0/mod.ts", "x/oak": "https://deno.land/x/oak@v16.0.0/mod.ts", "x/xml": "https://deno.land/x/xml@4.0.0/mod.ts", "x/deepmerge": "https://deno.land/x/deepmergets@v5.1.0/dist/deno/index.ts", - "esm/ua_parser": "https://esm.sh/ua-parser-js@2.0.0-beta.2", + "x/fest": "https://deno.land/x/fest@4.13.2/mod.ts", "esm/growthbook": "https://esm.sh/@growthbook/growthbook@1.0.0", - "esm/logtail": "https://esm.sh/@logtail/node@0.4.21" + "esm/logtail": "https://esm.sh/@logtail/node@0.4.21", + "npm/mongodb": "npm:mongodb@6.6.1" } } diff --git a/src/index.ts b/src/index.ts index fedeb2f..a0e0315 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,24 +3,25 @@ import factory from './common/core/factory.ts'; import { port } from './common/core/utils.ts'; import { AppContext } from './common/types/core.d.ts'; import { config } from './config/index.ts'; -import { news } from './news/index.ts'; +import { news, newsWorker } from './news/index.ts'; import { series } from './series/index.ts'; const router = new Router({ methods: ['GET'], strict: true, + sensitive: true, }); router.get('/config', async (ctx: AppContext) => { await config(ctx); }); -router.get('/news-sync', async (ctx: AppContext) => { - await news(ctx, true); +router.get('/news/sync', async (ctx: AppContext) => { + await newsWorker(ctx); }); router.get('/news', async (ctx: AppContext) => { - await news(ctx, false); + await news(ctx); }); router.get('/series', async (ctx: AppContext) => { diff --git a/src/news/index.ts b/src/news/index.ts index 6195328..cd31171 100644 --- a/src/news/index.ts +++ b/src/news/index.ts @@ -1,26 +1,31 @@ import type { AppContext } from '../common/types/core.d.ts'; import { isNewsApiv2Enabled } from '../common/experiment/index.ts'; import NewsRepository, {} from './repository/index.ts'; -import LocalSource from './local/index.ts'; -import { STATUS_CODE } from 'std/server/status'; -import { getCollection } from '../common/mongo/index.ts'; +import LocalSource from './local/source.ts'; +import { STATUS_CODE } from 'std/http'; +import { collection } from '../common/mongo/index.ts'; -export const news = async ({ response, state }: AppContext, sync: boolean) => { +export const newsWorker = async ({ response, state }: AppContext) => { + const { local } = state; + const repository = new NewsRepository( + new LocalSource(collection('news', local)), + ); + + await repository.sync(); + response.status = STATUS_CODE.NoContent; +}; + +export const news = async ({ response, state }: AppContext) => { const { local, features } = state; const repository = new NewsRepository( - new LocalSource(getCollection('news', local)), + new LocalSource(collection('news', local)), ); - if (sync) { - await repository.sync(); - response.status = STATUS_CODE.NoContent; + if (isNewsApiv2Enabled(features)) { + response.type = 'application/json'; + response.body = await repository.getLatest(); } else { - if (isNewsApiv2Enabled(features)) { - response.type = 'application/json'; - response.body = await repository.getLatest(); - } else { - response.type = 'application/xml'; - response.body = await repository.getLatestLegacy(); - } + response.type = 'application/xml'; + response.body = await repository.getLatestLegacy(); } }; diff --git a/src/news/local/index.ts b/src/news/local/index.ts index 605505f..c98a4e3 100644 --- a/src/news/local/index.ts +++ b/src/news/local/index.ts @@ -1,92 +1,2 @@ -import { Collection } from 'x/mongo'; -import { logger } from '../../common/core/logger.ts'; -import { IPaging } from '../../common/types/paging.d.ts'; -import { IResponse } from '../../common/types/response.d.ts'; -import { fromEntity, toEntity } from '../mapper.ts'; -import { News } from '../types.d.ts'; -import { NewsDocument } from './types.d.ts'; - -export default class LocalSource { - constructor( - private readonly collection?: Collection, - ) {} - - saveAll = async (news: News[]) => { - const entities = news.map(toEntity); - await this.collection?.insertMany( - entities, - ) - .then((result) => { - logger.debug('ObjectId', result); - }).catch((e) => { - logger.error('Unable to save news to collection', e); - return undefined; - }); - }; - - getLatestPublishedDate = async (): Promise => { - const data = await this.collection - ?.find({ project: ['published_on'] }) - .sort({ published_on: -1 }) - .limit(1) - .next() - .catch((e) => { - logger.error(e); - return undefined; - }); - - return data?.published_on ?? 0; - }; - - getAll = async ( - page: number, - size: number, - ): Promise> => { - // const { from, to } = pagination(page, size); - const count = await this.collection?.estimatedDocumentCount(); - - const data = await this.collection?.find() - .sort({ - published_on: { ascending: false }, - }) - .limit(page) - .skip(size + page) - .toArray() - .then((data) => { - logger.debug( - `Retrieved ${data?.length} records from local source`, - ); - return data; - }) - .catch((error) => { - logger.error(error); - return undefined; - }); - - const items = data?.map(fromEntity); - - return { - total: count ?? 0, - count: items?.length ?? 0, - page: page, - data: items ?? null, - }; - }; - - get = async ( - id: string, - ): Promise> => { - const data = await this.collection - ?.findOne({ id: { $eq: id } }) - .catch((error) => { - logger.error(error); - return undefined; - }); - - const item = fromEntity(data!); - - return { - data: item ?? null, - }; - }; -} +export * from './transformer.ts'; +export * from './types.d.ts'; diff --git a/src/news/local/source.ts b/src/news/local/source.ts new file mode 100644 index 0000000..f122bba --- /dev/null +++ b/src/news/local/source.ts @@ -0,0 +1,160 @@ +import { Collection, Filter, FindOptions, ObjectId, WithId } from 'npm/mongodb'; +import { logger } from '../../common/core/logger.ts'; +import { IPaging } from '../../common/types/paging.d.ts'; +import { IResponse } from '../../common/types/response.d.ts'; +import { fromEntity, toEntity } from '../mapper.ts'; +import { News } from '../types.d.ts'; +import { NewsDocument, NewsId } from './types.d.ts'; +import { between } from 'x/optic/profiler'; +import { projectionOf, sortOf } from '../../common/mongo/index.ts'; + +export default class LocalSource { + constructor( + private readonly collection?: Collection, + ) {} + + saveAll = async (news: News[]) => { + logger.mark('news_source_insertMany_start'); + const entities = news.map(toEntity); + await this.collection?.insertMany(entities) + .then((document) => { + logger.debug( + 'news.local.source:saveAll: Saved documents', + document.insertedCount, + ); + logger.mark('news_source_insertMany_end'); + }).catch((e) => { + logger.error( + 'news.local.source:saveAll: Unable to save news to collection', + e, + ); + }).finally(() => { + logger.measure( + between( + 'news_source_insertMany_start', + 'news_source_insertMany_end', + ), + ); + }); + }; + + getLatestPublishedDate = async (): Promise => { + const filter: Filter = { + id: { $exists: true }, + }; + const options: FindOptions> = { + projection: projectionOf({ published_on: 1 }), + sort: sortOf({ published_on: 'desc' }), + }; + logger.mark('news_source_get_latest_published_date_start'); + return await this.collection + ?.findOne(filter, options) + ?.then((data) => { + logger.mark('news_source_get_latest_published_date_end'); + return data?.published_on; + }) + ?.catch((e) => { + logger.error( + 'news.local.source:getLatestPublishedDate: Unable to fetch item', + e, + ); + return undefined; + }) + ?.finally(() => { + logger.measure( + between( + 'news_source_get_latest_published_date_start', + 'news_source_get_latest_published_date_end', + ), + ); + }) ?? 0; + }; + + getAll = async (id?: NewsId): Promise> => { + const filter: Filter = id + ? { + _id: { $gt: new ObjectId(id.cursor) }, + } + : {}; + const options: FindOptions> = { + sort: sortOf({ published_on: 'desc' }), + limit: 25, + }; + + logger.mark('news_source_get_all_start'); + const documents = await this.collection?.find(filter, options) + ?.toArray() + ?.then((data) => { + logger.mark('news_source_get_all_end'); + return data; + }) + ?.catch((error) => { + logger.warn( + 'news.local.source:getAll: Error fetching news collection', + error, + ); + return undefined; + }).finally(() => { + logger.measure( + between('news_source_get_all_start', 'news_source_get_all_end'), + ); + }); + + const items = documents?.map(fromEntity) ?? null; + const count = items?.length ?? 0; + + let first, last: string | undefined = undefined; + + if (documents && count > 0) { + first = documents[0]._id.toHexString(); + last = documents[count - 1]._id.toHexString(); + } + + return { + count: count, + first: first, + last: last, + data: items, + }; + }; + + get = async ( + id: NewsId, + ): Promise> => { + const filter: Filter = { + slug: { $eq: id.uuid }, + }; + logger.mark('news_source_get_start'); + const document = await this.collection + ?.findOne(filter) + ?.then(() => { + logger.mark('news_source_get_end'); + }) + ?.catch((error) => { + logger.warn( + 'news.local.source:get: Error fetching news collection', + error, + ); + return undefined; + }).finally(() => { + logger.measure( + between( + 'news_source_get_start', + 'news_source_get_end', + ), + ); + }); + + if (document) { + const item = fromEntity(document); + + return { + data: item ?? null, + }; + } + + return { + data: null, + }; + }; +} diff --git a/src/news/local/transformer.ts b/src/news/local/transformer.ts index f022d82..1791ddb 100644 --- a/src/news/local/transformer.ts +++ b/src/news/local/transformer.ts @@ -1,4 +1,4 @@ -import { Document } from 'x/mongo'; +import { Document } from 'npm/mongodb'; import { Transform } from '../../common/transformer/types.d.ts'; import { News } from '../types.d.ts'; diff --git a/src/news/local/types.d.ts b/src/news/local/types.d.ts index ce68395..2c33d61 100644 --- a/src/news/local/types.d.ts +++ b/src/news/local/types.d.ts @@ -1,12 +1,19 @@ -import { Document, ObjectId } from 'x/mongo'; +import { Document } from 'npm/mongodb'; +import { EntityCursor } from '../../common/mongo/types.d.ts'; export interface NewsDocument extends Document { - _id: ObjectId; + id: string; + slug: string; title: string; - image: string; author: string; + category: string; description: string; content: string; - link: string; + image: string; published_on: number; + link: string; +} + +export interface NewsId extends EntityCursor { + uuid: string; } diff --git a/src/news/mapper.ts b/src/news/mapper.ts index 3059897..7dd2c9f 100644 --- a/src/news/mapper.ts +++ b/src/news/mapper.ts @@ -1,28 +1,32 @@ -import { ObjectId } from 'x/mongo'; -import { News } from './types.d.ts'; +import { News, NewsEntity } from './types.d.ts'; import { NewsDocument } from './local/types.d.ts'; +import { OptionalId, WithId } from 'npm/mongodb'; -export const fromEntity = (data: NewsDocument): News => { +export const fromEntity = (data: WithId): NewsEntity => { return { + id: data._id.toHexString(), + slug: data.slug, title: data.title, - image: data.image, author: data.author, + category: data.category, description: data.description, content: data.content, + image: data.image, + publishedOn: data.published_on, link: data.link, - publishedOn: data.publishedOn, }; }; -export const toEntity = (data: News): NewsDocument => { +export const toEntity = (data: News): OptionalId => { return { - _id: new ObjectId(data.link), + slug: data.slug, + title: data.title, author: data.author, - content: data.content, + category: data.category, description: data.description, + content: data.content, image: data.image, - link: data.link, published_on: data.publishedOn, - title: data.title, + link: data.link, }; }; diff --git a/src/news/repository/index.ts b/src/news/repository/index.ts index 33f58e0..7920ff5 100644 --- a/src/news/repository/index.ts +++ b/src/news/repository/index.ts @@ -1,43 +1,75 @@ import { News } from '../types.d.ts'; import { latestNews } from '../service/index.ts'; -import LocalSource from '../local/index.ts'; +import LocalSource from '../local/source.ts'; import { IPaging } from '../../common/types/paging.d.ts'; import { transform } from '../transformer.ts'; import { currentDate, isOlderThan } from '../../common/core/utils.ts'; import { IResponse } from '../../common/types/response.d.ts'; import { parse } from 'x/xml'; import { logger } from '../../common/core/logger.ts'; +import { NewsId } from '../local/types.d.ts'; +import { between } from 'x/optic/profiler'; export default class NewsRepository { constructor( private readonly local: LocalSource, ) {} - sync = async () => { - const latestDate = await this.local.getLatestPublishedDate(); - if (isOlderThan(currentDate(), latestDate, 4)) { - const content = await latestNews('en-US'); + sync = async (locale: string = 'en-US') => { + const publishedOn = await this.local.getLatestPublishedDate(); + logger.mark('news_repository_sync_cache_start'); + if (isOlderThan(currentDate(), publishedOn, 4)) { + const content = await latestNews(locale); const document = parse(content, { flatten: true }); const news = transform(document); this.local.saveAll(news); } else { - logger.info( - 'Not updating local source, cached instance is still valid', + logger.debug( + 'news.repository.index:sync: Not updating local source, cached instance is still valid', ); } + logger.mark('news_repository_sync_cache_end'); + logger.measure( + between('news_repository_cache_start', 'news_repository_cache_end'), + ); }; - getLatest = async (): Promise> => { - const result = await this.local.getAll(1, 25); + getLatest = async (id?: NewsId): Promise> => { + logger.mark('news_repository_get_latest_start'); + const result = await this.local.getAll(id); + logger.mark('news_repository_get_latest_end'); + logger.measure( + between( + 'news_repository_get_latest_start', + 'news_repository_get_latest_end', + ), + ); return result; }; - getLatestLegacy = async (): Promise => { - const result = await latestNews('en-US'); + getLatestLegacy = async (locale: string = 'en-US'): Promise => { + logger.mark('news_repository_get_latest_legacy_start'); + const result = await latestNews(locale); + logger.mark('news_repository_get_latest_legacy_end'); + logger.measure( + between( + 'news_repository_get_latest_legacy_start', + 'news_repository_get_latest_legacy_end', + ), + ); return result; }; - getById = async (id: string): Promise> => { - return await this.local.get(id); + getById = async (id: NewsId): Promise> => { + logger.mark('news_repository_get_by_id_start'); + const result = await this.local.get(id); + logger.mark('news_repository_get_by_id_end'); + logger.measure( + between( + 'news_repository_get_by_id_start', + 'news_repository_get_by_id_end', + ), + ); + return result; }; } diff --git a/src/news/transformer.ts b/src/news/transformer.ts index 5d4342e..b1040c8 100644 --- a/src/news/transformer.ts +++ b/src/news/transformer.ts @@ -7,6 +7,13 @@ const sanitize = (content: string): string => { return content.replace(regex, ''); }; +const extractSlug = (uri: string): string => { + const url = new URL(uri); + const pathname = url.pathname; + const pathSegments = pathname.split('/'); + return pathSegments[pathSegments.length - 1]; +}; + export const transform = ( // deno-lint-ignore no-explicit-any document: Record, @@ -23,13 +30,15 @@ export const transform = ( } return items.map((item) => { return { + slug: extractSlug(item.guid), title: item.title, - image: item['media:thumbnail']['@url'], author: item.author, + category: item.category, description: sanitize(item.description), content: item['content:encoded'], - link: item.link, + image: item['media:thumbnail']['@url'], publishedOn: toEpotch(item.pubDate), + link: item.link, }; }); }; diff --git a/src/news/types.d.ts b/src/news/types.d.ts index b851b89..f8611e0 100644 --- a/src/news/types.d.ts +++ b/src/news/types.d.ts @@ -1,9 +1,13 @@ export type News = { + slug: string; title: string; - image: string; author: string; + category: string; description: string; content: string; - link: string; + image: string; publishedOn: number; + link: string; }; + +export type NewsEntity = News & { id: string }; diff --git a/src/series/index.ts b/src/series/index.ts index 15e941d..f62774e 100644 --- a/src/series/index.ts +++ b/src/series/index.ts @@ -1,9 +1,9 @@ import { Status } from 'x/oak'; import { AppContext, Error } from '../common/types/core.d.ts'; -import LocalSource from './local/index.ts'; +import LocalSource from './local/source.ts'; import SeriesRepository from './repository/series.ts'; import SeasonRepository from './repository/season.ts'; -import { getCollection } from '../common/mongo/index.ts'; +import { collection } from '../common/mongo/index.ts'; export const series = async ({ request, response, state }: AppContext) => { const params = request.url.searchParams; @@ -11,9 +11,9 @@ export const series = async ({ request, response, state }: AppContext) => { const id = Number(params.get('id')); const series = await new SeriesRepository( - new LocalSource(getCollection('series', state.local)), + new LocalSource(collection('series', state.local)), new SeasonRepository(), - ).getById(id); + ).getById({ anilist: id }); response.type = 'application/json'; response.status = Status.OK; @@ -22,7 +22,7 @@ export const series = async ({ request, response, state }: AppContext) => { response.type = 'application/json'; response.status = Status.BadRequest; response.body = { - message: `Missing required argument 'id'`, + message: `Missing required query parameter: 'id'`, }; } }; diff --git a/src/series/local/index.ts b/src/series/local/index.ts index 08181a9..c98a4e3 100644 --- a/src/series/local/index.ts +++ b/src/series/local/index.ts @@ -1,46 +1,2 @@ -import { Collection, Document, Filter, FindAndModifyOptions } from 'x/mongo'; -import { logger } from '../../common/core/logger.ts'; -import { IResponse } from '../../common/types/response.d.ts'; -import { MediaWithSeason } from '../types.d.ts'; -import { transform } from './transformer.ts'; - -export default class LocalSource { - constructor( - private readonly collection?: Collection, - ) {} - - get = async (id: number): Promise> => { - const item = await this.collection - ?.findOne({ 'mediaId.anilist': id }) - ?.then((document) => transform(document)) - ?.catch((e) => { - logger.error( - `seriese.local.index.LocalSource:get: Unable to find '${id}' in collection`, - e, - ); - return undefined; - }); - - return { - data: item ?? null, - }; - }; - - save = async (media: MediaWithSeason) => { - const filter: Filter = { - 'mediaId.anilist': media.mediaId.anilist, - }; - const options: FindAndModifyOptions = { - upsert: true, - update: media, - }; - await this.collection?.findAndModify(filter, options).then((result) => { - logger.debug('seriese.local.index.LocalSource:save: ObjectId', result); - }).catch((e) => { - logger.error( - 'seriese.local.index.LocalSource:save: Unable to save media to collection', - e, - ); - }); - }; -} +export * from './transformer.ts'; +export * from './types.d.ts'; diff --git a/src/series/local/source.ts b/src/series/local/source.ts new file mode 100644 index 0000000..d13b8c2 --- /dev/null +++ b/src/series/local/source.ts @@ -0,0 +1,82 @@ +import { + Collection, + Document, + Filter, + FindOneAndReplaceOptions, +} from 'npm/mongodb'; +import { logger } from '../../common/core/logger.ts'; +import { IResponse } from '../../common/types/response.d.ts'; +import { MediaWithSeason } from '../types.d.ts'; +import { transform } from './transformer.ts'; +import { MediaDocument } from './types.d.ts'; +import { MediaParamId } from './types.d.ts'; +import { FindOptions } from 'npm/mongodb'; +import { between } from 'x/optic'; + +export default class LocalSource { + constructor( + private readonly collection?: Collection, + ) {} + + get = async (mediaId: MediaParamId): Promise> => { + const filter: Filter = { + 'mediaId.anilist': mediaId.anilist, + }; + const options: FindOptions = {}; + logger.mark('series_source_get_start'); + const document = await this.collection + ?.findOne(filter, options) + ?.then((document) => { + logger.debug( + `seriese.local.source:get: Result from collection lookup`, + document?._id, + ); + logger.mark('series_source_get_end'); + return document; + }) + ?.catch((e) => { + logger.warn( + `seriese.local.source:get: Unable to find media in collection`, + [mediaId, e], + ); + return undefined; + }) + ?.finally(() => { + logger.measure( + between('series_source_get_start', 'series_source_get_end'), + ); + }); + + return { + data: transform(document) ?? null, + }; + }; + + save = async (media: MediaWithSeason) => { + const filter: Filter = { + 'mediaId.anilist': media.mediaId.anilist, + }; + const options: FindOneAndReplaceOptions = { + upsert: true, + }; + const replacement: MediaDocument = { + ...media, + }; + + logger.mark('series_source_save_start'); + await this.collection?.findOneAndReplace(filter, replacement, options) + ?.then((result) => { + logger.debug('seriese.local.source:save: Saved document', result?._id); + logger.mark('series_source_save_end'); + })?.catch((e) => { + logger.error( + 'seriese.local.source:save: Unable to save collection', + [filter, e], + ); + })?.finally(() => { + logger.measure( + between('series_source_save_start', 'series_source_save_end'), + ); + }); + }; +} diff --git a/src/series/local/transformer.ts b/src/series/local/transformer.ts index 08dc7cd..010e498 100644 --- a/src/series/local/transformer.ts +++ b/src/series/local/transformer.ts @@ -1,16 +1,38 @@ -import { Document } from 'x/mongo'; +import { WithId } from 'npm/mongodb'; import { Transform } from '../../common/transformer/types.d.ts'; -import { MediaWithSeason } from '../types.d.ts'; +import { MediaEntity } from '../types.d.ts'; +import { idOf, Optional } from '../../common/mongo/index.ts'; +import { MediaDocument } from './types.d.ts'; -export const transform: Transform< - Document | undefined, - MediaWithSeason | undefined -> = ( - sourceData, -) => { - if (sourceData) { - const { _id, ...rest } = sourceData; - return rest as MediaWithSeason; - } - return undefined; +const map = ( + document: WithId, +): Optional => { + return { + id: idOf(document._id), + mediaId: document.mediaId, + cover: document.cover, + banner: document.banner, + fanart: document.fanart, + format: document.format, + status: document.status, + source: document.source, + title: document.title, + themeSongs: document.themeSongs, + schedule: document.schedule, + ageRating: document.ageRating, + isAdult: document.isAdult, + trailers: document.trailers, + networks: document.networks, + image: document.image, + homepage: document.homepage, + description: document.description, + updatedAt: document.updatedAt, + airedEpisodes: document.airedEpisodes, + seasons: document.seasons, + }; }; + +export const transform: Transform< + Optional>, + Optional +> = (sourceData) => sourceData ? map(sourceData) : undefined; diff --git a/src/series/local/types.d.ts b/src/series/local/types.d.ts new file mode 100644 index 0000000..d876e68 --- /dev/null +++ b/src/series/local/types.d.ts @@ -0,0 +1,9 @@ +import { Document } from 'npm/mongodb'; +import { MediaWithSeason } from '../types.d.ts'; + +export interface MediaDocument extends Document, MediaWithSeason { +} + +export interface MediaParamId { + anilist: number; +} diff --git a/src/series/repository/series.ts b/src/series/repository/series.ts index 30799ff..fa28adc 100644 --- a/src/series/repository/series.ts +++ b/src/series/repository/series.ts @@ -12,7 +12,7 @@ import { getTraktShow } from '../service/trakt/index.ts'; import { seriesTransform } from '../transformer/series.ts'; import { MediaWithSeason } from '../types.d.ts'; import { isManga } from '../utils/index.ts'; -import LocalSource from '../local/index.ts'; +import LocalSource from '../local/source.ts'; import SeasonRepository from './season.ts'; import { seasonTransformer } from '../transformer/season.ts'; import { Theme } from '../service/theme/transformer/types.d.ts'; @@ -22,6 +22,7 @@ import { TmdbShow } from '../service/tmdb/types.d.ts'; import { AnimeRelationId } from '../service/arm/types.d.ts'; import { MergedSeason } from '../transformer/types.d.ts'; import { currentDate, isOlderThan } from '../../common/core/utils.ts'; +import { MediaParamId } from '../local/index.ts'; export default class SeriesRepository { constructor( @@ -30,9 +31,9 @@ export default class SeriesRepository { ) {} private fetchFromRemote = async ( - anilist: number, + id: MediaParamId, ): Promise => { - const relation = await getAniListRelationId(anilist); + const relation = await getAniListRelationId(id.anilist); const [notify, mal] = await Promise.all([ getNotifyAnime(relation?.notify), @@ -82,8 +83,8 @@ export default class SeriesRepository { return result; }; - getById = async (anilist: number): Promise> => { - const localContent = await this.local.get(anilist); + getById = async (id: MediaParamId): Promise> => { + const localContent = await this.local.get(id); if (localContent.data != null) { if (!isOlderThan(currentDate(), localContent.data.updatedAt, 2 * 24)) { @@ -91,7 +92,7 @@ export default class SeriesRepository { } } - const remoteContent = await this.fetchFromRemote(anilist); + const remoteContent = await this.fetchFromRemote(id); return { data: remoteContent, diff --git a/src/series/types.d.ts b/src/series/types.d.ts index 135a5d0..48c0a25 100644 --- a/src/series/types.d.ts +++ b/src/series/types.d.ts @@ -159,3 +159,7 @@ export interface Media { export interface MediaWithSeason extends Media { seasons: SeriesSeason[]; } + +export type MediaEntity = Media & MediaWithSeason & { + id: string; +};