Merge pull request #17 from sucho-archiving/v2

v2.0

Author: simonwiles

.github/workflows/update-media-and-deploy-site.yml

@@ -15,13 +15,13 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 18
-          cache: "yarn"
+          cache: "pnpm"
 
       - name: Install Dependencies
-        run: yarn install --frozen-lockfile
+        run: pnpm install --frozen-lockfile
 
       - name: Update Media Assets
-        run: yarn update-media
+        run: pnpm update-media
 
       - name: Check for Changes
         id: check
@@ -43,13 +43,6 @@ jobs:
           git commit -m "Automated Media Update $(date +"%Y-%m-%d")"
           git push --force
 
-      # if building the site, graphicsmagick is needed for creating derivative images
-      - name: Install graphicsmagick
-        if: steps.check.outputs.has_new_media == 'true'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y graphicsmagick
-
       - name: Build and Deploy Site
         if: steps.check.outputs.has_new_media == 'true'
-        run: yarn gh-deploy
+        run: pnpm gh-deploy

.gitignore

@@ -5,18 +5,13 @@ dist/
 node_modules/
 
 # logs
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 pnpm-debug.log*
 
-
-# environment variables
-.env
-.env.production
-
 # macOS-specific files
 .DS_Store
 
-
 .vscode/
+*.cpuprofile
+
+# assets generated at build-time
+assets/

.npmrc

@@ -0,0 +1 @@
+shamefully-hoist = true

README.md

@@ -18,9 +18,9 @@ The codebase in this repository uses the [Astro static site builder](https://ast
 
 ## Dataset and Asset Generation
 
-When `yarn dev` or `yarn build` is run, the build process will fetch and parse the latest data from the curated "Ready" tab on the submission sheet and synchronize the local media cache with the contents of the submissions folder on Google Drive.  The identifiers for the submission sheet and tab, etc. are found in [`src/config.mjs`](src/config.mjs).
+When `pnpm dev` or `pnpm build` is run, the build process will fetch and parse the latest data from the curated "Ready" tab on the submission sheet and synchronize the local media cache with the contents of the submissions folder on Google Drive.  The identifiers for the submission sheet and tab, etc. are found in [`src/config.mjs`](src/config.mjs).
 
-Running `yarn update-media` will fetch and synchronize the media files and then exit, and `yarn print-dataset` will fetch and parse the data and media files and then output the parsed data to `stdout` (it can take arguments to output different parts of the parsed dataset -- see [`src/dataset.mjs`](src/dataset.mjs) for details).
+Running `pnpm update-media` will fetch and synchronize the media files and then exit, and `pnpm print-dataset` will fetch and parse the data and media files and then output the parsed data to `stdout` (it can take arguments to output different parts of the parsed dataset -- see [`src/dataset.mjs`](src/dataset.mjs) for details).
 
 
 ## Development
@@ -30,24 +30,26 @@ Running `yarn update-media` will fetch and synchronize the media files and then
 
 ### Live-Updating Development Build
 
-1. Run `yarn` to install dependencies.
-1. Run `yarn dev` to start the development server and reflect your changes to the page as you save files.
+1. Run `pnpm install` to install dependencies.
+1. Run `pnpm dev` to start the development server and reflect your changes to the page as you save files.
 1. Visit `http://localhost:3000/` in your browser.
 
 
 ### Static Build
 
-1. Run `yarn build` to create a production build (the deployment url can be set in [`astro.config.mjs`](astro.config.mjs) but only matters for OpenGraph/Twitter metadata etc.).
-2. Run `yarn preview` to serve the contents of `dist/` using an http server at `http://localhost:3000/`.
+1. Run `pnpm build` to create a production build (the deployment url can be set in [`astro.config.mjs`](astro.config.mjs) but only matters for OpenGraph/Twitter metadata etc.).
+2. Run `pnpm preview` to serve the contents of `dist/` using an http server at `http://localhost:3000/`.
 
 
 ### Deploy to GH Pages
 
-1. Users with write permissions to this repository can run `yarn gh-deploy` from the project root to install dependencies, build the static assets, and update and push to the `gh-pages` branch.  
+1. Users with write permissions to this repository can run `pnpm gh-deploy` from the project root to install dependencies, build the static assets, and update and push to the `gh-pages` branch.  
   **Note:** the GitHub Actions workflow at [`.github/workflows/update-media-and-deploy-site.yml`](.github/workflows/update-media-and-deploy-site.yml) runs once a day and will automatically acquire new media, update the repository, and build and deploy the site as appropriate -- manual deploys should not generally be required.
 
 
 ### Additional Development Commands
 
-* `yarn update-media` (requires `node >= v18`) will fetch the spreadsheet, update the local working tree with any new (or missing) media files fetched from Google Drive, and delete any media files from `meme_media/` that do not correspond to records in the spreadsheet.
-* `yarn print-dataset` (requires `node >= v18`) will fetch the spreadsheet, process the data and media files, and then output the parsed dataset to `stdout` in JSON format.
+* `pnpm update-media` (requires `node >= v18`) will fetch the spreadsheet, update the local working tree with any new (or missing) media files fetched from Google Drive, and delete any media files from `meme_media/` that do not correspond to records in the spreadsheet.
+* `pnpm print-dataset` (requires `node >= v18`) will fetch the spreadsheet, process the data and media files, and then output the parsed dataset to `stdout` in JSON format.
+
+* All `pnpm` commands can be prepended with `LOG_LEVEL=DEBUG` for additional debugging information on the console.

astro.config.mjs

@@ -1,11 +1,14 @@
 import { defineConfig } from "astro/config";
+import svelte from "@astrojs/svelte";
 import { astroImageTools } from "astro-imagetools";
 
+import { memewallOgImage } from "./integrations/memewall-og-image.ts";
+
 // https://astro.build/config
 export default defineConfig({
   site: "https://memes.sucho.org/",
   trailingSlash: "always",
-  integrations: [astroImageTools],
+  integrations: [astroImageTools, svelte(), memewallOgImage()],
   vite: {
     ssr: { external: ["neat-csv"] },
     optimizeDeps: { exclude: ["neat-csv"] },

bin/astro-gh-deploy.sh

@@ -17,7 +17,7 @@ current_branch=$(git rev-parse --abbrev-ref HEAD);
 
 unclean=$(git status --porcelain) && [ -n "$unclean" ] && abort "Working directory is not clean -- aborting!";
 
-yarn build;
+pnpm build;
 [ ! -d "$BUILD_FOLDER" ] && abort "'$BUILD_FOLDER' does not exist -- aborting!";
 
 COMMIT_MESSAGE="Deploy from $(git log -n 1 --format="%h" HEAD) at $(date +"%Y-%m-%d %H:%M:%S %Z")";

integrations/memewall-og-image.ts

@@ -0,0 +1,62 @@
+import { spawn } from "child_process";
+
+import { pEventIterator } from "p-event";
+import { chromium } from "playwright";
+
+import type { AstroIntegration } from "astro";
+
+const delay = (milliseconds: number) =>
+  new Promise((resolve) => {
+    setTimeout(resolve, milliseconds);
+  });
+
+const memewallOgImage = (): AstroIntegration => ({
+  name: "memewall-og-image",
+  hooks: {
+    "astro:build:start": async () => {
+      const astroServerProcess = spawn("pnpm", ["dev"]);
+      let astroServerUrl: string | undefined;
+
+      const asyncIterator = pEventIterator(astroServerProcess.stdout, "data");
+      for await (const chunk of asyncIterator) {
+        // wait for the Astro dev server to be ready and to report its URL
+        if (
+          (astroServerUrl = chunk
+            .toString()
+            .match(/http:\/\/[\w\d]+:\d+\//)?.[0])
+        ) {
+          break;
+        }
+      }
+
+      if (!astroServerUrl) {
+        throw new Error("Server exited without reporting a URL!");
+      }
+
+      console.log(` --> Spawned Astro server listening at ${astroServerUrl}`);
+
+      const browser = await chromium.launch({
+        args: [
+          "--no-sandbox",
+          "--disable-setuid-sandbox",
+          "--disable-web-security",
+        ],
+      });
+
+      const page = await browser.newPage();
+      await page.setViewportSize({ width: 1200, height: 630 });
+      await page.goto(astroServerUrl, { waitUntil: "networkidle" });
+
+      // `waitUntil: "networkidle"` seems to be inadequate
+      console.log(` --> Waiting for the page to render before capturing...`);
+      await delay(1000);
+
+      await page.screenshot({ path: `./assets/open-graph.jpeg` });
+
+      await browser.close();
+      astroServerProcess.kill();
+    },
+  },
+});
+
+export { memewallOgImage };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sucho/meme-wall",
-  "version": "1.4.2",
+  "version": "2.0-rc-1",
   "private": true,
   "scripts": {
     "dev": "astro dev",
@@ -12,22 +12,32 @@
     "gh-deploy": "CNAME=memes.sucho.org bin/astro-gh-deploy.sh"
   },
   "devDependencies": {
+    "@astrojs/svelte": "^2.0.1",
     "astro": "^2.0.4",
-    "astro-imagetools": "^0.7.6",
+    "astro-imagetools": "^0.8.1",
     "csv-writer": "^1.6.0",
-    "gm": "^1.25.0",
     "image-size": "^1.0.2",
+    "loglevel": "^1.8.1",
     "neat-csv": "^7.0.0",
+    "p-event": "^5.0.1",
+    "playwright": "^1.31.2",
     "postcss-nested": "^6.0.0",
-    "prettier-plugin-astro": "^0.8.0"
+    "prettier-plugin-astro": "^0.8.0",
+    "svelte": "^3.55.1"
   },
   "dependencies": {
+    "@floating-ui/dom": "^1.2.1",
     "@fontsource/jost": "^4.5.13",
     "charts.css": "^0.9.0"
   },
-  "resolutions": {
-    "astro-imagetools/file-type": "^17.1.3",
-    "file-type": "^16.5.4",
-    "sharp": "^0.30.6"
+  "pnpm": {
+    "overrides": {
+      "astro-imagetools>file-type": "^17.1.3",
+      "file-type": "^16.5.4",
+      "sharp": "^0.30.6"
+    },
+    "patchedDependencies": {
+      "astro-imagetools@0.8.1": "patches/astro-imagetools@0.8.1.patch"
+    }
   }
 }

patches/astro-imagetools@0.8.1.patch

@@ -0,0 +1,60 @@
+diff --git a/api/utils/getImage.js b/api/utils/getImage.js
+index 12bc1a8fe664f77614d46df0619724239c07af60..147d0d012c06ec976eb90aa81883dd272c309971 100644
+--- a/api/utils/getImage.js
++++ b/api/utils/getImage.js
+@@ -37,7 +37,9 @@ export default async function ({
+ 
+   rest.aspect = `${imageWidth / imageHeight}`;
+ 
+-  if (!fallbackFormat) {
++  if (fallbackFormat === false) {
++    fallbackFormat = format;
++  } else if (!fallbackFormat) {
+     fallbackFormat = imageFormat;
+   }
+ 
+diff --git a/api/utils/getBreakpoints.js b/api/utils/getBreakpoints.js
+index 02a098e5d0415ca896b225fe6b235364d08af8f5..e04c2aa4c89d11556b751f82f27f41be177718cb 100644
+--- a/api/utils/getBreakpoints.js
++++ b/api/utils/getBreakpoints.js
+@@ -1,5 +1,5 @@
+ // @ts-check
+-import printWarning from "../../utils/printWarning.js";
++import log from "loglevel";
+ 
+ export default function getBreakpoints(breakpoints, imageWidth) {
+   if (Array.isArray(breakpoints)) {
+@@ -12,10 +12,7 @@ export default function getBreakpoints(breakpoints, imageWidth) {
+     if (breakpoints?.maxWidth) return breakpoints.maxWidth;
+ 
+     if (imageWidth > 3840) {
+-      printWarning({
+-        message:
+-          "The width of the source image is greater than 3840px. The generated breakpoints will be capped at 3840px. If you need breakpoints larger than this, please pass the maxWidth option to the breakpoints property.",
+-      });
++      log.debug("The width of the source image is greater than 3840px. The generated breakpoints will be capped at 3840px. If you need breakpoints larger than this, please pass the maxWidth option to the breakpoints property.");
+ 
+       return 3840;
+     }
+diff --git a/api/utils/getImage.js b/api/utils/getImage.js
+index 12bc1a8fe664f77614d46df0619724239c07af60..10f404af7f1f2b3553c25f9dda39c1d5649f1a48 100644
+--- a/api/utils/getImage.js
++++ b/api/utils/getImage.js
+@@ -5,6 +5,8 @@ import getImageSources from "./getImageSources.js";
+ import getProcessedImage from "./getProcessedImage.js";
+ import getArtDirectedImages from "./getArtDirectedImages.js";
+ 
++import log from "loglevel";
++
+ const imagesData = new Map();
+ 
+ export default async function ({
+@@ -83,7 +85,7 @@ export default async function ({
+ 
+   const end = performance.now();
+ 
+-  console.log(
++  log.debug(
+     `Responsive Image sets generated for ${type} at ${args[0].src} in ${
+       end - start
+     }ms`