feat: add workflows scripts

.github/workflows/refetch-articles.yml

@@ -0,0 +1,27 @@
+# ✍️ Description:
+# This action  will check the latest rss feed fetched  and check if there ara any new posts
+# Should be run on a schedule, e.g. every day at 1am and update the metadata for 10 repos at a time
+name: Refetch article
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  refetch-article:
+    name: Refetch article
+    runs-on: ubuntu-latest
+    timeout-minutes: 4
+
+    steps:
+      - name: 📦 Checkout project repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: 📦 Setupbun + install deps
+        uses: ./.github/actions/setup-bun-and-install
+
+      - name: 🏃‍♂️ Refetch article from rss
+        run: bun run refetch-articles

.github/workflows/refetch-podcast-episodes.yml

@@ -0,0 +1,26 @@
+# ✍️ Description:
+# This action  will check spotify API and check for the last podcasts episodes and add them to the database
+# Should be run on a schedule, e.g. every day at 1am and update the metadata for 10 repos at a time
+name: Refetch podcasts episodes (Spotify)
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+jobs:
+  refetch-podcasts-episodes:
+    name: Refetch podcasts episodes
+    runs-on: ubuntu-latest
+    timeout-minutes: 4
+
+    steps:
+      - name: 📦 Checkout project repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: 📦 Setupbun + install deps
+        uses: ./.github/actions/setup-bun-and-install
+
+      - name: 🏃‍♂️ Refetch podcasts episodes
+        run: bun run refetch-podcasts-episodes

.github/workflows/update-articles-metadata.yml

@@ -0,0 +1,27 @@
+# ✍️ Description:
+# This action is to add missed article metadata for fetched articles.
+# Should be run on a schedule, e.g. every day at 1am and update the metadata for 20 repos at a time
+name: Article Metadata Update
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  update-articles-metadata:
+    name: Update Articles metadata
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+
+    steps:
+      - name: 📦 Checkout project repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: 📦 Setupbun + install deps
+        uses: ./.github/actions/setup-bun-and-install
+
+      - name: 🏃‍♂️ Update Articles metadata
+        run: bun run update-articles-metadata

.github/workflows/update-repos-metadata.yml

@@ -0,0 +1,27 @@
+# ✍️ Description:
+# This action is to update github repos with the latest metadata (stars, forks, etc) from the GitHub API.
+# Should be run on a schedule, e.g. every day at 1am and update the metadata for 10 repos at a time
+name: Repos Metadata Update
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  update-repos-metadata:
+    name: Repos Metadata Update
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+
+    steps:
+      - name: 📦 Checkout project repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: 📦 Setupbun + install deps
+        uses: ./.github/actions/setup-bun-and-install
+
+      - name: 🏃‍♂️ Update Repos metadata
+        run: bun run update-repos-metadata

package.json

@@ -7,7 +7,11 @@
     "dev": "next dev",
     "lint": "SKIP_ENV_VALIDATION=1 next lint",
     "start": "next start",
-    "type-check": "tsc --noEmit"
+    "type-check": "tsc --noEmit",
+    "refetch-articles": "bun ./src/scripts/refetch-articles.ts",
+    "refetch-podcasts-episodes": "bun ./src/scripts/refetch-podcasts-episodes.ts",
+    "update-articles-metadata": "bun ./src/scripts/update-articles-metadata.ts",
+    "update-repos-metadata": "bun ./src/scripts/update-repos-metadata.ts"
   },
   "dependencies": {
     "@clerk/nextjs": "^4.26.2",

src/scripts/rss-refresher.ts → src/scripts/refetch-articles.ts

@@ -1,8 +1,9 @@
 #!/usr/bin/bun
 
 await import("../env.mjs");
-import { getXataClient } from "~/xata";
+import { type BlogsRecord, getXataClient } from "~/xata";
 import { extractRssFeed } from "~/utils/extract-rss-feed";
+import type { SelectedPick } from "@xata.io/client";
 
 /**
  * This script is used to fetch the RSS feed from blogs and update the article table in the database.
@@ -62,20 +63,10 @@ const addBulkArticles = async (articles: Article[]): Promise<number> => {
   }
 };
 
-const rssRefresher = async () => {
-  // only select blogs that have an rss feed and already validated by admin
-  // TODO: we should also check if last_rss_retrieved_at is older than 24h at least
-  const blog = await getXataClient()
-    .db.blogs.select(["rss", "title", "id"])
-    .filter({ draft: false, rss: { $contains: "http" } })
-    .sort("last_rss_retrieved_at", "asc")
-    .getFirst();
-
-  if (!blog) {
-    console.info("🚨 No blog found");
-    return;
-  }
-  console.log(`✅ Fetching RSS feed for ${blog.title} - ${blog.rss}`);
+const fetchRssFeed = async (
+  blog: SelectedPick<BlogsRecord, ("title" | "rss" | "id")[]>,
+) => {
+  console.log(`\n \n🚀 Fetching RSS feed for ${blog.title} - ${blog.rss} 🚀`);
   const feed = await extractRssFeed(blog.rss!);
 
   if (feed?.entries === undefined) {
@@ -118,4 +109,16 @@ const rssRefresher = async () => {
   }
 };
 
+const rssRefresher = async () => {
+  const blogs = await getXataClient()
+    .db.blogs.select(["rss", "title", "id"])
+    .filter({ draft: false, rss: { $contains: "http" } })
+    .sort("last_rss_retrieved_at", "asc")
+    .getMany();
+
+  for (const blog of blogs) {
+    await fetchRssFeed(blog);
+  }
+};
+
 await rssRefresher();

src/scripts/podcast-fetcher.ts → src/scripts/refetch-podcasts-episodes.ts

@@ -1,8 +1,14 @@
 #!/usr/bin/bun
 
 await import("../env.mjs");
+import type { SelectedPick } from "@xata.io/client";
 import { getShowEpisodes } from "~/utils/spotify";
-import { type EpisodesRecord, getXataClient } from "~/xata";
+import {
+  type EpisodesRecord,
+  getXataClient,
+  type PodcastsRecord,
+} from "~/xata";
+import { sleep } from "./utils";
 
 const MAX_EPISODE_PER_PODCAST = 20;
 type OPError = {
@@ -51,13 +57,9 @@ const addBulkEpisodes = async (episodes: EpisodesRecord[]): Promise<number> => {
   }
 };
 
-const podcastFetcher = async () => {
-  const podcast = await getXataClient()
-    .db.podcasts.select(["spotify_url", "id"])
-    .filter({ draft: false })
-    .sort("last_rss_retrieved_at", "asc")
-    .getFirst();
-
+const fetchPodcastEpisodes = async (
+  podcast: SelectedPick<PodcastsRecord, ("spotify_url" | "id")[]>,
+) => {
   if (!podcast || !podcast.spotify_url) {
     console.log("✅ No podcasts");
     return;
@@ -68,7 +70,7 @@ const podcastFetcher = async () => {
     return;
   }
   console.log(
-    `✅ Start refetching podcast episodes for ${podcast.spotify_url}`,
+    `\n\n🚀 Start refetching podcast episodes for ${podcast.spotify_url} 🚀`,
   );
   const data = await getShowEpisodes(showId);
   const episodes: Partial<EpisodesRecord>[] = data
@@ -104,4 +106,17 @@ const podcastFetcher = async () => {
   }
 };
 
+const podcastFetcher = async () => {
+  const podcasts = await getXataClient()
+    .db.podcasts.select(["spotify_url", "id"])
+    .filter({ draft: false })
+    .sort("last_rss_retrieved_at", "asc")
+    .getMany();
+
+  for (const pod of podcasts) {
+    await fetchPodcastEpisodes(pod);
+    await sleep(3000); // Make sure to not exceed the rate limit for the spotify API
+  }
+};
+
 await podcastFetcher();

src/scripts/update-articles-metadata.ts

@@ -8,7 +8,7 @@ import { type Operation, logResults } from "./utils";
 /**
  * This script will go throw articles with missed metadata and try to extract the metadata from the article url and update the article table in the database.
  */
-const MAX_ARTICLES_PER_EXECUTION = 5;
+const MAX_ARTICLES_PER_EXECUTION = 10;
 
 type Article = {
   id: string;

src/scripts/utils.ts

@@ -9,3 +9,6 @@ export const logResults = (operations: Operation[]) => {
     console.log(`${operation.isSuccess ? "✅" : "❌"} : ${operation.label}`);
   });
 };
+
+export const sleep = (ms: number) =>
+  new Promise((resolve) => setTimeout(resolve, ms));