Locally Develop with CF Workers (#89)

* locally develop with CF workers

* fix env types

* make rating analyzer have a new format to increase modularity

* removed console log

* readme updates

* Toggle mocked datastore/integrations based on deployment state (#92)

* Toggle mocked datastore/integrations based on deployment state

* tiny changes

---------

Co-authored-by: mfish33 <maxmfishernj@gmail.com>

* still not working

* fix broken package lock npm/cli#4828

---------

Co-authored-by: Chris Lawson <Chris2fourlaw@users.noreply.github.com>
Co-authored-by: AddisonTustin <addison@atustin.dev>

.github/workflows/deploy.yml

@@ -58,3 +58,11 @@ jobs:
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
         run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npm run deploy:beta
+
+      # We will deploy to dev so that cloudflare preview urls will point to the backend
+      # This will be a problem with multiple PRs, but hey we at least tried :)
+      - name: Deploy Dev
+        if: github.ref != 'refs/heads/beta' && github.ref != 'refs/heads/master'
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npm run deploy:dev

README.md

@@ -9,31 +9,27 @@ Lerna monorepo to contain all polyratings related code
 - [cron](./packages/cron/) - nightly job that syncs data between environments as well as backs up the professor list to a separate git [repo](TODO://PROVIDE_URL)
 - [eslint-config](./packages/eslint-config/) - shared eslint config that is enforced in all other packages
 
-## Getting Ready for development
-
-If you are not interested in developing the backend or cron packages you can skip to [setup](#setup)
-
-In order to set up for development you have one of two options:
-
-1. Following the instructions [here](./docs/deployment.md) to deploy polyratings to your personal cloudflare account. This will then allow you to do test deployments of your changes in an isolated environment.
-2. Reaching out to `user@domain.com` to receive credentials to the cloudflare account in order to be able to publish to the official dev environment. This option should only be taken if you are interested in working on polyratings in the long term and have demonstrated an interest through multiple previous code contributions.
-
 ## Setup
 
-Since this repository is organized using lerna setup is a little different than standard js projects.
-
 Install top level JS dependencies
 
 ```bash
 npm install
 ```
 
-This will install lerna and all of the dependencies for all of the sub packages and sym link dependent packages. Finally run:
+This will install lerna and all of the dependencies for all of the sub packages and sym link dependent packages. 
+
+
+Then run:
 
 ```bash
 npm run build
 ```
 
-This will build all of the projects and put shared files where they are supposed to be.
+Finally:
+```bash
+npm run start:local
+```
+> Note you can run `npm run start:dev` if you have access to the cloudflare account to use data stored in the dev KV 
 
-You can now start developing in your desired package. Follow the README in the specific package for specific information.
+This will start the local hot reload server for the frontend and backend. Follow the individual package READMEs for specific package information.

package.json

@@ -7,7 +7,8 @@
   "scripts": {
     "bootstrap": "npm install",
     "build": "lerna run build",
-    "start:dev": "lerna run start:dev",
+    "start:dev": "lerna run start:dev --stream",
+    "start:local": "lerna run start:local --stream",
     "test": "lerna run test --stream",
     "lint": "lerna run lint --stream",
     "fix": "lerna run fix --stream",

packages/backend/generateBackendTypes.js

@@ -11,11 +11,11 @@ function objToString(obj, ndeep = 1) {
       case "string": return `"${obj}"`;
       case "function": return obj.name || obj.toString();
       case "object": {
-          const indent = Array(ndeep * 4 + 1).join(" "); 
+          const indent = Array(ndeep * 4 + 1).join(" ");
           const isArray = Array.isArray(obj);
           const openBrace = "{["[Number(isArray)]
           const closeBrace = "}]"[Number(isArray)]
-          return `${openBrace + 
+          return `${openBrace +
           Object.keys(obj).map((key)=> `\n${indent}${key}: ${objToString(obj[key], ndeep +1)},`).join("")}\n${indent.slice(0,-4)}${closeBrace}`;
       }
       default: return obj.toString();
@@ -26,8 +26,8 @@ const workerToml = fs.readFileSync(path.resolve(__dirname, "./wrangler.toml"), "
 const parsedToml = toml.parse(workerToml);
 
 const nameSpaceDefinitions = Object.entries(parsedToml.env)
-.map(([envKey, envData]) => 
-    envData.kv_namespaces.reduce((acc,curr) => {
+.map(([envKey, envData]) =>
+    envData.kv_namespaces.reduce((acc, curr) => {
         acc[curr.binding] = {[envKey]: curr.id}
         return acc
     }, {})
@@ -44,9 +44,9 @@ const nameSpaceDefinitions = Object.entries(parsedToml.env)
 }, {})
 
 // eslint-disable-next-line prettier/prettier
-const tomlTypeOutput = 
+const tomlTypeOutput =
 `// Please do not modify this file it is generated by \`generateBackendTypes.ts\`
-/* eslint-disable */ 
+/* eslint-disable */
 
 export interface PolyratingsAPIEnv {
     url: string;
@@ -56,6 +56,9 @@ ${Object.entries(parsedToml.env).map(([envKey, envData]) =>
     url: "https://${envData.route.slice(0, -2)}",
 };`,
 ).join("\n")}
+export const LOCAL_ENV: PolyratingsAPIEnv = {
+    url: "http://${parsedToml.dev.ip}:${parsedToml.dev.port}",
+};
 
 export const cloudflareNamespaceInformation = ${objToString(nameSpaceDefinitions)} as const;
 

packages/backend/package.json

@@ -4,6 +4,8 @@
     "description": "Cloudflare Workers project for the Polyratings site backend",
     "private": true,
     "scripts": {
+        "start:dev": "wrangler dev --env dev --remote",
+        "start:local": "wrangler dev",
         "build:fast": "esbuild --format=esm --define:this=self --bundle src/index.ts --outdir=dist --metafile=stats/metadata.json",
         "build": "node generateBackendTypes.js && tsc && npm run build:fast",
         "stats": "esbuild-visualizer --metadata ./stats/metadata.json --filename ./stats/stats.html",

packages/backend/src/dao/discord-notification-dao.ts

@@ -3,12 +3,16 @@ interface WebhookBody {
     username?: string; // overrides webhook's default username
 }
 
-type DiscordUsername = "Pending Professor Notification" | "Received A Report";
+type NotificationEvent = "Pending Professor Notification" | "Received A Report";
 
-export class DiscordNotificationDAO {
+export type NotificationDAO = {
+    notify(username: NotificationEvent, content: string): Promise<void>;
+};
+
+export class DiscordNotificationDAO implements NotificationDAO {
     constructor(private webhookURL: string) {}
 
-    public async sendWebhook(username: DiscordUsername, content: string) {
+    public async notify(username: NotificationEvent, content: string) {
         const webhookBody: WebhookBody = {
             content,
             username,
@@ -24,3 +28,8 @@ export class DiscordNotificationDAO {
         });
     }
 }
+
+export class NoOpNotificationDao implements NotificationDAO {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars, class-methods-use-this
+    public async notify(_username: NotificationEvent, _content: string): Promise<void> {}
+}

packages/backend/src/dao/perspective-dao.ts → packages/backend/src/dao/rating-analyzer-dao.ts

@@ -2,10 +2,15 @@ import { PendingRating, PerspectiveAttributeScore } from "@backend/types/schema"
 
 const ANALYZE_COMMENT_URL = "https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze";
 
-export class PerspectiveDAO {
+export type AnalyzedRating = Record<string, number>;
+export type RatingAnalyzer = {
+    analyzeRaring(rating: PendingRating): Promise<AnalyzedRating>;
+};
+
+export class PerspectiveDAO implements RatingAnalyzer {
     constructor(private readonly apiKey: string) {}
 
-    async analyzeRaring(rating: PendingRating): Promise<AnalyzeCommentResponse["attributeScores"]> {
+    async analyzeRaring(rating: PendingRating): Promise<AnalyzedRating> {
         // TODO: Perhaps we should define a default request?
         const requestBody: AnalyzeCommentRequest = {
             comment: {
@@ -34,7 +39,23 @@ export class PerspectiveDAO {
         }
 
         const response = (await httpResponse.json()) as AnalyzeCommentResponse;
-        return response.attributeScores;
+        return Object.fromEntries(
+            Object.entries(response.attributeScores).map(
+                ([
+                    key,
+                    {
+                        summaryScore: { value },
+                    },
+                ]) => [key, value],
+            ),
+        );
+    }
+}
+
+export class PassThroughRatingAnalyzer implements RatingAnalyzer {
+    // eslint-disable-next-line class-methods-use-this, @typescript-eslint/no-unused-vars
+    async analyzeRaring(rating: PendingRating): Promise<AnalyzedRating> {
+        return {};
     }
 }
 

packages/backend/src/env.ts

@@ -1,17 +1,39 @@
 import { KVDAO } from "@backend/dao/kv-dao";
-import { PerspectiveDAO } from "@backend/dao/perspective-dao";
+import {
+    PassThroughRatingAnalyzer,
+    PerspectiveDAO,
+    RatingAnalyzer,
+} from "@backend/dao/rating-analyzer-dao";
 import { AuthStrategy } from "@backend/dao/auth-strategy";
-import { DiscordNotificationDAO } from "@backend/dao/discord-notification-dao";
+import {
+    DiscordNotificationDAO,
+    NoOpNotificationDao,
+    NotificationDAO,
+} from "@backend/dao/discord-notification-dao";
+import { z } from "zod";
 import { KvWrapper } from "./dao/kv-wrapper";
 
+export function getCloudflareEnv(rawEnv: Record<string, unknown>): CloudflareEnv {
+    if (!rawEnv?.IS_DEPLOYED) {
+        // Set empty default values that will cause dep injection to replace with no-ops
+        return cloudflareEnvParser.parse({
+            JWT_SIGNING_KEY: "TEST_SIGNING_SECRET",
+            PERSPECTIVE_API_KEY: "",
+            DISCORD_WEBHOOK_URL: "",
+            ...rawEnv,
+        });
+    }
+    return cloudflareEnvParser.parse(rawEnv);
+}
+
 export class Env {
     kvDao: KVDAO;
 
-    perspectiveDao: PerspectiveDAO;
+    ratingAnalyzer: RatingAnalyzer;
 
     authStrategy: AuthStrategy;
 
-    notificationDAO: DiscordNotificationDAO;
+    notificationDAO: NotificationDAO;
 
     constructor(env: CloudflareEnv) {
         this.kvDao = new KVDAO(
@@ -21,19 +43,38 @@ export class Env {
             new KvWrapper(env.POLYRATINGS_TEACHER_APPROVAL_QUEUE),
             new KvWrapper(env.POLYRATINGS_REPORTS),
         );
-        this.perspectiveDao = new PerspectiveDAO(env.PERSPECTIVE_API_KEY);
+        if (!env.IS_DEPLOYED && !env.PERSPECTIVE_API_KEY) {
+            // eslint-disable-next-line no-console
+            console.warn("Not using Perspective API. Please set PERSPECTIVE_API_KEY to enable");
+            this.ratingAnalyzer = new PassThroughRatingAnalyzer();
+        } else {
+            this.ratingAnalyzer = new PerspectiveDAO(env.PERSPECTIVE_API_KEY);
+        }
+        if (!env.IS_DEPLOYED && !env.DISCORD_WEBHOOK_URL) {
+            // eslint-disable-next-line no-console
+            console.warn("Not using Discord Notifier. Please set DISCORD_WEBHOOK_URL to enable");
+            this.notificationDAO = new NoOpNotificationDao();
+        } else {
+            this.notificationDAO = new DiscordNotificationDAO(env.DISCORD_WEBHOOK_URL);
+        }
+
         this.authStrategy = new AuthStrategy(env.JWT_SIGNING_KEY);
-        this.notificationDAO = new DiscordNotificationDAO(env.DISCORD_WEBHOOK_URL);
     }
 }
 
-export interface CloudflareEnv {
-    POLYRATINGS_TEACHERS: KVNamespace;
-    PROCESSING_QUEUE: KVNamespace;
-    POLYRATINGS_USERS: KVNamespace;
-    POLYRATINGS_TEACHER_APPROVAL_QUEUE: KVNamespace;
-    POLYRATINGS_REPORTS: KVNamespace;
-    JWT_SIGNING_KEY: string;
-    PERSPECTIVE_API_KEY: string;
-    DISCORD_WEBHOOK_URL: string;
-}
+export type CloudflareEnv = z.infer<typeof cloudflareEnvParser>;
+
+// Can not really verify that the env is actually of type `KVNamespace`
+// The least we can do is see if it is a non null object
+const kvNamespaceParser = z.custom<KVNamespace>((n) => n && n !== null && typeof n === "object");
+const cloudflareEnvParser = z.object({
+    POLYRATINGS_TEACHERS: kvNamespaceParser,
+    PROCESSING_QUEUE: kvNamespaceParser,
+    POLYRATINGS_USERS: kvNamespaceParser,
+    POLYRATINGS_TEACHER_APPROVAL_QUEUE: kvNamespaceParser,
+    POLYRATINGS_REPORTS: kvNamespaceParser,
+    JWT_SIGNING_KEY: z.string(),
+    PERSPECTIVE_API_KEY: z.string(),
+    DISCORD_WEBHOOK_URL: z.string(),
+    IS_DEPLOYED: z.boolean(),
+});

packages/backend/src/generated/tomlGenerated.ts

@@ -1,5 +1,5 @@
 // Please do not modify this file it is generated by `generateBackendTypes.ts`
-/* eslint-disable */ 
+/* eslint-disable */
 
 export interface PolyratingsAPIEnv {
     url: string;
@@ -13,6 +13,9 @@ export const BETA_ENV: PolyratingsAPIEnv = {
 export const PROD_ENV: PolyratingsAPIEnv = {
     url: "https://api-prod.polyratings.org",
 };
+export const LOCAL_ENV: PolyratingsAPIEnv = {
+    url: "http://localhost:3001",
+};
 
 export const cloudflareNamespaceInformation = {
     POLYRATINGS_TEACHERS: {