Merge pull request #203 from ubiquity-os-marketplace/development

Merge development into main

.github/workflows/compute.yml

@@ -17,6 +17,8 @@ on:
         description: "Ref"
       signature:
         description: "Payload signature from the kernel"
+      command:
+        description: "Command"
 
 jobs:
   compute:

.github/workflows/formatting-checks.yml

@@ -13,14 +13,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: "20.10.0"
+      - uses: oven-sh/setup-bun@v2
 
       - name: Run formatting checks
         run: |
-          yarn install
-          yarn eslint --fix-dry-run --ignore-pattern dist/
-          yarn format:cspell
-          yarn prettier --check .
+          bun install
+          bun eslint --fix-dry-run --ignore-pattern dist/
+          bun format:cspell
+          bun prettier --check .

.gitignore

@@ -11,4 +11,5 @@ static/dist
 junit.xml
 coverage
 test-dashboard.md
-*.private.env.json
+*.private.env.json
+output.html

eslint.config.mjs

@@ -9,7 +9,7 @@ export default tsEslint.config({
     "@typescript-eslint": tsEslint.plugin,
     "check-file": checkFile,
   },
-  ignores: [".github/knip.ts", "dist/", "tests/__mocks__/**", "coverage/**"],
+  ignores: [".github/knip.ts", "dist/", "tests/__mocks__/**", "coverage/**", "dist/**"],
   extends: [eslint.configs.recommended, ...tsEslint.configs.recommended, sonarjs.configs.recommended],
   languageOptions: {
     parser: tsEslint.parser,

knip.ts

@@ -5,7 +5,7 @@ const config: KnipConfig = {
   project: ["src/**/*.ts"],
   ignore: ["src/data-collection/examples/*.ts", "src/configuration/common-config-type.ts", "dist/**"],
   ignoreExportsUsedInFile: true,
-  ignoreDependencies: ["ts-node", "msw", "@mswjs/data", "@octokit/plugin-paginate-graphql"],
+  ignoreDependencies: ["ts-node", "msw", "@mswjs/data"],
   jest: {
     config: ["jest.config.ts"],
     entry: ["src/**/*.test.ts"],

manifest.json

@@ -28,6 +28,11 @@
             "default": true,
             "type": "boolean"
           },
+          "limitRewards": {
+            "default": true,
+            "description": "Should the rewards of non-assignees be limited to the task reward?",
+            "type": "boolean"
+          },
           "contentEvaluator": {
             "default": null,
             "anyOf": [

package.json

@@ -10,7 +10,6 @@
   "type": "module",
   "scripts": {
     "test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest --setupFiles dotenv/config --coverage",
-    "start": "tsx src/index.ts",
     "format": "run-s format:lint format:prettier format:cspell",
     "format:lint": "eslint --fix .",
     "format:prettier": "prettier --write .",
@@ -31,22 +30,19 @@
     "@actions/github": "^6.0.0",
     "@octokit/graphql-schema": "^15.25.0",
     "@octokit/rest": "^21.0.2",
-    "@sinclair/typebox": "^0.33.21",
+    "@sinclair/typebox": "0.34.3",
     "@supabase/supabase-js": "2.42.0",
     "@ubiquity-dao/rpc-handler": "1.3.0",
     "@ubiquity-os/permit-generation": "^2.0.6",
-    "@ubiquity-os/plugin-sdk": "^1.0.11",
+    "@ubiquity-os/plugin-sdk": "^1.1.0",
     "@ubiquity-os/ubiquity-os-logger": "^1.3.2",
-    "@octokit/plugin-paginate-graphql": "^5.2.4",
     "decimal.js": "10.4.3",
     "ethers": "^5.7.2",
     "js-tiktoken": "1.0.15",
     "jsdom": "24.0.0",
     "markdown-it": "14.1.0",
     "openai": "4.56.0",
-    "ts-jest": "^29.2.5",
-    "tsx": "4.7.1",
-    "yaml": "^2.6.0"
+    "yaml": "^2.6.1"
   },
   "devDependencies": {
     "@babel/core": "7.23.9",
@@ -83,6 +79,7 @@
     "sqlite": "^5.1.1",
     "sqlite3": "^5.1.7",
     "ts-node": "10.9.2",
+    "ts-jest": "^29.2.5",
     "typescript": "^5.6.3",
     "typescript-eslint": "^8.13.0"
   },

src/configuration/formatting-evaluator-config.ts

@@ -15,6 +15,7 @@ export const commentType = stringLiteralUnion(
 );
 
 export const wordRegex = /\b\w+\b/;
+export const urlRegex = /https?:\/\/\S+/g;
 
 const htmlEntity = Type.Object({
   score: Type.Number(),

src/index.ts

@@ -4,7 +4,7 @@ import { run } from "./run";
 import envConfigSchema, { EnvConfig } from "./types/env-type";
 import { PluginSettings, pluginSettingsSchema, SupportedEvents } from "./types/plugin-input";
 
-export default createActionsPlugin<PluginSettings, EnvConfig, SupportedEvents>(
+export default createActionsPlugin<PluginSettings, EnvConfig, null, SupportedEvents>(
   (context) => {
     return run(context);
   },

src/parser/content-evaluator-module.ts

@@ -109,7 +109,7 @@ export class ContentEvaluatorModule extends BaseModule {
     );
 
     if (Object.keys(relevancesByAi).length !== commentsToEvaluate.length + prCommentsToEvaluate.length) {
-      throw this.context.logger.fatal("Relevance / Comment length mismatch!", {
+      throw this.context.logger.error("Relevance / Comment length mismatch!", {
         relevancesByAi,
         commentsToEvaluate,
         prCommentsToEvaluate,

src/parser/data-purge-module.ts

@@ -33,6 +33,8 @@ export class DataPurgeModule extends BaseModule {
           .replace(/^\/.+/g, "")
           // Remove HTML comments
           .replace(/<!--[\s\S]*?-->/g, "")
+          // Remove the footnotes
+          .replace(/^###### .*?\[\^\d+\^][\s\S]*$/gm, "")
           // Keep only one new line needed by markdown-it package to convert to html
           .replace(/\n\s*\n/g, "\n")
           .trim();

src/parser/formatting-evaluator-module.ts

@@ -6,6 +6,7 @@ import { commentEnum, CommentType } from "../configuration/comment-types";
 import {
   FormattingEvaluatorConfiguration,
   formattingEvaluatorConfigurationType,
+  urlRegex,
   wordRegex,
 } from "../configuration/formatting-evaluator-config";
 import { IssueActivity } from "../issue-activity";
@@ -151,6 +152,7 @@ export class FormattingEvaluatorModule extends BaseModule {
   _classifyTagsWithWordCount(htmlElement: HTMLElement, commentType: GithubCommentScore["type"]) {
     const formatting: Record<string, { score: number; elementCount: number }> = {};
     const elements = htmlElement.getElementsByTagName("*");
+    const urlSet = new Set<string>();
 
     for (const element of elements) {
       const tagName = element.tagName.toLowerCase();
@@ -168,9 +170,26 @@ export class FormattingEvaluatorModule extends BaseModule {
         element.remove();
         continue;
       }
-      this._updateTagCount(formatting, tagName, score);
+      if (tagName === "a") {
+        const url = element.getAttribute("href");
+        if (url) {
+          urlSet.add(url.split(/[#?]/)[0]);
+        }
+      } else {
+        const bodyContent = element.textContent;
+        const matches = bodyContent?.match(urlRegex);
+        matches?.map((url) => url.split(/[#?]/)[0]).forEach((url) => urlSet.add(url));
+        this._updateTagCount(formatting, tagName, score);
+      }
     }
-    const words = this._countWordsFromRegex(htmlElement.textContent ?? "", this._multipliers[commentType]?.wordValue);
+    urlSet.forEach(() => {
+      this._updateTagCount(formatting, "a", this._multipliers[commentType].html["a"].score ?? 0);
+    });
+    const words = this._countWordsFromRegex(
+      htmlElement.textContent?.replace(urlRegex, "") ?? "",
+      this._multipliers[commentType]?.wordValue
+    );
+
     return { formatting, words };
   }
 

src/parser/processor.ts

@@ -34,14 +34,28 @@ export class Processor {
     return this;
   }
 
+  _getRewardsLimit() {
+    let taskReward = Infinity;
+    if (!this._configuration.limitRewards) {
+      return taskReward;
+    }
+    for (const item of Object.keys(this._result)) {
+      if (this._result[item].task) {
+        taskReward = this._result[item].task.reward * this._result[item].task.multiplier;
+        return taskReward;
+      }
+    }
+    return taskReward;
+  }
+
   async run(data: Readonly<IssueActivity>) {
     for (const transformer of this._transformers) {
       if (transformer.enabled) {
         this._result = await transformer.transform(data, this._result);
       }
       // Aggregate total result
       for (const item of Object.keys(this._result)) {
-        this._result[item].total = this._sumRewards(this._result[item]);
+        this._result[item].total = this._sumRewards(this._result[item], this._getRewardsLimit());
       }
     }
     return this._result;
@@ -51,21 +65,23 @@ export class Processor {
     const { file } = this._configuration;
     const result = JSON.stringify(this._result, typeReplacer, 2);
     if (!file) {
-      this._context.logger.debug(result);
+      this._context.logger.verbose(result);
     } else {
       fs.writeFileSync(file, result);
     }
     return result;
   }
 
-  _sumRewards(obj: Record<string, unknown>) {
+  _sumRewards(obj: Record<string, unknown>, taskRewardLimit = Infinity) {
     let totalReward = new Decimal(0);
 
     for (const [key, value] of Object.entries(obj)) {
       if (key === "reward" && typeof value === "number") {
-        totalReward = totalReward.add(value);
+        totalReward = totalReward.add(Math.min(value, taskRewardLimit));
       } else if (typeof value === "object") {
-        totalReward = totalReward.add(this._sumRewards(value as Record<string, unknown>));
+        totalReward = totalReward.add(
+          Math.min(this._sumRewards(value as Record<string, unknown>, taskRewardLimit), taskRewardLimit)
+        );
       }
     }
 

src/types/plugin-input.ts

@@ -33,6 +33,10 @@ export const pluginSettingsSchema = T.Object(
          * If set to false, the plugin runs even if the price label is missing, and will evaluate comments.
          */
         requirePriceLabel: T.Boolean({ default: true }),
+        limitRewards: T.Boolean({
+          default: true,
+          description: "Should the rewards of non-assignees be limited to the task reward?",
+        }),
         contentEvaluator: T.Union([contentEvaluatorConfigurationType, T.Null()], { default: null }),
         userExtractor: T.Union([userExtractorConfigurationType, T.Null()], { default: null }),
         dataPurge: T.Union([dataPurgeConfigurationType, T.Null()], { default: null }),
@@ -50,4 +54,4 @@ export const pluginSettingsSchema = T.Object(
 export type PluginSettings = StaticDecode<typeof pluginSettingsSchema>;
 
 export type SupportedEvents = "issues.closed";
-export type ContextPlugin = Context<PluginSettings, EnvConfig, SupportedEvents>;
+export type ContextPlugin = Context<PluginSettings, EnvConfig, null, SupportedEvents>;

src/web/api/index.ts

@@ -34,7 +34,11 @@ const baseApp = createPlugin<PluginSettings, EnvConfig, SupportedEvents>(
 
 const app = {
   fetch: async (request: Request, env: object, ctx: ExecutionContext) => {
-    if (request.method === "POST" && new URL(request.url).pathname === "/") {
+    if (
+      request.method === "POST" &&
+      new URL(request.url).pathname === "/" &&
+      request.headers.get("origin") === "http://localhost:4000"
+    ) {
       try {
         const originalBody = await request.json();
         const modifiedBody = await getPayload(
@@ -108,7 +112,7 @@ app.get("/openai/*", () => {
   return Response.json("OpenAI GET");
 });
 
-const port = 3000;
+const port = 4000;
 
 export default {
   fetch: app.fetch,