Merge pull request #108 from Giveth/develop

Staging to main

db/migrations/1724279465327-Data.js

@@ -0,0 +1,13 @@
+module.exports = class Data1724279465327 {
+  name = "Data1724279465327";
+
+  async up(db) {
+    await db.query(`ALTER TABLE "project" DROP COLUMN "source_status"`);
+    await db.query(`ALTER TABLE "project" ADD "rf_rounds" integer array`);
+  }
+
+  async down(db) {
+    await db.query(`ALTER TABLE "project" DROP COLUMN "rf_rounds"`);
+    await db.query(`ALTER TABLE "project" ADD "source_status" text`);
+  }
+};

schema.graphql

@@ -67,13 +67,13 @@ type Project @entity {
   url: String
   "Image of the project"
   image: String
-  "Project status at the source. e.g. Keeped, Removed, Missing in rf4"
-  sourceStatus: String
   "project data is imported from a source or not"
   imported: Boolean! @index
   lastUpdatedTimestamp: DateTime!
   attests: [ProjectAttestation!]! @derivedFrom(field: "project")
   attestedOrganisations: [OrganisationProject!]! @derivedFrom(field: "project")
+  "Rounds in which the project is included"
+  rfRounds: [Int!]
 }
 
 type OrganisationProject @entity {

src/constants.ts

@@ -37,3 +37,5 @@ export const IMPORT_PROJECT_CRON_SCHEDULE =
 export const DESCRIPTION_SUMMARY_LENGTH = Number(
   process.env.DESCRIPTION_SUMMARY_LENGTH || 300
 );
+
+export const AGORA_API_KEY = process.env.AGORA_API_KEY;

src/controllers/projectVerificationAttestation.ts

@@ -63,11 +63,15 @@ export const handleProjectAttestation = async (
 
   ctx.log.debug(`Processing project attestation with uid: ${uid}`);
 
-  const project = await getProject(
-    ctx,
-    projectVerificationAttestation.projectSource,
-    projectVerificationAttestation.projectId
-  );
+  let projectId = projectVerificationAttestation.projectId;
+  let projectSource = projectVerificationAttestation.projectSource;
+
+  if (projectVerificationAttestation.projectSource === "rf4") {
+    projectId = projectVerificationAttestation.projectId.replace("rf4-", "rf-");
+    projectSource = "rf";
+  }
+
+  const project = await getProject(ctx, projectSource, projectId);
 
   // Delete the previous attestation
   await removeDuplicateProjectAttestations(

src/features/import-projects/helpers.ts

@@ -17,7 +17,7 @@ export const updateOrCreateProject = async (
     descriptionHtmlField,
     urlField,
     imageField,
-    sourceStatusField,
+    rfRoundField,
   } = sourConfig;
 
   const projectId = project[idField].toLowerCase();
@@ -42,15 +42,15 @@ export const updateOrCreateProject = async (
   const url = project[urlField];
   const image = project[imageField];
   const descriptionHtml = descriptionHtmlField && project[descriptionHtmlField];
-  const sourceStatus = sourceStatusField && project[sourceStatusField];
+  const rfRound = rfRoundField && project[rfRoundField];
 
   if (existingProject) {
     const isUpdated =
       existingProject.title !== title ||
       existingProject.description !== description ||
       existingProject.url !== url ||
       existingProject.image !== image ||
-      existingProject.sourceStatus !== sourceStatus ||
+      (rfRound && !existingProject.rfRounds?.some((rfr) => rfr === rfRound)) ||
       existingProject.descriptionHtml !== descriptionHtml ||
       (!existingProject.descriptionSummary && description);
 
@@ -59,6 +59,8 @@ export const updateOrCreateProject = async (
     );
 
     if (isUpdated) {
+      const rfRounds = new Set(existingProject.rfRounds || []);
+      rfRound && rfRounds.add(rfRound);
       const updatedProject = new Project({
         ...existingProject,
         title,
@@ -67,8 +69,8 @@ export const updateOrCreateProject = async (
         url,
         descriptionHtml,
         descriptionSummary,
-        sourceStatus,
         lastUpdatedTimestamp: new Date(),
+        rfRounds: Array.from(rfRounds),
         imported: true,
       });
 
@@ -97,7 +99,7 @@ export const updateOrCreateProject = async (
       descriptionSummary,
       projectId,
       source,
-      sourceStatus,
+      rfRounds: rfRound ? [rfRound] : [],
       totalVouches: 0,
       totalFlags: 0,
       totalAttests: 0,

src/features/import-projects/index.ts

@@ -3,14 +3,17 @@ import { IMPORT_PROJECT_CRON_SCHEDULE } from "../../constants";
 import { fetchAndProcessGivethProjects } from "./giveth/index";
 import { fetchAndProcessRpgf3Projects } from "./rpgf";
 import { fetchAndProcessGitcoinProjects } from "./gitcoin";
-import { fetchAndProcessRf4Projects } from "./rf4";
+import { fetchRFProjectsByRound } from "./rf";
+import { fetchAndProcessRlProjects } from "./retroList";
 
 export const task = async () => {
   console.log("Importing Projects", new Date());
-  fetchAndProcessGivethProjects();
+  await fetchAndProcessGivethProjects();
+  await fetchAndProcessGitcoinProjects();
   // fetchAndProcessRpgf3Projects();
-  fetchAndProcessGitcoinProjects();
-  fetchAndProcessRf4Projects();
+  await fetchRFProjectsByRound(4);
+  // await fetchRFProjectsByRound(5); //TODO: It will fill on 20th Sep
+  await fetchAndProcessRlProjects(5);
 };
 
 export const importProjects = async () => {

src/features/import-projects/retroList/constants.ts

@@ -0,0 +1,20 @@
+import { SourceConfig } from "../types";
+import { RoundNumber } from "./type";
+
+// Define RL_API_URLS with specific round number keys
+export const RL_API_URLS: Record<RoundNumber, string> = {
+  4: process.env.RL4_API_URL || "https://round4-api-eas.retrolist.app/projects",
+  5:
+    process.env.RL5_API_URL ||
+    "https://round5-api-eas.retrolist.app/5/projects",
+};
+
+export const rlSourceConfig: SourceConfig = {
+  source: "rf",
+  idField: "id",
+  titleField: "name",
+  descriptionField: "description",
+  imageField: "bannerImageUrl",
+  urlField: "url",
+  rfRoundField: "rfRound",
+};

src/features/import-projects/retroList/helper.ts

@@ -0,0 +1,5 @@
+import { type RlProjectInfo } from "./type";
+
+export const generateRlUrl = (project: RlProjectInfo) => {
+  return `/project/${project.id}`;
+};

src/features/import-projects/retroList/index.ts

@@ -0,0 +1,22 @@
+import { updateOrCreateProject } from "../helpers";
+import { rlSourceConfig } from "./constants";
+import { generateRlUrl } from "./helper";
+import { fetchRlProjects } from "./service";
+
+export const fetchAndProcessRlProjects = async (round: number) => {
+  try {
+    const data = await fetchRlProjects(round);
+    if (!data) return;
+    for (const project of data) {
+      const processedProject = {
+        ...project,
+        url: generateRlUrl(project),
+        rfRound: round,
+      };
+
+      await updateOrCreateProject(processedProject, rlSourceConfig);
+    }
+  } catch (error: any) {
+    console.log("error on fetchAndProcessRlProjects", error.message);
+  }
+};

src/features/import-projects/retroList/service.ts

@@ -0,0 +1,39 @@
+// Assuming RL_API_URLS and RlProjectInfo are defined as mentioned
+import { RL_API_URLS } from "./constants";
+import { RlProjectInfo, RoundNumber } from "./type";
+
+export const fetchRlProjects = async (
+  round: number
+): Promise<RlProjectInfo[] | null> => {
+  console.log(`Fetching RL projects for round ${round}`);
+  try {
+    // Check if the round exists in RL_API_URLS
+    if (!(round in RL_API_URLS)) {
+      console.log(`Invalid round number: ${round}`);
+      return null;
+    }
+
+    // Use type assertion to narrow down the type of 'round' to RoundNumber
+    const url = RL_API_URLS[round as RoundNumber];
+    const res = await fetch(url, {
+      method: "GET",
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+
+    // Check if the response is ok (status code in the range 200-299)
+    if (!res.ok) {
+      console.log(`Failed to fetch data: ${res.status} ${res.statusText}`);
+      return null;
+    }
+
+    const data: RlProjectInfo[] = await res.json();
+    console.log(`Fetched ${data.length} projects for round ${round}`);
+
+    return data;
+  } catch (error: any) {
+    console.log("Error on fetching fetchRlProjects", error.message);
+    return null;
+  }
+};

src/features/import-projects/rf4/type.ts → src/features/import-projects/retroList/type.ts

@@ -1,4 +1,6 @@
-export type Rf4ProjectInfo = {
+export type RoundNumber = 4 | 5;
+
+export type RlProjectInfo = {
   id: string;
   name: string;
   displayName: string;

src/features/import-projects/rf/constants.ts

@@ -0,0 +1,14 @@
+import { SourceConfig } from "../types";
+
+export const RF_API_URL =
+  process.env.RF4_API_URL || "https://vote.optimism.io/api/v1";
+
+export const rfSourceConfig: SourceConfig = {
+  source: "rf",
+  idField: "id",
+  titleField: "name",
+  descriptionField: "description",
+  imageField: "projectCoverImageUrl",
+  urlField: "url",
+  rfRoundField: "rfRound",
+};

src/features/import-projects/rf/helpers.ts

@@ -0,0 +1,26 @@
+import { updateOrCreateProject } from "../helpers";
+import { rfSourceConfig } from "./constants";
+import { RfProjectInfo } from "./type";
+
+export const generateRfUrl = (project: RfProjectInfo) => {
+  return `/project/${project.id}`;
+};
+
+const processProject = (project: RfProjectInfo, round: number) => {
+  const projectData = {
+    ...project,
+    url: generateRfUrl(project),
+    rfRound: round,
+  };
+  return projectData;
+};
+
+export const saveBatchProjects = async (
+  projects: RfProjectInfo[],
+  round: number
+) => {
+  for (const _project of projects) {
+    const project = processProject(_project, round);
+    await updateOrCreateProject(project, rfSourceConfig);
+  }
+};

src/features/import-projects/rf/index.ts

@@ -0,0 +1,55 @@
+import { AGORA_API_KEY } from "../../../constants";
+import { RF_API_URL } from "./constants";
+import { saveBatchProjects } from "./helpers";
+import { RfApiResponse, RfProjectInfo } from "./type";
+
+export const fetchRFProjectsByRound = async (round: number) => {
+  let offset = 0;
+  const limit = 10;
+  let hasNext = true;
+
+  console.log(
+    `[${new Date().toISOString()}] - Fetching projects for round: ${round}`
+  );
+
+  if (!AGORA_API_KEY) {
+    console.error("AGORA_API_KEY is not set");
+    return;
+  }
+
+  try {
+    while (hasNext) {
+      const response = await fetch(
+        `${RF_API_URL}/retrofunding/rounds/${round}/projects?limit=${limit}&offset=${offset}`,
+        {
+          method: "GET",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${AGORA_API_KEY}`,
+          },
+        }
+      );
+      console.log(
+        `[${new Date().toISOString()}] - Fetching projects for round: ${round} at offset: ${offset} - ${response.status} - ${response.ok}`
+      );
+
+      if (!response.ok) {
+        throw new Error(
+          `HTTP error! status: ${response.status} for round: ${round} at offset: ${offset}`
+        );
+      }
+
+      const res: RfApiResponse = await response.json();
+
+      await saveBatchProjects(res.data, round);
+
+      hasNext = res.meta.has_next;
+      offset = res.meta.next_offset;
+    }
+  } catch (error) {
+    console.error(
+      `Error fetching projects for round: ${round} at offset: ${offset}`,
+      error
+    );
+  }
+};

src/features/import-projects/rf/type.ts

@@ -0,0 +1,22 @@
+export interface RfProjectInfo {
+  id: string;
+  category: string;
+  organization: string | null;
+  name: string;
+  description: string;
+  profileAvatarUrl: string;
+  projectCoverImageUrl: string;
+  team: string[];
+  github: string[];
+  packages: string[];
+  links: string[];
+}
+
+export interface RfApiResponse {
+  meta: {
+    has_next: boolean;
+    total_returned: number;
+    next_offset: number;
+  };
+  data: RfProjectInfo[];
+}