Merge pull request #60 from Giveth/add-gitcoin-projects

Add gitcoin projects

.env.template

@@ -26,4 +26,6 @@ SQUID_NETWORK=optimism-mainnet
 
 #APIs
 GIVETH_API_URL=https://somewhere.giveth.io/XXXXX
-RPGF3_API_URL=https://somewhere.RPGF3.io/XXXXX
+RPGF3_API_URL=https://somewhere.RPGF3.io/XXXXX
+GITCOIN_API_URL=https://somewhere.gitcoin.co/XXXXX
+IPFS_GATEWAY=https://ipfs.io/ipfs

src/features/import-projects/gitcoin/constants.ts

@@ -0,0 +1,16 @@
+export const GITCOIN_API_URL =
+  process.env.GITCOIN_API_URL ||
+  "https://grants-stack-indexer-v2.gitcoin.co/graphql";
+
+export const IPFS_GATEWAY = process.env.IPFS_GATEWAY || "https://ipfs.io/ipfs";
+
+export const GITCOIN_API_LIMIT = 100;
+
+export const gitcoinSourceConfig: SourceConfig = {
+  source: "gitcoin",
+  idField: "id",
+  titleField: "title",
+  descriptionField: "description",
+  slugField: "slug",
+  imageField: "image",
+};

src/features/import-projects/gitcoin/helpers.ts

@@ -0,0 +1,40 @@
+import type { GitcoinProjectInfo } from "./type";
+import { updateOrCreateProject } from "../helpers";
+import { IPFS_GATEWAY, gitcoinSourceConfig } from "./constants";
+
+const generateGitcoinSlug = (project: GitcoinProjectInfo) => {
+  const application = project.applications[0];
+
+  if (
+    !application ||
+    !application?.roundId ||
+    !application?.chainId ||
+    !application?.id
+  )
+    return "";
+
+  const { chainId, roundId, id } = application;
+  return `#/round/${chainId}/${roundId}/${id}`;
+};
+
+const convertIpfsHashToHttps = (hash: string) => {
+  return `${IPFS_GATEWAY}/${hash}`;
+};
+
+export const processProjectsBatch = async (
+  projectsBatch: GitcoinProjectInfo[]
+) => {
+  for (const project of projectsBatch) {
+    if (project.metadata?.type !== "project") continue;
+    const processedProject = {
+      id: project.id,
+      title: project.name || project.metadata?.title,
+      description: project.metadata?.description,
+      slug: generateGitcoinSlug(project),
+      image: project.metadata?.bannerImg
+        ? convertIpfsHashToHttps(project.metadata?.bannerImg)
+        : "",
+    };
+    await updateOrCreateProject(processedProject, gitcoinSourceConfig);
+  }
+};

src/features/import-projects/gitcoin/index.ts

@@ -0,0 +1,23 @@
+import { GITCOIN_API_LIMIT } from "./constants";
+import { processProjectsBatch } from "./helpers";
+import { fetchGitcoinProjectsBatch } from "./service";
+
+export const fetchAndProcessGitcoinProjects = async () => {
+  try {
+    let hasMoreProjects = true;
+    let skip = 0;
+    const limit = GITCOIN_API_LIMIT;
+
+    while (hasMoreProjects) {
+      const projectsBatch = await fetchGitcoinProjectsBatch(limit, skip);
+      if (projectsBatch.length > 0) {
+        await processProjectsBatch(projectsBatch);
+        skip += limit;
+      } else {
+        hasMoreProjects = false;
+      }
+    }
+  } catch (error: any) {
+    console.log("error on fetchAndProcessGitcoinProjects", error.message);
+  }
+};

src/features/import-projects/gitcoin/service.ts

@@ -0,0 +1,47 @@
+import { graphQLRequest } from "../../../helpers/request";
+import { GITCOIN_API_URL } from "./constants";
+import { GitcoinProjectInfo } from "./type";
+
+export const fetchGitcoinProjectsBatch = async (
+  first: number,
+  offset: number
+) => {
+  try {
+    const res = await graphQLRequest(
+      GITCOIN_API_URL,
+      `query fetchProjects($first: Int = 10, $offset: Int = 10) {
+        projects(
+          first: $first
+          offset: $offset
+          orderBy: ID_DESC
+          filter: {
+            tags: { contains: "allo-v2" }
+            projectType: { equalTo: CANONICAL }
+            chainId: { in: [1, 137, 10, 324, 42161, 42220, 43114, 534352, 8453] }
+            not: { tags: { contains: "program" } }
+            rounds: { every: { applicationsExist: true } }
+          }
+        ) {
+          id
+          name
+          applications {
+            roundId
+            chainId
+            id
+          }
+          metadata
+        }
+      }
+      `,
+      {
+        first,
+        offset,
+      }
+    );
+
+    return res.data.projects;
+  } catch (error: any) {
+    console.log("error on fetchGitcoinProjectsBatch", error.message);
+    return [];
+  }
+};

src/features/import-projects/gitcoin/type.ts

@@ -0,0 +1,26 @@
+export type GitcoinProjectInfo = {
+  id: string;
+  name: string;
+  applications: GitcoinApplication[];
+  metadata: GitcoinMetadata;
+};
+
+type GitcoinApplication = {
+  roundId: string;
+  chainId: number;
+  id: string;
+};
+
+type GitcoinMetadata = {
+  type: string;
+  title: string;
+  logoImg: string;
+  website: string;
+  bannerImg: string;
+  createdAt: number;
+  credentials: {};
+  description: string;
+  logoImgData: {};
+  bannerImgData: {};
+  projectTwitter: string;
+};

src/features/import-projects/giveth/constants.ts

@@ -1,6 +1,8 @@
 export const GIVETH_API_URL =
   process.env.GIVETH_API_URL || "https://mainnet.serve.giveth.io/graphql";
 
+export const GIVETH_API_LIMIT = 100;
+
 export const givethSourceConfig: SourceConfig = {
   source: "giveth",
   idField: "id",

src/features/import-projects/giveth/index.ts

@@ -1,11 +1,12 @@
+import { GIVETH_API_LIMIT } from "./constants";
 import { processProjectsBatch } from "./helpers";
 import { fetchGivethProjectsBatch } from "./service";
 
 export const fetchAndProcessGivethProjects = async () => {
   try {
     let hasMoreProjects = true;
     let skip = 0;
-    const limit = 10;
+    const limit = GIVETH_API_LIMIT;
 
     while (hasMoreProjects) {
       const projectsBatch = await fetchGivethProjectsBatch(limit, skip);

src/features/import-projects/giveth/service.ts

@@ -34,22 +34,3 @@ export const fetchGivethProjectsBatch = async (limit: number, skip: number) => {
     return [];
   }
 };
-
-export const fetchGivethProjects = async () => {
-  let allProjects: GivethProjectInfo[] = [];
-  let hasMoreProjects = true;
-  let skip = 0;
-  const limit = 10;
-
-  while (hasMoreProjects) {
-    const projectsBatch = await fetchGivethProjectsBatch(limit, skip);
-    if (projectsBatch.length > 0) {
-      allProjects = allProjects.concat(projectsBatch);
-      skip += limit;
-    } else {
-      hasMoreProjects = false;
-    }
-  }
-
-  return allProjects;
-};

src/features/import-projects/helpers.ts

@@ -15,7 +15,8 @@ export const updateOrCreateProject = async (
     imageField,
   } = sourConfig;
 
-  const id = `${source}-${project[idField]}`;
+  const projectId = project[idField].toLowerCase();
+  const id = `${source}-${projectId}`;
 
   const dataSource = await getDataSource();
   if (!dataSource) {
@@ -66,7 +67,7 @@ export const updateOrCreateProject = async (
       description: project[descriptionField],
       image: project[imageField],
       slug: project[slugField],
-      projectId: project[idField],
+      projectId: projectId,
       source: source,
       totalVouches: 0,
       totalFlags: 0,

src/features/import-projects/index.ts

@@ -2,11 +2,13 @@ import cron from "node-cron";
 import { IMPORT_PROJECT_CRON_SCHEDULE } from "../../constants";
 import { fetchAndProcessGivethProjects } from "./giveth/index";
 import { fetchAndProcessRpgf3Projects } from "./rpgf";
+import { fetchAndProcessGitcoinProjects } from "./gitcoin";
 
 export const task = async () => {
   console.log("Importing Projects", new Date());
   fetchAndProcessGivethProjects();
   fetchAndProcessRpgf3Projects();
+  fetchAndProcessGitcoinProjects();
 };
 
 export const importProjects = async () => {