Merge pull request #83 from Giveth/develop

Support organisation start block

.gitignore

@@ -12,5 +12,5 @@
 # Backup file
 .env.backup
 
-# Exclude add-organisation updates on pushes
-add-organisation.js
+# Ogranisation config
+org-config.jsonc

README.md

@@ -1,97 +1,20 @@
-[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/subsquid/squid-evm-template)
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/Giveth/DeVouch-BE)
 
-# Minimal EVM squid
+# Introduction
 
-This is a starter template of a squid indexer for EVM networks (Ethereum, Polygon, BSC, etc.). See [Squid SDK docs](https://docs.subsquid.io/) for a complete reference.
+Devouch is a decentralized application that allows users to attest to projects credibility by their vouches or flags. The backend is built on Subsquid and uses a Postgres database.
 
-To extract EVM logs and transactions by a topic or a contract address, use [`.addLog()`](https://docs.subsquid.io/evm-indexing/configuration/evm-logs/), [`.addTransaction()`](https://docs.subsquid.io/evm-indexing/configuration/transactions/), [`.addTrace()`](https://docs.subsquid.io/evm-indexing/configuration/traces/) or [`.addStateDiff()`](https://docs.subsquid.io/evm-indexing/configuration/state-diffs/) methods of the `EvmBatchProcessor` instance defined in `src/processor.ts`. Select data fields with [`.setFields()`](https://docs.subsquid.io/evm-indexing/configuration/data-selection/).
+# Getting Started
 
-The requested data is transformed in batches by a single handler provided to the `processor.run()` method. 
 
-For a full list of supported networks and config options,
-check the [`EvmBatchProcessor` overview](https://docs.subsquid.io/evm-indexing/evm-processor/) and the [setup section](https://docs.subsquid.io/evm-indexing/configuration/).
-
-For a step-by-step migration guide from TheGraph, see [the dedicated docs page](https://docs.subsquid.io/migrate/migrate-subgraph/).
-
-Dependencies: Node.js v16 or newer, Git, Docker.
-
-## Quickstart
-
-```bash
-# 0. Install @subsquid/cli a.k.a. the sqd command globally
-npm i -g @subsquid/cli
-
-# 1. Retrieve the template
-sqd init my_squid_name -t evm
-cd my_squid_name
-
-# 2. Install dependencies
-npm ci
-
-# 3. Start a Postgres database container and detach
-sqd up
-
-# 4. Build the squid
-sqd build
-
-# 5. Start both the squid processor and the GraphQL server
-sqd run .
-```
-A GraphiQL playground will be available at [localhost:4350/graphql](http://localhost:4350/graphql).
-
-You can also start squid services one by one:
-```bash
-sqd process
-sqd serve
-```
-
-## Dev flow
-
-### 1. Define database schema
-
-Start development by defining the schema of the target database via `schema.graphql`.
-Schema definition consists of regular graphql type declarations annotated with custom directives.
-Full description of `schema.graphql` dialect is available [here](https://docs.subsquid.io/store/postgres/schema-file/).
-
-### 2. Generate TypeORM classes
-
-Mapping developers use TypeORM [EntityManager](https://typeorm.io/#/working-with-entity-manager)
-to interact with target database during data processing. All necessary entity classes are
-generated by the squid framework from `schema.graphql`. This is done by running `sqd codegen`
-command.
-
-### 3. Generate database migrations
-
-All database changes are applied through migration files located at `db/migrations`.
-`squid-typeorm-migration(1)` tool provides several commands to drive the process.
+## Add new organisation
 
 ```bash
-## drop create the database
-sqd down
-sqd up
+# 0. Copy the org-config.template.json to org-config.json
+cp org-config.template.json org-config.json
 
-## replace any old schemas with a new one made from the entities
-sqd migration:generate
+# 1. Fill the config with your organisation data
+# 2. Run the script to add your organisation data to migrations
+npm run add-organisation
 ```
-See [docs on database migrations](https://docs.subsquid.io/store/postgres/db-migrations/) for more details.
-
-### 4. Import ABI contract and generate interfaces to decode events
-
-It is necessary to import the respective ABI definition to decode EVM logs. One way to generate a type-safe facade class to decode EVM logs is by placing the relevant JSON ABIs to `./abi`, then using `squid-evm-typegen(1)` via an `sqd` script:
-
-```bash
-sqd typegen
-```
-
-See more details on the [`squid-evm-typegen` doc page](https://docs.subsquid.io/evm-indexing/squid-evm-typegen).
-
-## Project conventions
-
-Squid tools assume a certain [project layout](https://docs.subsquid.io/basics/squid-structure):
-
-* All compiled js files must reside in `lib` and all TypeScript sources in `src`.
-The layout of `lib` must reflect `src`.
-* All TypeORM classes must be exported by `src/model/index.ts` (`lib/model` module).
-* Database schema must be defined in `schema.graphql`.
-* Database migrations must reside in `db/migrations` and must be plain js files.
-* `sqd(1)` and `squid-*(1)` executables consult `.env` file for environment variables.
+Then create a PR to the main branch to be reviewed and merged.

db/create-organisation-add-migration.js

@@ -8,7 +8,8 @@ exports.default = function createOrganisationAddMigration(
   schemaId,
   authorizedAttestor,
   color = null,
-  network = "eth-sepolia"
+  network = "eth-sepolia",
+  startBlock = null
 ) {
   const timestamp = new Date().getTime() + ADD_ORG_MIGRATION_OFFSET;
   const fileName = `${timestamp}-Add${organisationName}.js`;
@@ -25,12 +26,13 @@ exports.default = function createOrganisationAddMigration(
         if (SQUID_NETWORK !== "${network}") return;
         // add organisation with name "${organisationName}" and schema id "${schemaId}"
         await db.query(
-        \`INSERT INTO "organisation" ("id", "name", "issuer", "color") 
+        \`INSERT INTO "organisation" ("id", "name", "issuer", "color", "start_block") 
           VALUES (
             '${schemaId.toLocaleLowerCase()}',
             '${organisationName}',
             '${authorizedAttestor.toLocaleLowerCase()}',
-            ${color ? "'" + color.toLocaleLowerCase() + "'" : null}
+            ${color ? "'" + color.toLocaleLowerCase() + "'" : null},
+            ${startBlock}
           )\`
         );
     }

db/migrations/1718184531403-Data.js

@@ -0,0 +1,11 @@
+module.exports = class Data1718184531403 {
+    name = 'Data1718184531403'
+
+    async up(db) {
+        await db.query(`ALTER TABLE "organisation" ADD "start_block" integer`)
+    }
+
+    async down(db) {
+        await db.query(`ALTER TABLE "organisation" DROP COLUMN "start_block"`)
+    }
+}

org-config.schema.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Organization Configuration",
+  "description": "Schema for the organization configuration file.",
+  "type": "object",
+
+  "properties": {
+    "$schema": {
+      "description": "The schema version.",
+      "type": "string"
+    },
+    "name": {
+      "description": "The name of the organization.",
+      "type": "string"
+    },
+    "schemaId": {
+      "description": "The UID for the attestation schema.",
+      "type": "string",
+      "pattern": "^0x[A-Fa-f0-9]{64}$"
+    },
+    "authorizedAttestor": {
+      "description": "The address of the authorized attestor.",
+      "type": "string",
+      "pattern": "^0x[A-Fa-f0-9]{40}$"
+    },
+
+    "network": {
+      "description": "The network for the organization.",
+      "type": "string",
+      "enum": ["eth-sepolia", "optimism-mainnet"]
+    },
+
+    "color": {
+      "description": "The organisaiton color in UI.",
+      "type": "string",
+      "pattern": "^#[A-Fa-f0-9]{6}$"
+    },
+    "startBlock": {
+      "description": "The block number at which the organization was created.",
+      "type": "integer"
+    }
+  },
+  "required": ["name", "schemaId", "authorizedAttestor", "network"],
+  "additionalProperties": false
+}

org-config.template.jsonc

@@ -0,0 +1,13 @@
+{
+  "$schema": "./org-config.schema.json",
+
+  "name": "Giveth Verifier",
+  "schemaId": "0xf63f2a7159ee674aa6fce42196a8bb0605eafcf20c19e91a7eafba8d39fa0404",
+  "authorizedAttestor": "0x93E79499b00a2fdAAC38e6005B0ad8E88b177346",
+
+  "network": "optimism-mainnet",
+
+  // Optional
+  "color": "#FFFFFF",
+  "startBlock": 999999999999,
+}

package.json

@@ -9,7 +9,8 @@
     "test": "npm run test:run-fresh-db; dotenvx run --env-file=.env.test -- jest --runInBand",
     "clear:generate:migration:run:locally": "sqd codegen ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d;sqd migration:generate; sqd run",
     "clear:run:locally": "sqd build ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d; sqd run",
-    "run:locally": "docker compose -f docker-compose-potgres.yml up -d; sqd build; sqd run"
+    "run:locally": "docker compose -f docker-compose-potgres.yml up -d; sqd build; sqd run",
+    "add-organization": "node add-organisation.js"
   },
   "dependencies": {
     "@ethereum-attestation-service/eas-sdk": "^1.5.0",
@@ -21,6 +22,7 @@
     "dotenv": "^16.4.4",
     "ethers": "^6.12.1",
     "html-to-text": "^9.0.5",
+    "jsonc-parser": "^3.2.1",
     "node-cron": "^3.0.3",
     "pg": "^8.11.5",
     "showdown": "^2.1.0",

schema.graphql

@@ -35,6 +35,8 @@ type Organisation @entity {
   issuer: String!
   "Color of the organization"
   color: String
+  "The first attestation block number"
+  startBlock: Int
   "Organization Attestors"
   attestors: [AttestorOrganisation!]! @derivedFrom(field: "organisation")
   attestedProjects: [OrganisationProject!]! @derivedFrom(field: "organisation")

src/controllers/utils/databaseHelper.ts

@@ -1,9 +1,21 @@
 import { DataHandlerContext } from "@subsquid/evm-processor";
+import { createOrmConfig } from "@subsquid/typeorm-config";
 import { Store } from "@subsquid/typeorm-store";
 import { assert } from "console";
-import { EntityManager } from "typeorm/entity-manager/EntityManager";
+import { DataSource, EntityManager } from "typeorm";
 
-export const getEntityManger = (
+let connection: DataSource | undefined;
+export async function getEntityManagerByConnection(): Promise<EntityManager> {
+  if (!connection) {
+    let cfg = createOrmConfig({ projectDir: __dirname + "/../../.." });
+    (cfg.entities as string[]).push(__dirname + "/../../model/generated/*.ts");
+
+    connection = await new DataSource(cfg).initialize();
+  }
+  return connection.createEntityManager();
+}
+
+export const getEntityMangerByContext = (
   ctx: DataHandlerContext<Store>
 ): EntityManager => {
   const em = (ctx.store as unknown as { em: () => EntityManager }).em();

src/controllers/utils/modelHelper.ts

@@ -6,7 +6,7 @@ import {
   OrganisationProject,
   Project,
 } from "../../model";
-import { getEntityManger } from "./databaseHelper";
+import { getEntityMangerByContext } from "./databaseHelper";
 import { ProjectStats } from "./types";
 
 export const upsertOrganisatoinProject = async (
@@ -32,7 +32,7 @@ export const updateProjectAttestationCounts = async (
   ctx: DataHandlerContext<Store>,
   project: Project
 ): Promise<void> => {
-  const em = getEntityManger(ctx);
+  const em = getEntityMangerByContext(ctx);
   const projectStats = await getProjectStats(ctx, project);
 
   project.totalVouches = projectStats.pr_total_vouches;
@@ -96,7 +96,7 @@ export const getProjectStats = async (
   ctx: DataHandlerContext<Store>,
   project: Project
 ): Promise<ProjectStats> => {
-  const em = getEntityManger(ctx);
+  const em = getEntityMangerByContext(ctx);
 
   const result = await em.query(
     `

src/features/import-projects/gitcoin/constants.ts

@@ -1,3 +1,5 @@
+import { SourceConfig } from "../types";
+
 export const GITCOIN_API_URL =
   process.env.GITCOIN_API_URL ||
   "https://grants-stack-indexer-v2.gitcoin.co/graphql";

src/features/import-projects/giveth/constants.ts

@@ -1,3 +1,5 @@
+import { SourceConfig } from "../types";
+
 export const GIVETH_API_URL =
   process.env.GIVETH_API_URL || "https://mainnet.serve.giveth.io/graphql";
 

src/features/import-projects/helpers.ts

@@ -3,6 +3,7 @@ import { Project } from "../../model";
 import { getDataSource } from "../../helpers/db";
 import { DESCRIPTION_SUMMARY_LENGTH } from "../../constants";
 import { convert } from "html-to-text";
+import { SourceConfig } from "./types";
 
 export const updateOrCreateProject = async (
   project: any,

src/features/import-projects/rf4/constants.ts

@@ -1,3 +1,5 @@
+import { SourceConfig } from "../types";
+
 export const RF4_API_URL =
   process.env.RF4_API_URL || "https://round4-api-eas.retrolist.app/projects";
 

src/features/import-projects/rpgf/constants.ts

@@ -1,3 +1,5 @@
+import { SourceConfig } from "../types";
+
 export const RPGF3_API_URL =
   process.env.RPGF3_API_URL || "https://backend.pairwise.vote/mock/projects";
 

src/features/import-projects/types.ts

@@ -1,4 +1,4 @@
-interface SourceConfig {
+export interface SourceConfig {
   source: string;
   idField: string;
   titleField: string;

src/main.ts

@@ -1,23 +1,27 @@
 import { TypeormDatabase } from "@subsquid/typeorm-store";
-import { processor } from "./processor";
+import { Processor } from "./processor";
 import * as EASContract from "./abi/EAS";
 import { processAttest } from "./mappings/attest";
 import { processRevokeLog } from "./mappings/revoke";
 import { importProjects } from "./features/import-projects/index";
 
-processor.run(new TypeormDatabase({ supportHotBlocks: false }), async (ctx) => {
-  for (let _block of ctx.blocks) {
-    for (let _log of _block.logs) {
-      switch (_log.topics[0]) {
-        case EASContract.events.Attested.topic:
-          await processAttest(ctx, _log);
-          break;
+Processor.getInstance().then(async (processor) => {
+  processor.run(
+    new TypeormDatabase({ supportHotBlocks: false }),
+    async (ctx) => {
+      for (let _block of ctx.blocks) {
+        for (let _log of _block.logs) {
+          switch (_log.topics[0]) {
+            case EASContract.events.Attested.topic:
+              await processAttest(ctx, _log);
+              break;
 
-        case EASContract.events.Revoked.topic:
-          await processRevokeLog(ctx, _log);
+            case EASContract.events.Revoked.topic:
+              await processRevokeLog(ctx, _log);
+          }
+        }
       }
     }
-  }
+  );
 });
-
 importProjects();