From cae545264bfcce16e52a08dd9aa911999ab24ea0 Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Wed, 12 Jun 2024 14:52:35 +0330 Subject: [PATCH 1/2] Added support for organisation start block - Changed organisation add procedure --- .gitignore | 4 +- README.md | 99 ++-------------- db/create-organisation-add-migration.js | 8 +- db/migrations/1718184531403-Data.js | 11 ++ org-config.schema.json | 45 ++++++++ org-config.template.jsonc | 13 +++ package-lock.json | 6 + package.json | 4 +- schema.graphql | 2 + src/controllers/utils/databaseHelper.ts | 16 ++- src/controllers/utils/modelHelper.ts | 6 +- .../import-projects/gitcoin/constants.ts | 2 + .../import-projects/giveth/constants.ts | 2 + src/features/import-projects/helpers.ts | 1 + src/features/import-projects/rf4/constants.ts | 2 + .../import-projects/rpgf/constants.ts | 2 + src/features/import-projects/types.ts | 2 +- src/main.ts | 28 +++-- src/model/generated/organisation.model.ts | 6 + src/processor.ts | 107 ++++++++++++------ src/test/utils.ts | 4 +- 21 files changed, 221 insertions(+), 149 deletions(-) create mode 100644 db/migrations/1718184531403-Data.js create mode 100644 org-config.schema.json create mode 100644 org-config.template.jsonc diff --git a/.gitignore b/.gitignore index 6122a86..73a2054 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,5 @@ # Backup file .env.backup -# Exclude add-organisation updates on pushes -add-organisation.js \ No newline at end of file +# Rogranisation config +org-config.jsonc \ No newline at end of file diff --git a/README.md b/README.md index fc0d3c0..feea365 100644 --- a/README.md +++ b/README.md @@ -1,97 +1,20 @@ -[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/subsquid/squid-evm-template) +[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/Giveth/DeVouch-BE) -# Minimal EVM squid +# Introduction -This is a starter template of a squid indexer for EVM networks (Ethereum, Polygon, BSC, etc.). See [Squid SDK docs](https://docs.subsquid.io/) for a complete reference. +Devouch is a decentralized application that allows users to attest to projects credibility by their vouches or flags. The backend is built on Subsquid and uses a Postgres database. -To extract EVM logs and transactions by a topic or a contract address, use [`.addLog()`](https://docs.subsquid.io/evm-indexing/configuration/evm-logs/), [`.addTransaction()`](https://docs.subsquid.io/evm-indexing/configuration/transactions/), [`.addTrace()`](https://docs.subsquid.io/evm-indexing/configuration/traces/) or [`.addStateDiff()`](https://docs.subsquid.io/evm-indexing/configuration/state-diffs/) methods of the `EvmBatchProcessor` instance defined in `src/processor.ts`. Select data fields with [`.setFields()`](https://docs.subsquid.io/evm-indexing/configuration/data-selection/). +# Getting Started -The requested data is transformed in batches by a single handler provided to the `processor.run()` method. -For a full list of supported networks and config options, -check the [`EvmBatchProcessor` overview](https://docs.subsquid.io/evm-indexing/evm-processor/) and the [setup section](https://docs.subsquid.io/evm-indexing/configuration/). - -For a step-by-step migration guide from TheGraph, see [the dedicated docs page](https://docs.subsquid.io/migrate/migrate-subgraph/). - -Dependencies: Node.js v16 or newer, Git, Docker. - -## Quickstart - -```bash -# 0. Install @subsquid/cli a.k.a. the sqd command globally -npm i -g @subsquid/cli - -# 1. Retrieve the template -sqd init my_squid_name -t evm -cd my_squid_name - -# 2. Install dependencies -npm ci - -# 3. Start a Postgres database container and detach -sqd up - -# 4. Build the squid -sqd build - -# 5. Start both the squid processor and the GraphQL server -sqd run . -``` -A GraphiQL playground will be available at [localhost:4350/graphql](http://localhost:4350/graphql). - -You can also start squid services one by one: -```bash -sqd process -sqd serve -``` - -## Dev flow - -### 1. Define database schema - -Start development by defining the schema of the target database via `schema.graphql`. -Schema definition consists of regular graphql type declarations annotated with custom directives. -Full description of `schema.graphql` dialect is available [here](https://docs.subsquid.io/store/postgres/schema-file/). - -### 2. Generate TypeORM classes - -Mapping developers use TypeORM [EntityManager](https://typeorm.io/#/working-with-entity-manager) -to interact with target database during data processing. All necessary entity classes are -generated by the squid framework from `schema.graphql`. This is done by running `sqd codegen` -command. - -### 3. Generate database migrations - -All database changes are applied through migration files located at `db/migrations`. -`squid-typeorm-migration(1)` tool provides several commands to drive the process. +## Add new organisation ```bash -## drop create the database -sqd down -sqd up +# 0. Copy the org-config.template.json to org-config.json +cp org-config.template.json org-config.json -## replace any old schemas with a new one made from the entities -sqd migration:generate +# 1. Fill the config with your organisation data +# 2. Run the script to add your organisation data to migrations +npm run add-organisation ``` -See [docs on database migrations](https://docs.subsquid.io/store/postgres/db-migrations/) for more details. - -### 4. Import ABI contract and generate interfaces to decode events - -It is necessary to import the respective ABI definition to decode EVM logs. One way to generate a type-safe facade class to decode EVM logs is by placing the relevant JSON ABIs to `./abi`, then using `squid-evm-typegen(1)` via an `sqd` script: - -```bash -sqd typegen -``` - -See more details on the [`squid-evm-typegen` doc page](https://docs.subsquid.io/evm-indexing/squid-evm-typegen). - -## Project conventions - -Squid tools assume a certain [project layout](https://docs.subsquid.io/basics/squid-structure): - -* All compiled js files must reside in `lib` and all TypeScript sources in `src`. -The layout of `lib` must reflect `src`. -* All TypeORM classes must be exported by `src/model/index.ts` (`lib/model` module). -* Database schema must be defined in `schema.graphql`. -* Database migrations must reside in `db/migrations` and must be plain js files. -* `sqd(1)` and `squid-*(1)` executables consult `.env` file for environment variables. +Then create a PR to the main branch to be reviewed and merged. \ No newline at end of file diff --git a/db/create-organisation-add-migration.js b/db/create-organisation-add-migration.js index 28926cb..bdd1651 100644 --- a/db/create-organisation-add-migration.js +++ b/db/create-organisation-add-migration.js @@ -8,7 +8,8 @@ exports.default = function createOrganisationAddMigration( schemaId, authorizedAttestor, color = null, - network = "eth-sepolia" + network = "eth-sepolia", + startBlock = null ) { const timestamp = new Date().getTime() + ADD_ORG_MIGRATION_OFFSET; const fileName = `${timestamp}-Add${organisationName}.js`; @@ -25,12 +26,13 @@ exports.default = function createOrganisationAddMigration( if (SQUID_NETWORK !== "${network}") return; // add organisation with name "${organisationName}" and schema id "${schemaId}" await db.query( - \`INSERT INTO "organisation" ("id", "name", "issuer", "color") + \`INSERT INTO "organisation" ("id", "name", "issuer", "color", "start_block") VALUES ( '${schemaId.toLocaleLowerCase()}', '${organisationName}', '${authorizedAttestor.toLocaleLowerCase()}', - ${color ? "'" + color.toLocaleLowerCase() + "'" : null} + ${color ? "'" + color.toLocaleLowerCase() + "'" : null}, + ${startBlock} )\` ); } diff --git a/db/migrations/1718184531403-Data.js b/db/migrations/1718184531403-Data.js new file mode 100644 index 0000000..8a6ef13 --- /dev/null +++ b/db/migrations/1718184531403-Data.js @@ -0,0 +1,11 @@ +module.exports = class Data1718184531403 { + name = 'Data1718184531403' + + async up(db) { + await db.query(`ALTER TABLE "organisation" ADD "start_block" integer`) + } + + async down(db) { + await db.query(`ALTER TABLE "organisation" DROP COLUMN "start_block"`) + } +} diff --git a/org-config.schema.json b/org-config.schema.json new file mode 100644 index 0000000..2b89d9a --- /dev/null +++ b/org-config.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Organization Configuration", + "description": "Schema for the organization configuration file.", + "type": "object", + + "properties": { + "$schema": { + "description": "The schema version.", + "type": "string" + }, + "name": { + "description": "The name of the organization.", + "type": "string" + }, + "schemaId": { + "description": "The UID for the attestation schema.", + "type": "string", + "pattern": "^0x[A-Fa-f0-9]{64}$" + }, + "authorizedAttestor": { + "description": "The address of the authorized attestor.", + "type": "string", + "pattern": "^0x[A-Fa-f0-9]{40}$" + }, + + "network": { + "description": "The network for the organization.", + "type": "string", + "enum": ["eth-sepolia", "optimism-mainnet"] + }, + + "color": { + "description": "The organisaiton color in UI.", + "type": "string", + "pattern": "^#[A-Fa-f0-9]{6}$" + }, + "startBlock": { + "description": "The block number at which the organization was created.", + "type": "integer" + } + }, + "required": ["name", "schemaId", "authorizedAttestor", "network"], + "additionalProperties": false +} diff --git a/org-config.template.jsonc b/org-config.template.jsonc new file mode 100644 index 0000000..7fd431d --- /dev/null +++ b/org-config.template.jsonc @@ -0,0 +1,13 @@ +{ + "$schema": "./org-config.schema.json", + + "name": "Giveth Verifier", + "schemaId": "0xf63f2a7159ee674aa6fce42196a8bb0605eafcf20c19e91a7eafba8d39fa0404", + "authorizedAttestor": "0x93E79499b00a2fdAAC38e6005B0ad8E88b177346", + + "network": "optimism-mainnet", + + // Optional + "color": "#FFFFFF", + "startBlock": 999999999999, +} diff --git a/package-lock.json b/package-lock.json index b2c7a53..bd1fc1d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "dotenv": "^16.4.4", "ethers": "^6.12.1", "html-to-text": "^9.0.5", + "jsonc-parser": "^3.2.1", "node-cron": "^3.0.3", "pg": "^8.11.5", "showdown": "^2.1.0", @@ -9319,6 +9320,11 @@ "node": ">=6" } }, + "node_modules/jsonc-parser": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.1.tgz", + "integrity": "sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA==" + }, "node_modules/jsonfile": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", diff --git a/package.json b/package.json index fc87226..587c8f9 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "test": "npm run test:run-fresh-db; dotenvx run --env-file=.env.test -- jest --runInBand", "clear:generate:migration:run:locally": "sqd codegen ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d;sqd migration:generate; sqd run", "clear:run:locally": "sqd build ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d; sqd run", - "run:locally": "docker compose -f docker-compose-potgres.yml up -d; sqd build; sqd run" + "run:locally": "docker compose -f docker-compose-potgres.yml up -d; sqd build; sqd run", + "add-organization": "node add-organisation.js" }, "dependencies": { "@ethereum-attestation-service/eas-sdk": "^1.5.0", @@ -21,6 +22,7 @@ "dotenv": "^16.4.4", "ethers": "^6.12.1", "html-to-text": "^9.0.5", + "jsonc-parser": "^3.2.1", "node-cron": "^3.0.3", "pg": "^8.11.5", "showdown": "^2.1.0", diff --git a/schema.graphql b/schema.graphql index 54308af..4206bfa 100644 --- a/schema.graphql +++ b/schema.graphql @@ -35,6 +35,8 @@ type Organisation @entity { issuer: String! "Color of the organization" color: String + "The first attestation block number" + startBlock: Int "Organization Attestors" attestors: [AttestorOrganisation!]! @derivedFrom(field: "organisation") attestedProjects: [OrganisationProject!]! @derivedFrom(field: "organisation") diff --git a/src/controllers/utils/databaseHelper.ts b/src/controllers/utils/databaseHelper.ts index a54a93d..e731bd6 100644 --- a/src/controllers/utils/databaseHelper.ts +++ b/src/controllers/utils/databaseHelper.ts @@ -1,9 +1,21 @@ import { DataHandlerContext } from "@subsquid/evm-processor"; +import { createOrmConfig } from "@subsquid/typeorm-config"; import { Store } from "@subsquid/typeorm-store"; import { assert } from "console"; -import { EntityManager } from "typeorm/entity-manager/EntityManager"; +import { DataSource, EntityManager } from "typeorm"; -export const getEntityManger = ( +let connection: DataSource | undefined; +export async function getEntityManagerByConnection(): Promise { + if (!connection) { + let cfg = createOrmConfig({ projectDir: __dirname + "/../../.." }); + (cfg.entities as string[]).push(__dirname + "/../../model/generated/*.ts"); + + connection = await new DataSource(cfg).initialize(); + } + return connection.createEntityManager(); +} + +export const getEntityMangerByContext = ( ctx: DataHandlerContext ): EntityManager => { const em = (ctx.store as unknown as { em: () => EntityManager }).em(); diff --git a/src/controllers/utils/modelHelper.ts b/src/controllers/utils/modelHelper.ts index b5b129f..ce26bd0 100644 --- a/src/controllers/utils/modelHelper.ts +++ b/src/controllers/utils/modelHelper.ts @@ -6,7 +6,7 @@ import { OrganisationProject, Project, } from "../../model"; -import { getEntityManger } from "./databaseHelper"; +import { getEntityMangerByContext } from "./databaseHelper"; import { ProjectStats } from "./types"; export const upsertOrganisatoinProject = async ( @@ -32,7 +32,7 @@ export const updateProjectAttestationCounts = async ( ctx: DataHandlerContext, project: Project ): Promise => { - const em = getEntityManger(ctx); + const em = getEntityMangerByContext(ctx); const projectStats = await getProjectStats(ctx, project); project.totalVouches = projectStats.pr_total_vouches; @@ -96,7 +96,7 @@ export const getProjectStats = async ( ctx: DataHandlerContext, project: Project ): Promise => { - const em = getEntityManger(ctx); + const em = getEntityMangerByContext(ctx); const result = await em.query( ` diff --git a/src/features/import-projects/gitcoin/constants.ts b/src/features/import-projects/gitcoin/constants.ts index 5571b94..e98ad19 100644 --- a/src/features/import-projects/gitcoin/constants.ts +++ b/src/features/import-projects/gitcoin/constants.ts @@ -1,3 +1,5 @@ +import { SourceConfig } from "../types"; + export const GITCOIN_API_URL = process.env.GITCOIN_API_URL || "https://grants-stack-indexer-v2.gitcoin.co/graphql"; diff --git a/src/features/import-projects/giveth/constants.ts b/src/features/import-projects/giveth/constants.ts index 12a2169..d2160ff 100644 --- a/src/features/import-projects/giveth/constants.ts +++ b/src/features/import-projects/giveth/constants.ts @@ -1,3 +1,5 @@ +import { SourceConfig } from "../types"; + export const GIVETH_API_URL = process.env.GIVETH_API_URL || "https://mainnet.serve.giveth.io/graphql"; diff --git a/src/features/import-projects/helpers.ts b/src/features/import-projects/helpers.ts index f07ae6f..a87522f 100644 --- a/src/features/import-projects/helpers.ts +++ b/src/features/import-projects/helpers.ts @@ -3,6 +3,7 @@ import { Project } from "../../model"; import { getDataSource } from "../../helpers/db"; import { DESCRIPTION_SUMMARY_LENGTH } from "../../constants"; import { convert } from "html-to-text"; +import { SourceConfig } from "./types"; export const updateOrCreateProject = async ( project: any, diff --git a/src/features/import-projects/rf4/constants.ts b/src/features/import-projects/rf4/constants.ts index 715a3d2..6e54bc4 100644 --- a/src/features/import-projects/rf4/constants.ts +++ b/src/features/import-projects/rf4/constants.ts @@ -1,3 +1,5 @@ +import { SourceConfig } from "../types"; + export const RF4_API_URL = process.env.RF4_API_URL || "https://round4-api-eas.retrolist.app/projects"; diff --git a/src/features/import-projects/rpgf/constants.ts b/src/features/import-projects/rpgf/constants.ts index bf7d641..5565108 100644 --- a/src/features/import-projects/rpgf/constants.ts +++ b/src/features/import-projects/rpgf/constants.ts @@ -1,3 +1,5 @@ +import { SourceConfig } from "../types"; + export const RPGF3_API_URL = process.env.RPGF3_API_URL || "https://backend.pairwise.vote/mock/projects"; diff --git a/src/features/import-projects/types.ts b/src/features/import-projects/types.ts index 9444c0f..2c334d6 100644 --- a/src/features/import-projects/types.ts +++ b/src/features/import-projects/types.ts @@ -1,4 +1,4 @@ -interface SourceConfig { +export interface SourceConfig { source: string; idField: string; titleField: string; diff --git a/src/main.ts b/src/main.ts index 95e6ade..14a603c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,23 +1,27 @@ import { TypeormDatabase } from "@subsquid/typeorm-store"; -import { processor } from "./processor"; +import { Processor } from "./processor"; import * as EASContract from "./abi/EAS"; import { processAttest } from "./mappings/attest"; import { processRevokeLog } from "./mappings/revoke"; import { importProjects } from "./features/import-projects/index"; -processor.run(new TypeormDatabase({ supportHotBlocks: false }), async (ctx) => { - for (let _block of ctx.blocks) { - for (let _log of _block.logs) { - switch (_log.topics[0]) { - case EASContract.events.Attested.topic: - await processAttest(ctx, _log); - break; +Processor.getInstance().then(async (processor) => { + processor.run( + new TypeormDatabase({ supportHotBlocks: false }), + async (ctx) => { + for (let _block of ctx.blocks) { + for (let _log of _block.logs) { + switch (_log.topics[0]) { + case EASContract.events.Attested.topic: + await processAttest(ctx, _log); + break; - case EASContract.events.Revoked.topic: - await processRevokeLog(ctx, _log); + case EASContract.events.Revoked.topic: + await processRevokeLog(ctx, _log); + } + } } } - } + ); }); - importProjects(); diff --git a/src/model/generated/organisation.model.ts b/src/model/generated/organisation.model.ts index b2db066..4eb6a52 100644 --- a/src/model/generated/organisation.model.ts +++ b/src/model/generated/organisation.model.ts @@ -33,6 +33,12 @@ export class Organisation { @Column_("text", {nullable: true}) color!: string | undefined | null + /** + * The first attestation block number + */ + @Column_("int4", {nullable: true}) + startBlock!: number | undefined | null + /** * Organization Attestors */ diff --git a/src/processor.ts b/src/processor.ts index 7fd0b6a..d8f7f74 100644 --- a/src/processor.ts +++ b/src/processor.ts @@ -10,42 +10,79 @@ import { import * as EASContract from "./abi/EAS"; import { EAS_CONTRACT_ADDRESS, LOOKUP_ARCHIVE, START_BLOCK } from "./constants"; +import { getEntityManagerByConnection } from "./controllers/utils/databaseHelper"; +import { Organisation } from "./model"; -export const processor = new EvmBatchProcessor() - // Lookup archive by the network name in Subsquid registry - // See https://docs.subsquid.io/evm-indexing/supported-networks/ - .setGateway(LOOKUP_ARCHIVE) - // Chain RPC endpoint is required for - // - indexing unfinalized blocks https://docs.subsquid.io/basics/unfinalized-blocks/ - // - querying the contract state https://docs.subsquid.io/evm-indexing/query-state/ - .setRpcEndpoint({ - // Set the URL via .env for local runs or via secrets when deploying to Subsquid Cloud - // https://docs.subsquid.io/deploy-squid/env-variables/ - url: assertNotNull(process.env.RPC_ENDPOINT), - // More RPC connection options at https://docs.subsquid.io/evm-indexing/configuration/initialization/#set-data-source - rateLimit: 10, - }) - .setFinalityConfirmation(1) - .setFields({ - log: { - topics: true, - data: true, - transactionHash: true, - }, - }) - .setBlockRange({ - from: START_BLOCK, - }) - .addLog({ - address: [EAS_CONTRACT_ADDRESS], - topic0: [ - EASContract.events.Attested.topic, - EASContract.events.Revoked.topic, - ], - transaction: true, - }); - -export type Fields = EvmBatchProcessorFields; +export class Processor { + private static instance: EvmBatchProcessor; + + static async getInstance(): Promise { + if (!Processor.instance) { + const em = await getEntityManagerByConnection(); + const result = await em + .getRepository(Organisation) + .query( + "select min(start_block) as start_block from organisation where start_block is not null" + ); + + let startBlock = Math.min( + result[0].start_block || Number.MAX_SAFE_INTEGER, + START_BLOCK + ); + + console.log("############################"); + if (result[0].start_block) { + console.log("Default start block: " + START_BLOCK); + console.log("Start block from DB: " + result[0].start_block); + } + console.log("Subsquid config start block: " + startBlock); + console.log("############################"); + + const start_block = Math.min( + result[0].start_block || Number.MAX_SAFE_INTEGER, + START_BLOCK + ); + + console.log("result", result); + + Processor.instance = new EvmBatchProcessor() + // Lookup archive by the network name in Subsquid registry + // See https://docs.subsquid.io/evm-indexing/supported-networks/ + .setGateway(LOOKUP_ARCHIVE) + // Chain RPC endpoint is required for + // - indexing unfinalized blocks https://docs.subsquid.io/basics/unfinalized-blocks/ + // - querying the contract state https://docs.subsquid.io/evm-indexing/query-state/ + .setRpcEndpoint({ + // Set the URL via .env for local runs or via secrets when deploying to Subsquid Cloud + // https://docs.subsquid.io/deploy-squid/env-variables/ + url: assertNotNull(process.env.RPC_ENDPOINT), + // More RPC connection options at https://docs.subsquid.io/evm-indexing/configuration/initialization/#set-data-source + rateLimit: 10, + }) + .setFinalityConfirmation(1) + .setFields({ + log: { + topics: true, + data: true, + transactionHash: true, + }, + }) + .setBlockRange({ + from: START_BLOCK, + }) + .addLog({ + address: [EAS_CONTRACT_ADDRESS], + topic0: [ + EASContract.events.Attested.topic, + EASContract.events.Revoked.topic, + ], + transaction: true, + }); + } + return Processor.instance; + } +} +export type Fields = EvmBatchProcessorFields; export type Block = BlockHeader; export type Log = _Log; export type Transaction = _Transaction; diff --git a/src/test/utils.ts b/src/test/utils.ts index 96cce4b..ded0044 100644 --- a/src/test/utils.ts +++ b/src/test/utils.ts @@ -1,6 +1,6 @@ -import { Store, TypeormDatabase } from "@subsquid/typeorm-store"; +import { Store } from "@subsquid/typeorm-store"; import { createOrmConfig } from "@subsquid/typeorm-config"; -import { DataSource, DataSourceOptions, EntityManager } from "typeorm"; +import { DataSource, EntityManager } from "typeorm"; import { DataHandlerContext } from "@subsquid/evm-processor"; // import dotenv from "dotenv"; From f1b497b104be2049347cff71a45614ba433e05bc Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Wed, 12 Jun 2024 17:12:06 +0330 Subject: [PATCH 2/2] Fixed typo --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 73a2054..8a8dfac 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,5 @@ # Backup file .env.backup -# Rogranisation config +# Ogranisation config org-config.jsonc \ No newline at end of file