From d413d724c7a8af251ed76c3050d8c1df2577f573 Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Mon, 16 Sep 2024 19:05:45 +0530 Subject: [PATCH] Add ETH RPC API to get logs (#536) * Add eth_getLogs API handler * Transform events into logs * Update codegen templates * Allow GET requests * Increment package verisons * Remove unnecessary todo * Add limit on getLogs results size * Fix config template --- lerna.json | 2 +- packages/cache/package.json | 2 +- packages/cli/package.json | 12 +- packages/cli/src/server.ts | 9 +- packages/codegen/package.json | 4 +- .../src/templates/config-template.handlebars | 6 + .../templates/database-template.handlebars | 6 + .../src/templates/indexer-template.handlebars | 8 ++ .../src/templates/package-template.handlebars | 10 +- packages/graph-node/package.json | 10 +- packages/graph-node/test/utils/indexer.ts | 6 + packages/ipld-eth-client/package.json | 6 +- packages/peer/package.json | 2 +- packages/rpc-eth-client/package.json | 8 +- packages/solidity-mapper/package.json | 2 +- packages/test/package.json | 2 +- packages/tracing-client/package.json | 2 +- packages/util/package.json | 8 +- packages/util/src/config.ts | 3 + packages/util/src/constants.ts | 2 + packages/util/src/database.ts | 6 + packages/util/src/eth-rpc-handlers.ts | 124 +++++++++++++++++- packages/util/src/server.ts | 8 +- packages/util/src/types.ts | 1 + 24 files changed, 207 insertions(+), 42 deletions(-) diff --git a/lerna.json b/lerna.json index bf0b3e435..b91965a9d 100644 --- a/lerna.json +++ b/lerna.json @@ -2,7 +2,7 @@ "packages": [ "packages/*" ], - "version": "0.2.106", + "version": "0.2.107", "npmClient": "yarn", "useWorkspaces": true, "command": { diff --git a/packages/cache/package.json b/packages/cache/package.json index 24b19313b..02ee47380 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/cache", - "version": "0.2.106", + "version": "0.2.107", "description": "Generic object cache", "main": "dist/index.js", "scripts": { diff --git a/packages/cli/package.json b/packages/cli/package.json index fee221ca6..1ca515032 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/cli", - "version": "0.2.106", + "version": "0.2.107", "main": "dist/index.js", "license": "AGPL-3.0", "scripts": { @@ -15,13 +15,13 @@ }, "dependencies": { "@apollo/client": "^3.7.1", - "@cerc-io/cache": "^0.2.106", - "@cerc-io/ipld-eth-client": "^0.2.106", + "@cerc-io/cache": "^0.2.107", + "@cerc-io/ipld-eth-client": "^0.2.107", "@cerc-io/libp2p": "^0.42.2-laconic-0.1.4", "@cerc-io/nitro-node": "^0.1.15", - "@cerc-io/peer": "^0.2.106", - "@cerc-io/rpc-eth-client": "^0.2.106", - "@cerc-io/util": "^0.2.106", + "@cerc-io/peer": "^0.2.107", + "@cerc-io/rpc-eth-client": "^0.2.107", + "@cerc-io/util": "^0.2.107", "@ethersproject/providers": "^5.4.4", "@graphql-tools/utils": "^9.1.1", "@ipld/dag-cbor": "^8.0.0", diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 98c64f9a4..a92b3529f 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -323,7 +323,14 @@ export class ServerCmd { // Create an Express app const app: Application = express(); - const server = await createAndStartServer(app, typeDefs, resolvers, ethRPCHandlers, config.server, paymentsManager); + const server = await createAndStartServer( + app, + typeDefs, + resolvers, + ethRPCHandlers, + config.server, + paymentsManager + ); await startGQLMetricsServer(config); diff --git a/packages/codegen/package.json b/packages/codegen/package.json index a3c944d22..750f57fc6 100644 --- a/packages/codegen/package.json +++ b/packages/codegen/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/codegen", - "version": "0.2.106", + "version": "0.2.107", "description": "Code generator", "private": true, "main": "index.js", @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { - "@cerc-io/util": "^0.2.106", + "@cerc-io/util": "^0.2.107", "@graphql-tools/load-files": "^6.5.2", "@npmcli/package-json": "^5.0.0", "@poanet/solidity-flattener": "https://github.com/vulcanize/solidity-flattener.git", diff --git a/packages/codegen/src/templates/config-template.handlebars b/packages/codegen/src/templates/config-template.handlebars index 50f513f70..37ca714e9 100644 --- a/packages/codegen/src/templates/config-template.handlebars +++ b/packages/codegen/src/templates/config-template.handlebars @@ -25,6 +25,12 @@ # Flag to specify whether RPC endpoint supports block hash as block tag parameter rpcSupportsBlockHashParam = true + # Enable ETH JSON RPC server at /rpc + enableEthRPCServer = true + + # Max number of logs that can be returned in a single getLogs request (default: 10000) + ethGetLogsResultLimit = 10000 + # Server GQL config [server.gql] path = "/graphql" diff --git a/packages/codegen/src/templates/database-template.handlebars b/packages/codegen/src/templates/database-template.handlebars index 31dc706c6..826c59fd3 100644 --- a/packages/codegen/src/templates/database-template.handlebars +++ b/packages/codegen/src/templates/database-template.handlebars @@ -199,6 +199,12 @@ export class Database implements DatabaseInterface { return this._baseDatabase.getEventsInRange(repo, fromBlockNumber, toBlockNumber); } + async getEvents (options: FindManyOptions): Promise> { + const repo = this._conn.getRepository(Event); + + return this._baseDatabase.getEvents(repo, options); + } + async saveEventEntity (queryRunner: QueryRunner, entity: Event): Promise { const repo = queryRunner.manager.getRepository(Event); return this._baseDatabase.saveEventEntity(repo, entity); diff --git a/packages/codegen/src/templates/indexer-template.handlebars b/packages/codegen/src/templates/indexer-template.handlebars index 11624ccca..453d8fe9f 100644 --- a/packages/codegen/src/templates/indexer-template.handlebars +++ b/packages/codegen/src/templates/indexer-template.handlebars @@ -188,6 +188,10 @@ export class Indexer implements IndexerInterface { return this._storageLayoutMap; } + get contractMap (): Map { + return this._contractMap; + } + {{#if (subgraphPath)}} get graphWatcher (): GraphWatcher { return this._graphWatcher; @@ -671,6 +675,10 @@ export class Indexer implements IndexerInterface { return this._baseIndexer.getEventsInRange(fromBlockNumber, toBlockNumber, this._serverConfig.gql.maxEventsBlockRange); } + async getEvents (options: FindManyOptions): Promise> { + return this._db.getEvents(options); + } + async getSyncStatus (): Promise { return this._baseIndexer.getSyncStatus(); } diff --git a/packages/codegen/src/templates/package-template.handlebars b/packages/codegen/src/templates/package-template.handlebars index e01ea43a9..8605251f3 100644 --- a/packages/codegen/src/templates/package-template.handlebars +++ b/packages/codegen/src/templates/package-template.handlebars @@ -41,12 +41,12 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.3.19", - "@cerc-io/cli": "^0.2.106", - "@cerc-io/ipld-eth-client": "^0.2.106", - "@cerc-io/solidity-mapper": "^0.2.106", - "@cerc-io/util": "^0.2.106", + "@cerc-io/cli": "^0.2.107", + "@cerc-io/ipld-eth-client": "^0.2.107", + "@cerc-io/solidity-mapper": "^0.2.107", + "@cerc-io/util": "^0.2.107", {{#if (subgraphPath)}} - "@cerc-io/graph-node": "^0.2.106", + "@cerc-io/graph-node": "^0.2.107", {{/if}} "@ethersproject/providers": "^5.4.4", "debug": "^4.3.1", diff --git a/packages/graph-node/package.json b/packages/graph-node/package.json index 642be2c50..4c59dab73 100644 --- a/packages/graph-node/package.json +++ b/packages/graph-node/package.json @@ -1,10 +1,10 @@ { "name": "@cerc-io/graph-node", - "version": "0.2.106", + "version": "0.2.107", "main": "dist/index.js", "license": "AGPL-3.0", "devDependencies": { - "@cerc-io/solidity-mapper": "^0.2.106", + "@cerc-io/solidity-mapper": "^0.2.107", "@ethersproject/providers": "^5.4.4", "@graphprotocol/graph-ts": "^0.22.0", "@nomiclabs/hardhat-ethers": "^2.0.2", @@ -51,9 +51,9 @@ "dependencies": { "@apollo/client": "^3.3.19", "@cerc-io/assemblyscript": "0.19.10-watcher-ts-0.1.2", - "@cerc-io/cache": "^0.2.106", - "@cerc-io/ipld-eth-client": "^0.2.106", - "@cerc-io/util": "^0.2.106", + "@cerc-io/cache": "^0.2.107", + "@cerc-io/ipld-eth-client": "^0.2.107", + "@cerc-io/util": "^0.2.107", "@types/json-diff": "^0.5.2", "@types/yargs": "^17.0.0", "bn.js": "^4.11.9", diff --git a/packages/graph-node/test/utils/indexer.ts b/packages/graph-node/test/utils/indexer.ts index 3930e97e4..c26caf0c5 100644 --- a/packages/graph-node/test/utils/indexer.ts +++ b/packages/graph-node/test/utils/indexer.ts @@ -91,6 +91,12 @@ export class Indexer implements IndexerInterface { return undefined; } + async getEvents (options: FindManyOptions): Promise> { + assert(options); + + return []; + } + async getSyncStatus (): Promise { return undefined; } diff --git a/packages/ipld-eth-client/package.json b/packages/ipld-eth-client/package.json index dee53e3ad..41803ac89 100644 --- a/packages/ipld-eth-client/package.json +++ b/packages/ipld-eth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/ipld-eth-client", - "version": "0.2.106", + "version": "0.2.107", "description": "IPLD ETH Client", "main": "dist/index.js", "scripts": { @@ -20,8 +20,8 @@ "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { "@apollo/client": "^3.7.1", - "@cerc-io/cache": "^0.2.106", - "@cerc-io/util": "^0.2.106", + "@cerc-io/cache": "^0.2.107", + "@cerc-io/util": "^0.2.107", "cross-fetch": "^3.1.4", "debug": "^4.3.1", "ethers": "^5.4.4", diff --git a/packages/peer/package.json b/packages/peer/package.json index ef20f4abd..db89e39db 100644 --- a/packages/peer/package.json +++ b/packages/peer/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/peer", - "version": "0.2.106", + "version": "0.2.107", "description": "libp2p module", "main": "dist/index.js", "exports": "./dist/index.js", diff --git a/packages/rpc-eth-client/package.json b/packages/rpc-eth-client/package.json index b13e3e3bf..0fa9b9394 100644 --- a/packages/rpc-eth-client/package.json +++ b/packages/rpc-eth-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/rpc-eth-client", - "version": "0.2.106", + "version": "0.2.107", "description": "RPC ETH Client", "main": "dist/index.js", "scripts": { @@ -19,9 +19,9 @@ }, "homepage": "https://github.com/cerc-io/watcher-ts#readme", "dependencies": { - "@cerc-io/cache": "^0.2.106", - "@cerc-io/ipld-eth-client": "^0.2.106", - "@cerc-io/util": "^0.2.106", + "@cerc-io/cache": "^0.2.107", + "@cerc-io/ipld-eth-client": "^0.2.107", + "@cerc-io/util": "^0.2.107", "chai": "^4.3.4", "ethers": "^5.4.4", "left-pad": "^1.3.0", diff --git a/packages/solidity-mapper/package.json b/packages/solidity-mapper/package.json index df25a1d24..ee2038a86 100644 --- a/packages/solidity-mapper/package.json +++ b/packages/solidity-mapper/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/solidity-mapper", - "version": "0.2.106", + "version": "0.2.107", "main": "dist/index.js", "license": "AGPL-3.0", "devDependencies": { diff --git a/packages/test/package.json b/packages/test/package.json index a85411417..8c8d5248a 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/test", - "version": "0.2.106", + "version": "0.2.107", "main": "dist/index.js", "license": "AGPL-3.0", "private": true, diff --git a/packages/tracing-client/package.json b/packages/tracing-client/package.json index fa8553d84..6db7e6aa5 100644 --- a/packages/tracing-client/package.json +++ b/packages/tracing-client/package.json @@ -1,6 +1,6 @@ { "name": "@cerc-io/tracing-client", - "version": "0.2.106", + "version": "0.2.107", "description": "ETH VM tracing client", "main": "dist/index.js", "scripts": { diff --git a/packages/util/package.json b/packages/util/package.json index 130f5e572..7c60e7c90 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -1,13 +1,13 @@ { "name": "@cerc-io/util", - "version": "0.2.106", + "version": "0.2.107", "main": "dist/index.js", "license": "AGPL-3.0", "dependencies": { "@apollo/utils.keyvaluecache": "^1.0.1", "@cerc-io/nitro-node": "^0.1.15", - "@cerc-io/peer": "^0.2.106", - "@cerc-io/solidity-mapper": "^0.2.106", + "@cerc-io/peer": "^0.2.107", + "@cerc-io/solidity-mapper": "^0.2.107", "@cerc-io/ts-channel": "1.0.3-ts-nitro-0.1.1", "@ethersproject/properties": "^5.7.0", "@ethersproject/providers": "^5.4.4", @@ -55,7 +55,7 @@ "yargs": "^17.0.1" }, "devDependencies": { - "@cerc-io/cache": "^0.2.106", + "@cerc-io/cache": "^0.2.107", "@nomiclabs/hardhat-waffle": "^2.0.1", "@types/bunyan": "^1.8.8", "@types/express": "^4.17.14", diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index 9567a87a0..62b3b096e 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -255,6 +255,9 @@ export interface ServerConfig { // Enable ETH JSON RPC server at /rpc enableEthRPCServer: boolean; + + // Max number of logs that can be returned in a single getLogs request + ethGetLogsResultLimit?: number; } export interface FundingAmountsConfig { diff --git a/packages/util/src/constants.ts b/packages/util/src/constants.ts index 7c1fd66e3..7bc135aec 100644 --- a/packages/util/src/constants.ts +++ b/packages/util/src/constants.ts @@ -30,3 +30,5 @@ export const DEFAULT_PREFETCH_BATCH_SIZE = 10; export const DEFAULT_MAX_GQL_CACHE_SIZE = Math.pow(2, 20) * 8; // 8 MB export const SUPPORTED_PAID_RPC_METHODS = ['eth_getBlockByHash', 'eth_getStorageAt', 'eth_getBlockByNumber']; + +export const DEFAULT_ETH_GET_LOGS_RESULT_LIMIT = 10000; diff --git a/packages/util/src/database.ts b/packages/util/src/database.ts index 2fcd642fd..e51f3629b 100644 --- a/packages/util/src/database.ts +++ b/packages/util/src/database.ts @@ -523,6 +523,12 @@ export class Database { return events; } + async getEvents (repo: Repository, options: FindManyOptions): Promise> { + const events = repo.find(options); + + return events; + } + async saveEventEntity (repo: Repository, entity: EventInterface): Promise { const event = await repo.save(entity); eventCount.inc(1); diff --git a/packages/util/src/eth-rpc-handlers.ts b/packages/util/src/eth-rpc-handlers.ts index 406a6a207..7cfbb16ae 100644 --- a/packages/util/src/eth-rpc-handlers.ts +++ b/packages/util/src/eth-rpc-handlers.ts @@ -1,9 +1,10 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { utils } from 'ethers'; +import { Between, Equal, FindConditions, In, LessThanOrEqual, MoreThanOrEqual } from 'typeorm'; import { JsonRpcProvider } from '@ethersproject/providers'; -import { IndexerInterface } from './types'; +import { EventInterface, IndexerInterface } from './types'; +import { DEFAULT_ETH_GET_LOGS_RESULT_LIMIT } from './constants'; const CODE_INVALID_PARAMS = -32602; const CODE_INTERNAL_ERROR = -32603; @@ -16,7 +17,10 @@ const ERROR_CONTRACT_NOT_RECOGNIZED = 'Contract not recognized'; const ERROR_CONTRACT_METHOD_NOT_FOUND = 'Contract method not found'; const ERROR_METHOD_NOT_IMPLEMENTED = 'Method not implemented'; const ERROR_INVALID_BLOCK_TAG = 'Invalid block tag'; +const ERROR_INVALID_BLOCK_HASH = 'Invalid block hash'; const ERROR_BLOCK_NOT_FOUND = 'Block not found'; +const ERROR_TOPICS_FILTER_NOT_SUPPORTED = 'Topics filter not supported'; +const ERROR_LIMIT_EXCEEDED = 'Query results exceeds limit'; const DEFAULT_BLOCK_TAG = 'latest'; @@ -53,7 +57,7 @@ export const createEthRPCHandlers = async ( const { to, data } = args[0]; const blockTag = args.length > 1 ? args[1] : DEFAULT_BLOCK_TAG; - const blockHash = await parseBlockTag(indexer, ethProvider, blockTag); + const blockHash = await parseEthCallBlockTag(indexer, ethProvider, blockTag); const watchedContract = indexer.getWatchedContracts().find(contract => contract.address === to); if (!watchedContract) { @@ -100,12 +104,87 @@ export const createEthRPCHandlers = async ( }, eth_getLogs: async (args: any, callback: any) => { - // TODO: Implement + try { + if (args.length === 0) { + throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_CONTRACT_INSUFFICIENT_PARAMS); + } + + const params = args[0]; + + // Parse arg params into where options + const where: FindConditions = {}; + + // TODO: Support topics filter + if (params.topics) { + throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_TOPICS_FILTER_NOT_SUPPORTED); + } + + // Address filter, address or a list of addresses + if (params.address) { + if (Array.isArray(params.address)) { + if (params.address.length > 0) { + where.contract = In(params.address); + } + } else { + where.contract = Equal(params.address); + } + } + + // Block hash takes precedence over fromBlock / toBlock if provided + if (params.blockHash) { + // Validate input block hash + if (!utils.isHexString(params.blockHash, 32)) { + throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_INVALID_BLOCK_HASH); + } + + where.block = { + blockHash: params.blockHash + }; + } else if (params.fromBlock || params.toBlock) { + const fromBlockNumber = params.fromBlock ? await parseEthGetLogsBlockTag(indexer, params.fromBlock) : null; + const toBlockNumber = params.toBlock ? await parseEthGetLogsBlockTag(indexer, params.toBlock) : null; + + if (fromBlockNumber && toBlockNumber) { + // Both fromBlock and toBlock set + where.block = { blockNumber: Between(fromBlockNumber, toBlockNumber) }; + } else if (fromBlockNumber) { + // Only fromBlock set + where.block = { blockNumber: MoreThanOrEqual(fromBlockNumber) }; + } else if (toBlockNumber) { + // Only toBlock set + where.block = { blockNumber: LessThanOrEqual(toBlockNumber) }; + } + } + + // Fetch events from the db + // Load block relation + const resultLimit = indexer.serverConfig.ethGetLogsResultLimit || DEFAULT_ETH_GET_LOGS_RESULT_LIMIT; + const events = await indexer.getEvents({ where, relations: ['block'], take: resultLimit + 1 }); + + // Limit number of results can be returned by a single query + if (events.length > resultLimit) { + throw new ErrorWithCode(CODE_SERVER_ERROR, `${ERROR_LIMIT_EXCEEDED}: ${resultLimit}`); + } + + // Transform events into result logs + const result = await transformEventsToLogs(events); + + callback(null, result); + } catch (error: any) { + let callBackError; + if (error instanceof ErrorWithCode) { + callBackError = { code: error.code, message: error.message }; + } else { + callBackError = { code: CODE_SERVER_ERROR, message: error.message }; + } + + callback(callBackError); + } } }; }; -const parseBlockTag = async (indexer: IndexerInterface, ethProvider: JsonRpcProvider, blockTag: string): Promise => { +const parseEthCallBlockTag = async (indexer: IndexerInterface, ethProvider: JsonRpcProvider, blockTag: string): Promise => { if (utils.isHexString(blockTag)) { // Return value if hex string is of block hash length if (utils.hexDataLength(blockTag) === 32) { @@ -132,3 +211,38 @@ const parseBlockTag = async (indexer: IndexerInterface, ethProvider: JsonRpcProv throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_INVALID_BLOCK_TAG); }; + +const parseEthGetLogsBlockTag = async (indexer: IndexerInterface, blockTag: string): Promise => { + if (utils.isHexString(blockTag)) { + return Number(blockTag); + } + + if (blockTag === DEFAULT_BLOCK_TAG) { + const syncStatus = await indexer.getSyncStatus(); + if (!syncStatus) { + throw new ErrorWithCode(CODE_INTERNAL_ERROR, 'SyncStatus not found'); + } + + return syncStatus.latestProcessedBlockNumber; + } + + throw new ErrorWithCode(CODE_INVALID_PARAMS, ERROR_INVALID_BLOCK_TAG); +}; + +const transformEventsToLogs = async (events: Array): Promise => { + return events.map(event => { + const parsedExtraInfo = JSON.parse(event.extraInfo); + + return { + address: event.contract.toLowerCase(), + blockHash: event.block.blockHash, + blockNumber: `0x${event.block.blockNumber.toString(16)}`, + transactionHash: event.txHash, + transactionIndex: `0x${parsedExtraInfo.tx.index.toString(16)}`, + logIndex: `0x${parsedExtraInfo.logIndex.toString(16)}`, + data: parsedExtraInfo.data, + topics: parsedExtraInfo.topics, + removed: event.block.isPruned + }; + }); +}; diff --git a/packages/util/src/server.ts b/packages/util/src/server.ts index d7acc26ac..491fd24d8 100644 --- a/packages/util/src/server.ts +++ b/packages/util/src/server.ts @@ -110,7 +110,13 @@ export const createAndStartServer = async ( app.use( ETH_RPC_PATH, jsonParser(), - // TODO: Handle GET requests as well to match Geth's behaviour + (req: any, res: any, next: () => void) => { + // Convert all GET requests to POST to avoid getting rejected from jayson server middleware + if (jayson.Utils.isMethod(req, 'GET')) { + req.method = 'POST'; + } + next(); + }, rpcServer.middleware() ); } diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index 5afca6688..f799fde1e 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -169,6 +169,7 @@ export interface IndexerInterface { getBlockProgressEntities (where: FindConditions, options: FindManyOptions): Promise getEntitiesForBlock (blockHash: string, tableName: string): Promise getEvent (id: string): Promise + getEvents (options: FindManyOptions): Promise> getSyncStatus (): Promise getStateSyncStatus (): Promise getBlocks (blockFilter: { blockHash?: string, blockNumber?: number }): Promise>