Consensus indexer converts data to CID

.github/workflows/build.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Node.js 📦
         uses: actions/setup-node@v2
         with:
-          node-version: '18'
+          node-version: "20"
 
       - name: Enable Corepack 📦
         run: corepack enable

indexers/db/docker-entrypoint-initdb.d/init-db.sql

@@ -192,6 +192,7 @@ CREATE TABLE consensus.events (
     phase text NOT NULL,
     pos integer NOT NULL,
     args text NOT NULL,
+    cid text,
     _id uuid NOT NULL,
     _block_range int8range NOT NULL
 );
@@ -226,6 +227,7 @@ CREATE TABLE consensus.extrinsics (
     tip numeric NOT NULL,
     fee numeric NOT NULL,
     pos integer NOT NULL,
+    cid text,
     _id uuid NOT NULL,
     _block_range int8range NOT NULL
 );
@@ -1093,11 +1095,11 @@ CREATE INDEX "0x30b779cc3aeeeec6" ON consensus.events USING gist (extrinsic_id,
 CREATE INDEX "0x358cafe370ac92ef" ON consensus.accounts USING gist (total, _block_range);
 CREATE INDEX "0x3ae5d1670e99e612" ON consensus.transfers USING gist ("timestamp", _block_range);
 CREATE INDEX "0x3d8ee08d232943ea" ON consensus.sections USING btree (id);
-CREATE INDEX "0x3fa25df2b17c6d2f" ON consensus.extrinsics USING gist (signature, _block_range);
 CREATE INDEX "0x4131da2c2ec8b5b7" ON consensus.transfers USING gist (date, _block_range);
 CREATE INDEX "0x43615b2452f72359" ON consensus.account_profiles USING btree (id);
 CREATE INDEX "0x444de3b3611c1fcd" ON consensus.account_histories USING gist (created_at, _block_range);
 CREATE INDEX "0x4a66afa700f00759" ON consensus.events USING gist (block_hash, _block_range);
+CREATE INDEX "0x4b2e240a52e09d22" ON consensus.extrinsics USING gist (cid, _block_range);
 CREATE INDEX "0x4c738865df5c227d" ON consensus.account_profiles USING gist (created_at, _block_range);
 CREATE INDEX "0x4cb388e53e3e30f3" ON consensus.accounts USING btree (id);
 CREATE INDEX "0x53fb0cf455c914c8" ON consensus.account_profiles USING gist (updated_at, _block_range);
@@ -1106,6 +1108,7 @@ CREATE INDEX "0x57fc196dcc99a091" ON consensus.events USING gist (block_height,
 CREATE INDEX "0x5921649101eeb57a" ON consensus.transfers USING gist (created_at, _block_range);
 CREATE INDEX "0x59386a58438fa05a" ON consensus.extrinsics USING gist (section, _block_range);
 CREATE INDEX "0x59f75d2bc1e6a0bc" ON consensus.blocks USING gist ("timestamp", _block_range);
+CREATE INDEX "0x5b5ab04a8ff6f214" ON consensus.events USING gist (cid, _block_range);
 CREATE INDEX "0x5c04eee35ba10ef1" ON consensus.extrinsic_modules USING gist (method, _block_range);
 CREATE INDEX "0x6008270492da5713" ON consensus.events USING gist ("timestamp", _block_range);
 CREATE INDEX "0x6131d72d57f2a188" ON consensus.blocks USING gist (hash, _block_range);
@@ -1153,7 +1156,6 @@ CREATE INDEX "0x5b57ecd94445ad2e" ON dictionary.extrinsics USING btree (call);
 CREATE INDEX "0x62b8f3181611d490" ON dictionary.events USING btree (module);
 CREATE INDEX "0xc0c9768d1987b60f" ON dictionary.events USING btree (block_height);
 
-
 CREATE INDEX "0x0237b4bb45c8cd3d" ON leaderboard.farmer_vote_and_block_total_values USING btree (created_at);
 CREATE INDEX "0x02f8da0e922b0056" ON leaderboard.account_transfer_sender_total_count_histories USING gist (created_at, _block_range);
 CREATE INDEX "0x03f87f972ff639fb" ON leaderboard.nominator_deposits_total_counts USING btree (rank);

indexers/mainnet/consensus/package.json

@@ -32,8 +32,9 @@
   "license": "MIT",
   "devDependencies": {
     "@apollo/client": "^3.7.16",
-    "@autonomys/auto-consensus": "^1.0.6",
-    "@autonomys/auto-utils": "^1.0.6",
+    "@autonomys/auto-consensus": "^1.0.8",
+    "@autonomys/auto-dag-data": "^1.0.8",
+    "@autonomys/auto-utils": "^1.0.8",
     "@polkadot/api": "^12.4.2",
     "@polkadot/types": "^14.1.1",
     "@polkadot/util": "^13.2.2",

indexers/mainnet/consensus/schema.graphql

@@ -42,12 +42,13 @@ type Extrinsic @entity {
   timestamp: Date! @index
   nonce: BigInt!
   signer: String! @index
-  signature: String! @index
+  signature: String!
   args: String!
   error: String!
   tip: BigInt!
   fee: BigInt!
   pos: Int!
+  cid: String @index
 }
 
 type Event @entity {
@@ -65,6 +66,7 @@ type Event @entity {
   phase: String!
   pos: Int!
   args: String!
+  cid: String @index
 }
 
 # Accounts Entities

indexers/mainnet/consensus/src/mappings/db.ts

@@ -91,7 +91,8 @@ export async function createAndSaveExtrinsic(
   error: string,
   tip: bigint,
   fee: bigint,
-  pos: number
+  pos: number,
+  cid?: string
 ): Promise<Extrinsic> {
   const extrinsicId = `${blockHeight}-${indexInBlock}`;
   const sortId = getSortId(blockHeight, BigInt(indexInBlock));
@@ -115,6 +116,7 @@ export async function createAndSaveExtrinsic(
     tip,
     fee,
     pos,
+    cid,
   });
   await extrinsic.save();
   return extrinsic;
@@ -131,7 +133,8 @@ export async function createAndSaveEvent(
   timestamp: Date,
   phase: string,
   pos: number,
-  args: string
+  args: string,
+  cid?: string
 ): Promise<Event> {
   const id = `${blockHeight}-${indexInBlock.toString()}`;
   const sortId = getSortId(blockHeight, BigInt(indexInBlock));
@@ -150,6 +153,7 @@ export async function createAndSaveEvent(
     phase,
     pos,
     args,
+    cid,
   });
   await event.save();
   return event;

indexers/mainnet/consensus/src/mappings/helper.ts

@@ -1,5 +1,8 @@
+import { cidOfNode, cidToString, decodeNode } from "@autonomys/auto-dag-data";
+import { stringify } from "@autonomys/auto-utils";
 import { SubstrateBlock } from "@subql/types";
-import { decodeLog } from "./utils";
+import { Cid, ModifiedArgs, ParsedArgs } from "./types";
+import { decodeLog, hexToUint8Array } from "./utils";
 
 const DEFAULT_ACCOUNT_ID = "0x00";
 
@@ -27,3 +30,17 @@ export const getBlockAuthor = (block: SubstrateBlock): string => {
   }
   return DEFAULT_ACCOUNT_ID;
 };
+
+export const parseDataToCid = (data: string): ParsedArgs => {
+  let cid: Cid = undefined;
+  let modifiedArgs: ModifiedArgs = undefined;
+  try {
+    const node = decodeNode(hexToUint8Array(data));
+    cid = cidToString(cidOfNode(node));
+    modifiedArgs = stringify({ cid });
+  } catch (error) {
+    logger.error("Error decoding remark or seedHistory extrinsic");
+    logger.error(error);
+  }
+  return { cid, modifiedArgs };
+};

indexers/mainnet/consensus/src/mappings/mappingHandlers.ts

@@ -1,3 +1,6 @@
+global.TextEncoder = require("util").TextEncoder;
+global.TextDecoder = require("util").TextDecoder;
+
 import { blockchainSize, spacePledge } from "@autonomys/auto-consensus";
 import type { ApiAtBlockHash } from "@autonomys/auto-utils";
 import { stringify } from "@autonomys/auto-utils";
@@ -12,7 +15,7 @@ import {
   createAndSaveExtrinsic,
   createAndSaveLog,
 } from "./db";
-import { getBlockAuthor } from "./helper";
+import { getBlockAuthor, parseDataToCid } from "./helper";
 import {
   handleExtrinsic,
   handleFarmerBlockRewardEvent,
@@ -133,6 +136,22 @@ export async function handleCall(_call: SubstrateExtrinsic): Promise<void> {
       : 0
     : 0;
 
+  // Detect data storage extrinsics and parse args to cid
+  let cid: string | undefined = undefined;
+  let args: string = stringify(methodToPrimitive.args);
+  if (
+    (methodToHuman.section === "historySeeding" &&
+      methodToHuman.method === "seedHistory") ||
+    (methodToHuman.section === "system" &&
+      (methodToHuman.method === "remarkWithEvent" ||
+        methodToHuman.method === "remark"))
+  ) {
+    const parsedArgs = parseDataToCid(methodToPrimitive.args.remark);
+    cid = parsedArgs.cid;
+    // The args parameter will be replaced by `{ "cid": "bafkr6i..." }` to minimize the size of the db
+    args = parsedArgs.modifiedArgs ?? stringify(methodToPrimitive.args);
+  }
+
   await createAndSaveExtrinsic(
     hash.toString(),
     BigInt(number.toString()),
@@ -145,11 +164,12 @@ export async function handleCall(_call: SubstrateExtrinsic): Promise<void> {
     BigInt(nonce.toString()),
     signer.toString(),
     signature.toString(),
-    stringify(methodToPrimitive.args),
+    args,
     error,
     BigInt(tip.toString()),
     fee,
-    pos
+    pos,
+    cid
   );
 
   return await handleExtrinsic(_call);
@@ -182,6 +202,16 @@ export async function handleEvent(_event: SubstrateEvent): Promise<void> {
   const extrinsicId = extrinsic ? number + "-" + extrinsic.idx.toString() : "";
   const extrinsicHash = extrinsic ? extrinsic.extrinsic.hash.toString() : "";
 
+  // Detect data storage extrinsics and parse args to cid
+  let cid: string | undefined = undefined;
+  let args: string = stringify(primitive.data);
+  if (human.section === "system" && human.method === "Remarked") {
+    const parsedArgs = parseDataToCid(primitive.data[1]);
+    cid = parsedArgs.cid;
+    // The args parameter will be replaced by `{ "cid": "bafkr6i..." }` to minimize the size of the db
+    args = parsedArgs.modifiedArgs ?? stringify(primitive.data);
+  }
+
   await createAndSaveEvent(
     BigInt(number.toString()),
     hash.toString(),
@@ -193,7 +223,8 @@ export async function handleEvent(_event: SubstrateEvent): Promise<void> {
     timestamp ? timestamp : new Date(0),
     eventRecord ? eventRecord.phase.type : "",
     pos,
-    stringify(primitive.data)
+    args,
+    cid
   );
 
   switch (`${event.section}.${event.method}`) {

indexers/mainnet/consensus/src/mappings/types.ts

@@ -22,3 +22,11 @@ export type LogValue = {
   data: any;
   engine?: string;
 };
+
+export type Cid = string | undefined;
+export type ModifiedArgs = string | undefined;
+
+export type ParsedArgs = {
+  cid: Cid;
+  modifiedArgs: ModifiedArgs;
+};

indexers/mainnet/consensus/src/mappings/utils.ts

@@ -48,3 +48,11 @@ export const getSortId = (
   const str2 = indexInBlock.toString().padStart(totalLength, "0");
   return `${str1}-${str2}`;
 };
+
+export const hexToUint8Array = (hex: string): Uint8Array => {
+  if (hex.length % 2 !== 0)
+    throw new Error("Hex string must have an even length");
+  return new Uint8Array(
+    hex.match(/.{1,2}/g)?.map((byte) => parseInt(byte, 16)) || []
+  );
+};

indexers/taurus/consensus/package.json

@@ -32,8 +32,9 @@
   "license": "MIT",
   "devDependencies": {
     "@apollo/client": "^3.7.16",
-    "@autonomys/auto-consensus": "^1.0.6",
-    "@autonomys/auto-utils": "^1.0.6",
+    "@autonomys/auto-consensus": "^1.0.8",
+    "@autonomys/auto-dag-data": "^1.0.8",
+    "@autonomys/auto-utils": "^1.0.8",
     "@polkadot/api": "^12.4.2",
     "@polkadot/types": "^14.1.1",
     "@polkadot/util": "^13.2.2",

indexers/taurus/consensus/schema.graphql

@@ -42,12 +42,13 @@ type Extrinsic @entity {
   timestamp: Date! @index
   nonce: BigInt!
   signer: String! @index
-  signature: String! @index
+  signature: String!
   args: String!
   error: String!
   tip: BigInt!
   fee: BigInt!
   pos: Int!
+  cid: String @index
 }
 
 type Event @entity {
@@ -65,6 +66,7 @@ type Event @entity {
   phase: String!
   pos: Int!
   args: String!
+  cid: String @index
 }
 
 # Accounts Entities

indexers/taurus/consensus/src/mappings/db.ts

@@ -91,7 +91,8 @@ export async function createAndSaveExtrinsic(
   error: string,
   tip: bigint,
   fee: bigint,
-  pos: number
+  pos: number,
+  cid?: string
 ): Promise<Extrinsic> {
   const extrinsicId = `${blockHeight}-${indexInBlock}`;
   const sortId = getSortId(blockHeight, BigInt(indexInBlock));
@@ -115,6 +116,7 @@ export async function createAndSaveExtrinsic(
     tip,
     fee,
     pos,
+    cid,
   });
   await extrinsic.save();
   return extrinsic;
@@ -131,7 +133,8 @@ export async function createAndSaveEvent(
   timestamp: Date,
   phase: string,
   pos: number,
-  args: string
+  args: string,
+  cid?: string
 ): Promise<Event> {
   const id = `${blockHeight}-${indexInBlock.toString()}`;
   const sortId = getSortId(blockHeight, BigInt(indexInBlock));
@@ -150,6 +153,7 @@ export async function createAndSaveEvent(
     phase,
     pos,
     args,
+    cid,
   });
   await event.save();
   return event;

indexers/taurus/consensus/src/mappings/helper.ts

@@ -1,5 +1,8 @@
+import { cidOfNode, cidToString, decodeNode } from "@autonomys/auto-dag-data";
+import { stringify } from "@autonomys/auto-utils";
 import { SubstrateBlock } from "@subql/types";
-import { decodeLog } from "./utils";
+import { Cid, ModifiedArgs, ParsedArgs } from "./types";
+import { decodeLog, hexToUint8Array } from "./utils";
 
 const DEFAULT_ACCOUNT_ID = "0x00";
 
@@ -27,3 +30,17 @@ export const getBlockAuthor = (block: SubstrateBlock): string => {
   }
   return DEFAULT_ACCOUNT_ID;
 };
+
+export const parseDataToCid = (data: string): ParsedArgs => {
+  let cid: Cid = undefined;
+  let modifiedArgs: ModifiedArgs = undefined;
+  try {
+    const node = decodeNode(hexToUint8Array(data));
+    cid = cidToString(cidOfNode(node));
+    modifiedArgs = stringify({ cid });
+  } catch (error) {
+    logger.error("Error decoding remark or seedHistory extrinsic");
+    logger.error(error);
+  }
+  return { cid, modifiedArgs };
+};