diff --git a/.env.example b/.env.example index b6d62bcb..da63237e 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,12 @@ PORT=3000 URL=http://127.0.0.1:3000 DATABASE_URI=postgres://postgres:postgres@localhost:5432/postgres -SUBSCAN_NETWORK=spiritnet -SECRET_SUBSCAN= BLOCKCHAIN_ENDPOINT=wss://kilt-rpc.dwellir.com DID= SECRET_PAYER_MNEMONIC= SECRET_AUTHENTICATION_MNEMONIC= SECRET_ASSERTION_METHOD_MNEMONIC= SECRET_KEY_AGREEMENT_MNEMONIC= +GRAPHQL_ENDPOINT=https://indexer.kilt.io/ +# while using the "...kilt.io" endpoints, please add a trailing "/" so that polkadot.js displays the right colors. +POLKADOT_RPC_ENDPOINT=kilt.ibp.network diff --git a/package.json b/package.json index 32b8aebe..27d6f394 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "stop-db": "docker rm -f postgres", "migrate": "sequelize-cli db:migrate", "seed": "sequelize-cli db:seed:all || exit 0", - "build": "export $(cat .env.example) && astro build", + "build": "export $(grep -v '^#' .env.example | xargs) && astro build", "start": "pnpm migrate && pnpm seed && node --enable-source-maps dist/server/entry.mjs" }, "dependencies": { diff --git a/src/components/CTypeDetails/CTypeDetails.astro b/src/components/CTypeDetails/CTypeDetails.astro index 788071d9..0b2541e9 100644 --- a/src/components/CTypeDetails/CTypeDetails.astro +++ b/src/components/CTypeDetails/CTypeDetails.astro @@ -30,11 +30,11 @@ const { title, creator, properties, + block, createdAt, description, - extrinsicHash, schema, - attestationsCount, + attestationsCreated, tags, } = Astro.props.cTypeData; @@ -46,7 +46,7 @@ const schemaV1 = CType.fromProperties('', {}).$schema; const version = schema === schemaV1 ? 'V1' : 'draft-01'; const kiltCType = CType.fromProperties(title, properties, version); -const { subscan, w3nOrigin } = configuration; +const { w3nOrigin, indexer } = configuration; ---
@@ -65,7 +65,7 @@ const { subscan, w3nOrigin } = configuration;
Number of attestations -

{attestationsCount}

+

{attestationsCreated}

{ @@ -96,12 +96,12 @@ const { subscan, w3nOrigin } = configuration; }
- Transaction + Registration Block - {extrinsicHash} + {block}
diff --git a/src/components/CTypeDetails/__snapshots__/CTypeDetails.test.ts.snap b/src/components/CTypeDetails/__snapshots__/CTypeDetails.test.ts.snap index 5deecfdb..a5e9732d 100644 --- a/src/components/CTypeDetails/__snapshots__/CTypeDetails.test.ts.snap +++ b/src/components/CTypeDetails/__snapshots__/CTypeDetails.test.ts.snap @@ -120,8 +120,8 @@ exports[`CTypeDetails > should handle kitchen sink CType 1`] = `
- Transaction - 0xexamplehash + Registration Block + 456
Technical Details @@ -214,8 +214,8 @@ exports[`CTypeDetails > should handle nested CType 1`] = `
- Transaction - 0xexamplehash + Registration Block + 456
Technical Details @@ -280,8 +280,8 @@ exports[`CTypeDetails > should handle nested CType property 1`] = `
- Transaction - 0xexamplenestedhash + Registration Block + 321
Technical Details @@ -344,8 +344,8 @@ exports[`CTypeDetails > should match snapshot 1`] = `
- Transaction - 0xexamplehash + Registration Block + 123
Tags diff --git a/src/components/CTypeOverview/CTypeOverview.astro b/src/components/CTypeOverview/CTypeOverview.astro index 61b03cbb..88e9244a 100644 --- a/src/components/CTypeOverview/CTypeOverview.astro +++ b/src/components/CTypeOverview/CTypeOverview.astro @@ -9,7 +9,7 @@ interface Props { cTypeData: CTypeData; } -const { title, id, description, attestationsCount, tags } = +const { title, id, description, attestationsCreated, tags } = Astro.props.cTypeData; --- @@ -36,7 +36,7 @@ const { title, id, description, attestationsCount, tags } =

Number of Attestations: - {attestationsCount} + {attestationsCreated}

diff --git a/src/components/CreateForm/CreateForm.tsx b/src/components/CreateForm/CreateForm.tsx index 63ddd9eb..6c975159 100644 --- a/src/components/CreateForm/CreateForm.tsx +++ b/src/components/CreateForm/CreateForm.tsx @@ -1,3 +1,11 @@ +import { + connect, + CType, + Did, + disconnect, + type KiltAddress, +} from '@kiltprotocol/sdk-js'; +import { web3FromSource } from '@polkadot/extension-dapp'; import { type FocusEvent, type FormEvent, @@ -6,28 +14,19 @@ import { useCallback, useState, } from 'react'; -import { web3FromSource } from '@polkadot/extension-dapp'; -import { - Blockchain, - connect, - CType, - Did, - disconnect, - type KiltAddress, -} from '@kiltprotocol/sdk-js'; import styles from './CreateForm.module.css'; +import { generatePath, paths } from '../../paths'; +import { getBlockchainEndpoint } from '../../utilities/getBlockchainEndpoint'; +import { offsets } from '../../utilities/offsets'; +import { Modal } from '../Modal/Modal'; +import { PropertyFields } from '../PropertyFields/PropertyFields'; +import { getProperties } from '../PropertyFields/getProperties'; import { type InjectedAccount, SelectAccount, } from '../SelectAccount/SelectAccount'; -import { Modal } from '../Modal/Modal'; -import { PropertyFields } from '../PropertyFields/PropertyFields'; -import { getProperties } from '../PropertyFields/getProperties'; -import { offsets } from '../../utilities/offsets'; -import { getBlockchainEndpoint } from '../../utilities/getBlockchainEndpoint'; -import { generatePath, paths } from '../../paths'; import { useSupportedExtensions } from './useSupportedExtensions'; @@ -139,15 +138,12 @@ export function CreateForm() { const authorizedTx = api.tx(authorized.signed); const injected = await web3FromSource(account.meta.source); - const signed = await authorizedTx.signAsync(account.address, injected); - await Blockchain.submitSignedTx(signed); + await authorizedTx.signAndSend(account.address, injected); - const extrinsicHash = signed.hash.toHex(); const response = await fetch(paths.ctypes, { method: 'POST', body: JSON.stringify({ cType, - extrinsicHash, creator, description, tags, diff --git a/src/components/ctype.test.ts b/src/components/ctype.test.ts index bc22687f..262bb9d5 100644 --- a/src/components/ctype.test.ts +++ b/src/components/ctype.test.ts @@ -12,7 +12,6 @@ describe('endpoint /ctype', () => { const properties = {}; const cType = CType.fromProperties('New CType', properties); const creator = 'did:kilt:4rrkiRTZgsgxjJDFkLsivqqKTqdUTuxKk3FX3mKFAeMxsR5E'; - const extrinsicHash = '0x1234'; const description = 'A CType'; const response = await fetch(endpoint, { @@ -21,7 +20,6 @@ describe('endpoint /ctype', () => { body: JSON.stringify({ cType, creator, - extrinsicHash, description, tags: ['test', 'example'], }), @@ -44,7 +42,7 @@ describe('endpoint /ctype', () => { title: cType.title, properties, creator, - extrinsicHash, + attestationsCreated: 0, description, block: null, isHidden: false, @@ -63,9 +61,8 @@ describe('endpoint /ctype', () => { const properties = {}; const cType = CType.fromProperties('New CType', properties); const creator = 'did:kilt:4rrkiRTZgsgxjJDFkLsivqqKTqdUTuxKk3FX3mKFAeMxsR5E'; - const extrinsicHash = '0x1234'; const tags = [] as string[]; - const body = JSON.stringify({ cType, creator, extrinsicHash, tags }); + const body = JSON.stringify({ cType, creator, tags }); const created = await fetch(endpoint, { method, headers, body }); expect(created.status).toBe(StatusCodes.CREATED); @@ -78,9 +75,8 @@ describe('endpoint /ctype', () => { const properties = {}; const cType = CType.fromProperties('New CType', properties); const creator = 'invalid'; - const extrinsicHash = '0x1234'; const tags = [] as string[]; - const body = JSON.stringify({ cType, creator, extrinsicHash, tags }); + const body = JSON.stringify({ cType, creator, tags }); const response = await fetch(endpoint, { method, headers, body }); expect(response.status).toBe(StatusCodes.UNPROCESSABLE_ENTITY); @@ -89,9 +85,8 @@ describe('endpoint /ctype', () => { it('should return an error if the CType is not valid', async () => { const cType = { invalid: 'CType' }; const creator = 'did:kilt:4rrkiRTZgsgxjJDFkLsivqqKTqdUTuxKk3FX3mKFAeMxsR5E'; - const extrinsicHash = '0x1234'; const tags = [] as string[]; - const body = JSON.stringify({ cType, creator, extrinsicHash, tags }); + const body = JSON.stringify({ cType, creator, tags }); const response = await fetch(endpoint, { method, headers, body }); expect(response.status).toBe(StatusCodes.UNPROCESSABLE_ENTITY); diff --git a/src/components/index/__snapshots__/index.test.ts.snap b/src/components/index/__snapshots__/index.test.ts.snap index baadab17..4269aef5 100644 --- a/src/components/index/__snapshots__/index.test.ts.snap +++ b/src/components/index/__snapshots__/index.test.ts.snap @@ -22,7 +22,7 @@ exports[`index.astro > should render all CTypes except hidden ones 1`] = ` This is an example of a CType with a nested property

- Number of Attestations:2 + Number of Attestations:22

  • @@ -77,7 +77,7 @@ exports[`index.astro > should render search results 1`] = ` This is an example of a CType with a nested property

    - Number of Attestations:2 + Number of Attestations:22

    • @@ -124,7 +124,7 @@ exports[`index.astro > should render tag results 1`] = ` This is an example of a CType with a nested property

    - Number of Attestations:2 + Number of Attestations:22

  • diff --git a/src/components/index/index.test.ts b/src/components/index/index.test.ts index 9415ce84..2979f556 100644 --- a/src/components/index/index.test.ts +++ b/src/components/index/index.test.ts @@ -2,9 +2,7 @@ import { beforeEach, describe, expect, it } from 'vitest'; import { getSnapshotHtmlForPath } from '../../../testing/getSnapshotHtmlForPath'; import { CType as CTypeModel } from '../../models/ctype'; -import { Attestation as AttestationModel } from '../../models/attestation'; import { mockCTypes } from '../../utilities/mockCTypes'; -import { mockAttestations } from '../../utilities/mockAttestations'; import { initializeDatabase } from '../../utilities/sequelize'; import { resetDatabase } from '../../../testing/resetDatabase'; import { Tag } from '../../models/tag'; @@ -18,7 +16,6 @@ beforeEach(async () => { mockCTypes.hidden, ]); await Tag.create({ tagName: 'example', cTypeId: mockCTypes.example.id }); - await AttestationModel.bulkCreate(mockAttestations); }); describe('index.astro', () => { diff --git a/src/migrations/20240807132131-dropAttestations.cjs b/src/migrations/20240807132131-dropAttestations.cjs new file mode 100644 index 00000000..73c6534e --- /dev/null +++ b/src/migrations/20240807132131-dropAttestations.cjs @@ -0,0 +1,68 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface) { + /** + * Reverts (combines `down` functions of) following migrations: + * - `20230712145719-foreignKeys.cjs` + * - `20230711125237-attestations.cjs` + */ + await queryInterface.removeConstraint( + 'Attestations', + 'Attestations_cTypeId_fkey', + ); + + await queryInterface.dropTable('Attestations'); + }, + + async down(queryInterface, Sequelize) { + const { sequelize } = queryInterface; + + /** + * Recreates (combines `up` functions of) following migrations: + * - `20230712145719-foreignKeys.cjs` + * - `20230711125237-attestations.cjs` + */ + sequelize.define( + 'Attestation', + { + claimHash: { + type: Sequelize.STRING, + primaryKey: true, + }, + cTypeId: { + type: Sequelize.STRING, + allowNull: false, + }, + owner: { + type: Sequelize.STRING, + allowNull: false, + }, + delegationId: { + type: Sequelize.STRING, + }, + createdAt: { + type: Sequelize.DATE, + allowNull: false, + }, + extrinsicHash: { + type: Sequelize.STRING, + allowNull: false, + }, + block: { + type: Sequelize.STRING, + }, + }, + { indexes: [{ fields: ['cTypeId'] }] }, + ); + await sequelize.sync(); + + await queryInterface.addConstraint('Attestations', { + type: 'foreign key', + name: 'Attestations_cTypeId_fkey', + fields: ['cTypeId'], + references: { table: 'CTypes', field: 'id' }, + }); + }, +}; diff --git a/src/migrations/20240807162056-removeExtrinsicHash.cjs b/src/migrations/20240807162056-removeExtrinsicHash.cjs new file mode 100644 index 00000000..58abfc5c --- /dev/null +++ b/src/migrations/20240807162056-removeExtrinsicHash.cjs @@ -0,0 +1,54 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface) { + // First, drop the 'search' column since it depends on 'extrinsicHash' + await queryInterface.removeColumn('CTypes', 'search'); + + // safely drop the 'extrinsicHash' column + await queryInterface.removeColumn('CTypes', 'extrinsicHash'); + + // recreate the 'search' column without 'extrinsicHash' + await queryInterface.addColumn( + 'CTypes', + 'search', + `tsvector generated always as (to_tsvector('english', + coalesce("id"::text, '') || ' ' || + coalesce("schema"::text, '') || ' ' || + coalesce("title"::text, '') || ' ' || + coalesce("properties"::text, '') || ' ' || + coalesce("type"::text, '') || ' ' || + coalesce("creator"::text, '') || ' ' || + coalesce("block"::text, '') || ' ' || + coalesce("description"::text, '')) + ) stored`, + ); + }, + + async down(queryInterface, Sequelize) { + // First, drop the 'search' column + await queryInterface.removeColumn('CTypes', 'search'); + // Recreate the 'extrinsicHash' column + await queryInterface.addColumn('CTypes', 'extrinsicHash', { + type: Sequelize.STRING, + allowNull: false, + defaultValue: '0x1ABE11ABEBECAFE501A', + }); + await queryInterface.addColumn( + 'CTypes', + 'search', + `tsvector generated always as (to_tsvector('english', + coalesce("id"::text, '') || ' ' || + coalesce("schema"::text, '') || ' ' || + coalesce("title"::text, '') || ' ' || + coalesce("properties"::text, '') || ' ' || + coalesce("type"::text, '') || ' ' || + coalesce("creator"::text, '') || ' ' || + coalesce("extrinsicHash"::text, '') || ' ' || + coalesce("block"::text, '') || ' ' || + coalesce("description"::text, '')) + ) stored`, + ); + }, +}; diff --git a/src/migrations/20240808141601-attestationsCreatedField.cjs b/src/migrations/20240808141601-attestationsCreatedField.cjs new file mode 100644 index 00000000..11e440f8 --- /dev/null +++ b/src/migrations/20240808141601-attestationsCreatedField.cjs @@ -0,0 +1,16 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('CTypes', 'attestationsCreated', { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0, + }); + }, + + async down(queryInterface) { + await queryInterface.removeColumn('CTypes', 'attestationsCreated'); + }, +}; diff --git a/src/models/attestation.ts b/src/models/attestation.ts deleted file mode 100644 index a767d616..00000000 --- a/src/models/attestation.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { type IAttestation, type ICType } from '@kiltprotocol/sdk-js'; -import { - DataTypes, - Model, - type ModelAttributes, - type Sequelize, -} from 'sequelize'; - -export interface AttestationData - extends Omit { - cTypeId: ICType['$id']; - createdAt: Date; - extrinsicHash: string; - block: string; -} - -export class Attestation extends Model {} - -export const AttestationModelDefinition: ModelAttributes = { - claimHash: { - type: DataTypes.STRING, - primaryKey: true, - }, - cTypeId: { - type: DataTypes.STRING, - allowNull: false, - }, - owner: { - type: DataTypes.STRING, - allowNull: false, - }, - delegationId: { - type: DataTypes.STRING, - }, - createdAt: { - type: DataTypes.DATE, - allowNull: false, - }, - extrinsicHash: { - type: DataTypes.STRING, - allowNull: false, - }, - block: { - type: DataTypes.STRING, - }, -}; - -export function initAttestation(sequelize: Sequelize) { - Attestation.init(AttestationModelDefinition, { - sequelize, - indexes: [{ fields: ['cTypeId'] }], - }); -} diff --git a/src/models/ctype.ts b/src/models/ctype.ts index df873c48..3c57227a 100644 --- a/src/models/ctype.ts +++ b/src/models/ctype.ts @@ -1,7 +1,11 @@ import { type DidUri, type ICType } from '@kiltprotocol/sdk-js'; -import { DataTypes, Model, type ModelAttributes, Sequelize } from 'sequelize'; +import { + DataTypes, + Model, + type ModelAttributes, + type Sequelize, +} from 'sequelize'; -import { Attestation } from './attestation'; import { Tag } from './tag'; interface CTypeDataInput extends Omit { @@ -9,13 +13,12 @@ interface CTypeDataInput extends Omit { schema: ICType['$schema']; creator: DidUri; createdAt: Date; - extrinsicHash: string; block: string | null; description: string | null; + attestationsCreated?: number; } export interface CTypeData extends CTypeDataInput { - attestationsCount: string; isHidden: boolean; tags?: Array>; } @@ -51,10 +54,6 @@ export const CTypeModelDefinition: ModelAttributes = { type: DataTypes.DATE, allowNull: false, }, - extrinsicHash: { - type: DataTypes.STRING, - allowNull: false, - }, block: { type: DataTypes.STRING, }, @@ -66,10 +65,15 @@ export const CTypeModelDefinition: ModelAttributes = { allowNull: false, defaultValue: false, }, + attestationsCreated: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0, + }, }; const fields = Object.keys(CTypeModelDefinition) - .filter((name) => name !== 'createdAt') + .filter((name) => !['createdAt', 'isHidden'].includes(name)) .map((name) => `coalesce("${name}"::text, '')`) .join(` || ' ' || `); @@ -91,25 +95,11 @@ export function initCType(sequelize: Sequelize) { ...Object.keys(CTypeModelDefinition).filter( (key) => key !== 'search', ), - [ - Sequelize.literal( - `coalesce( - (select count(*) - from "Attestations" - where "Attestations"."cTypeId" = "CType"."id" - group by "CType"."id"), - 0)`, - ), - 'attestationsCount', - ], ], }, }, }); - CType.hasMany(Attestation, { foreignKey: 'cTypeId' }); - Attestation.belongsTo(CType, { foreignKey: 'cTypeId' }); - CType.hasMany(Tag, { foreignKey: 'cTypeId', as: 'tags' }); Tag.belongsTo(CType, { foreignKey: 'cTypeId' }); } diff --git a/src/pages/ctype.ts b/src/pages/ctype.ts index 806bb87d..7c38ef79 100644 --- a/src/pages/ctype.ts +++ b/src/pages/ctype.ts @@ -1,5 +1,5 @@ import { type APIContext } from 'astro'; -import { CType, Did, type DidUri, type HexString } from '@kiltprotocol/sdk-js'; +import { CType, Did, type DidUri } from '@kiltprotocol/sdk-js'; import { StatusCodes } from 'http-status-codes'; import { CType as CTypeModel } from '../models/ctype'; @@ -14,14 +14,13 @@ import { getRequestJson } from '../utilities/getRequestJson'; interface Input { cType: unknown; creator: DidUri; - extrinsicHash: HexString; description: string; tags: string[]; } export async function POST({ request, url }: APIContext) { try { - const { cType, creator, extrinsicHash, description, tags } = + const { cType, creator, description, tags } = await getRequestJson(request); if (!CType.isICType(cType)) { @@ -52,7 +51,6 @@ export async function POST({ request, url }: APIContext) { id, schema, creator, - extrinsicHash, description, createdAt: new Date(), ...rest, diff --git a/src/pages/tag/[tag].astro b/src/pages/tag/[tag].astro index dc2ec855..732d7973 100644 --- a/src/pages/tag/[tag].astro +++ b/src/pages/tag/[tag].astro @@ -28,6 +28,7 @@ const query = Astro.url.searchParams.get('query'); const where = { block: { [Op.ne]: null }, + isHidden: false, ...(query && { search: { [Op.match]: fn('websearch_to_tsquery', 'english', query) }, }), diff --git a/src/utilities/configuration.ts b/src/utilities/configuration.ts index a3830391..4275cb31 100644 --- a/src/utilities/configuration.ts +++ b/src/utilities/configuration.ts @@ -12,18 +12,18 @@ class ConfigurationError extends Error { process.exit(1); } } - -const subscan = { - network: import.meta.env.SUBSCAN_NETWORK as string, - secret: import.meta.env.SECRET_SUBSCAN as string, +const indexer = { + graphqlEndpoint: import.meta.env.GRAPHQL_ENDPOINT as string, + polkadotRPCEndpoint: import.meta.env.POLKADOT_RPC_ENDPOINT as string, }; -if (!subscan.network) { - throw new ConfigurationError('No subscan network provided'); +if (!indexer.graphqlEndpoint) { + throw new ConfigurationError('No endpoint for the GraphQL server provided'); } -if (!subscan.secret) { - throw new ConfigurationError('No subscan secret provided'); +if (!indexer.polkadotRPCEndpoint) { + throw new ConfigurationError( + 'No R.P.C. endpoint for the polkadot.js explorer provided', + ); } - const blockchainEndpoint = import.meta.env.BLOCKCHAIN_ENDPOINT as string; if (!blockchainEndpoint) { throw new ConfigurationError('No blockchain endpoint provided'); @@ -70,7 +70,6 @@ export const configuration = { databaseUri: (import.meta.env.DATABASE_URI as string) || 'postgres://postgres:postgres@localhost:5432/postgres', - subscan, blockchainEndpoint, did, authenticationMnemonic, @@ -78,4 +77,5 @@ export const configuration = { keyAgreementMnemonic, payerMnemonic, w3nOrigin: w3nOrigins[blockchainEndpoint] || 'https://w3n.id', + indexer, }; diff --git a/src/utilities/indexer/fragments.ts b/src/utilities/indexer/fragments.ts new file mode 100644 index 00000000..2d71f2dd --- /dev/null +++ b/src/utilities/indexer/fragments.ts @@ -0,0 +1,18 @@ +// GraphQL provides reusable units called fragments. +// Fragments let you construct sets of fields, and then include them in queries where needed. +// You can use the fragments by including them as fields prefixed by 3 points "...", like shown on 'wholeAttestation'. +// See documentation here: https://graphql.org/learn/queries/#fragments +// Try out yourself under https://indexer.kilt.io/ & https://dev-indexer.kilt.io/ + +export const wholeBlock = ` +fragment wholeBlock on Block { + id + hash + timeStamp +}`; + +export const DidNames = ` +fragment DidNames on Did { + id + web3NameId +}`; diff --git a/src/utilities/indexer/queryCTypes.ts b/src/utilities/indexer/queryCTypes.ts new file mode 100644 index 00000000..ab8c0da8 --- /dev/null +++ b/src/utilities/indexer/queryCTypes.ts @@ -0,0 +1,93 @@ +import { type DidUri, type HexString, type ICType } from '@kiltprotocol/sdk-js'; + +import { Op } from 'sequelize'; + +import { CType as CTypeModel } from '../../models/ctype'; + +import { logger } from '../logger'; + +import { matchesGenerator, QUERY_SIZE } from './queryFromIndexer'; +import { DidNames, wholeBlock } from './fragments'; + +// When modifying queries, first try them out on https://indexer.kilt.io/ or https://dev-indexer.kilt.io/ + +function buildCTypeQueries(fromDate: Date) { + return (offset: number) => ` + query { + cTypes(orderBy: ID_ASC, first: ${QUERY_SIZE}, offset: ${offset}, filter: { registrationBlock: { timeStamp: { greaterThan: "${fromDate.toISOString()}" } }}) { + totalCount + nodes { + id + author {...DidNames} + registrationBlock {...wholeBlock} + attestationsCreated + attestationsRevoked + attestationsRemoved + validAttestations + definition + } + } + } + ${wholeBlock} + ${DidNames} +`; +} + +/** Expected structure of responses for queries defined above. */ +interface QueriedCType { + id: ICType['$id']; + attestationsCreated: number; + author: { + id: DidUri; + web3NameId: string; + }; + registrationBlock: { + id: string; // Block Ordinal Number, without punctuation + hash: HexString; + timeStamp: string; // ISO8601 Date String, like 2022-02-09T13:09:18.217 + }; + definition: string; // stringified JSON of cType Schema +} + +export async function queryCTypes() { + const latestCType = await CTypeModel.findOne({ + order: [['createdAt', 'DESC']], + where: { + block: { + [Op.not]: null, + }, + }, + }); + + const fromDate = latestCType ? latestCType.dataValues.createdAt : new Date(0); + + const entitiesGenerator = matchesGenerator( + buildCTypeQueries(fromDate), + ); + + for await (const entity of entitiesGenerator) { + const { + id: cTypeId, + author, + registrationBlock, + definition, + attestationsCreated, + } = entity; + + const { id: creator } = author; + const { $schema, ...rest } = JSON.parse(definition) as Omit; + + const newCType = await CTypeModel.upsert({ + id: cTypeId, + schema: $schema, + createdAt: new Date(registrationBlock.timeStamp + 'Z'), + creator, + block: registrationBlock.id, + ...rest, + attestationsCreated, + }); + logger.info( + `Added new CType to data base: ${JSON.stringify(newCType, null, 2)}`, + ); + } +} diff --git a/src/utilities/indexer/queryFromIndexer.ts b/src/utilities/indexer/queryFromIndexer.ts new file mode 100644 index 00000000..9c2e7615 --- /dev/null +++ b/src/utilities/indexer/queryFromIndexer.ts @@ -0,0 +1,110 @@ +import { got } from 'got'; + +import { configuration } from '../configuration'; +import { logger } from '../logger'; +import { sleep } from '../sleep'; + +const { indexer } = configuration; + +const QUERY_INTERVAL_MS = 1000; +export const QUERY_SIZE = 50; + +// /** Example Query. */ +// const queryBlocks = ` +// query { +// blocks(orderBy: TIME_STAMP_DESC, first: 3) { +// totalCount +// nodes { +// id +// timeStamp +// hash +// } +// } +// } +// `; + +interface FetchedData { + data: Record< + string, + { + totalCount?: number; + nodes?: Array>; + } + >; +} + +export async function queryFromIndexer(query: string) { + logger.debug( + `Querying from GraphQL under ${indexer.graphqlEndpoint}, using this payload: ${query} `, + ); + const { data } = await got + .post(indexer.graphqlEndpoint, { + json: { + query, + }, + }) + .json(); + + const entities = Object.entries(data); + + const [name, { totalCount, nodes: matches }] = entities[0]; + + if (entities.length > 1) { + logger.error( + `Please, avoid multiple queries in a single request. Processing just '${name}' from here.`, + ); + } + + if (totalCount === undefined) { + throw new Error( + 'The query did not ask for total count. Please add field "totalCount" to your query.', + ); + } + if (matches === undefined) { + throw new Error( + 'You need to include "nodes" as a field (with subfields) on your query to get matches.', + ); + } + logger.info( + `Completed querying '${name}' from GraphQL under ${indexer.graphqlEndpoint}.`, + ); + + logger.info( + `Got ${matches.length} out of ${totalCount} '${name}' matching query.`, + ); + + return { totalCount, matches }; +} + +export async function* matchesGenerator( + buildQuery: (offset: number) => string, +): AsyncGenerator { + if (indexer.graphqlEndpoint === 'NONE') { + return; + } + const query = buildQuery(0); + const { totalCount, matches } = await queryFromIndexer(query); + + if (totalCount === 0) { + logger.debug( + `The Indexed Data under "${indexer.graphqlEndpoint}" has no matches for query: ${query}.`, + ); + return; + } + + if (totalCount === matches.length) { + for (const match of matches) { + yield match as ExpectedQueryResults; + } + return; + } + + for (let offset = 0; offset < totalCount; offset += QUERY_SIZE) { + const { matches } = await queryFromIndexer(buildQuery(offset)); + + for (const match of matches) { + yield match as ExpectedQueryResults; + } + await sleep(QUERY_INTERVAL_MS); + } +} diff --git a/src/utilities/indexer/updateAttestationsCreated.ts b/src/utilities/indexer/updateAttestationsCreated.ts new file mode 100644 index 00000000..b3f745eb --- /dev/null +++ b/src/utilities/indexer/updateAttestationsCreated.ts @@ -0,0 +1,64 @@ +import { type ICType } from '@kiltprotocol/sdk-js'; + +import { Op } from 'sequelize'; + +import { CType as CTypeModel } from '../../models/ctype'; + +import { logger } from '../logger'; + +import { matchesGenerator, QUERY_SIZE } from './queryFromIndexer'; + +// When modifying queries, first try them out on https://indexer.kilt.io/ or https://dev-indexer.kilt.io/ + +function buildQueriesForAttestationsCreated() { + return (offset: number) => ` + query { + attestationsCreated: cTypes(orderBy: ID_ASC, first: ${QUERY_SIZE}, offset: ${offset}) { + totalCount + nodes { + cTypeId: id + attestationsCreated + registrationBlockId + } + } + } + `; +} + +/** Expected structure of responses for queries defined above. */ +interface QueriedAttestationsCreated { + cTypeId: ICType['$id']; + attestationsCreated: number; + registrationBlockId: string; // Block Ordinal Number, without punctuation +} + +export async function updateAttestationsCreated() { + const entitiesGenerator = matchesGenerator( + buildQueriesForAttestationsCreated(), + ); + + for await (const entity of entitiesGenerator) { + const { cTypeId, attestationsCreated } = entity; + + const cTypeToUpdate = await CTypeModel.findOne({ + where: { + id: { + [Op.eq]: cTypeId, + }, + attestationsCreated: { + [Op.ne]: attestationsCreated, + }, + }, + }); + + if (!cTypeToUpdate) { + continue; + } + + logger.info( + `Updating Attestation Count of cType "${cTypeToUpdate.getDataValue('id')}" from ${cTypeToUpdate.getDataValue('attestationsCreated')} to ${attestationsCreated}`, + ); + cTypeToUpdate.set('attestationsCreated', attestationsCreated); + await cTypeToUpdate.save(); + } +} diff --git a/src/utilities/indexer/watchIndexer.ts b/src/utilities/indexer/watchIndexer.ts new file mode 100644 index 00000000..ed38140d --- /dev/null +++ b/src/utilities/indexer/watchIndexer.ts @@ -0,0 +1,22 @@ +import { configuration } from '../configuration'; +import { sleep } from '../sleep'; + +import { queryCTypes } from './queryCTypes'; +import { updateAttestationsCreated } from './updateAttestationsCreated'; + +const { isTest } = configuration; + +const SCAN_INTERVAL_MS = 10 * 60 * 1000; + +export function watchIndexer() { + if (isTest) { + return; + } + (async () => { + while (true) { + await queryCTypes(); + await updateAttestationsCreated(); + await sleep(SCAN_INTERVAL_MS); + } + })(); +} diff --git a/src/utilities/initialize.ts b/src/utilities/initialize.ts index 22a69eb6..e22df823 100644 --- a/src/utilities/initialize.ts +++ b/src/utilities/initialize.ts @@ -1,11 +1,11 @@ import { initializeDatabase, trackDatabaseConnection } from './sequelize'; import { initKilt } from './initKilt'; -import { watchSubScan } from './watchSubScan'; +import { watchIndexer } from './indexer/watchIndexer'; export async function initialize() { await initializeDatabase(); await initKilt(); trackDatabaseConnection(); - watchSubScan(); + watchIndexer(); } diff --git a/src/utilities/mockAttestations.ts b/src/utilities/mockAttestations.ts deleted file mode 100644 index aa7f4637..00000000 --- a/src/utilities/mockAttestations.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { AttestationData } from '../models/attestation'; - -export const mockAttestations: AttestationData[] = [ - { - claimHash: '0x1', - cTypeId: 'kilt:ctype:0x1', - owner: 'did:kilt:4pehddkhEanexVTTzWAtrrfo2R7xPnePpuiJLC7shQU894aY', - delegationId: null, - createdAt: new Date('2023-06-01T12:00:00'), - extrinsicHash: '0xextrinsichash', - block: '321', - }, - { - claimHash: '0x2', - cTypeId: 'kilt:ctype:0x2', - owner: 'did:kilt:4pehddkhEanexVTTzWAtrrfo2R7xPnePpuiJLC7shQU894aY', - delegationId: null, - createdAt: new Date('2023-07-01T12:00:00'), - extrinsicHash: '0xextrinsichash', - block: '456', - }, - { - claimHash: '0x3', - cTypeId: 'kilt:ctype:0x2', - owner: 'did:kilt:4pehddkhEanexVTTzWAtrrfo2R7xPnePpuiJLC7shQU894aY', - delegationId: null, - createdAt: new Date('2023-07-01T12:00:00'), - extrinsicHash: '0xextrinsichash', - block: '456', - }, -]; diff --git a/src/utilities/mockCTypes.ts b/src/utilities/mockCTypes.ts index a6431d8e..ef2c6935 100644 --- a/src/utilities/mockCTypes.ts +++ b/src/utilities/mockCTypes.ts @@ -9,10 +9,9 @@ export const mockCTypes: Record = { type: 'object', creator: 'did:kilt:4pehddkhEanexVTTzWAtrrfo2R7xPnePpuiJLC7shQU894aY', createdAt: new Date('2023-05-01T12:00:00'), - extrinsicHash: '0xexamplehash', description: 'This is some example cType data', block: '123', - attestationsCount: '1', + attestationsCreated: 1, tags: [ { dataValues: { @@ -35,10 +34,9 @@ export const mockCTypes: Record = { type: 'object', creator: 'did:kilt:4pehddkhEanexVTTzWAtrrfo2R7xPnePpuiJLC7shQU894aY', createdAt: new Date('2023-05-01T12:01:00'), - extrinsicHash: '0xexamplenestedhash', description: 'This is an example of a CType with a nested property', block: '321', - attestationsCount: '22', + attestationsCreated: 22, isHidden: false, }, nestedCType: { @@ -56,10 +54,9 @@ export const mockCTypes: Record = { type: 'object', creator: 'did:kilt:4pehddkhEanexVTTzWAtrrfo2R7xPnePpuiJLC7shQU894aY', createdAt: new Date('2023-05-01T12:02:00'), - extrinsicHash: '0xexamplehash', description: 'This is an example of a CType with a nested CType', block: '456', - attestationsCount: '333', + attestationsCreated: 333, isHidden: false, }, hidden: { @@ -70,10 +67,9 @@ export const mockCTypes: Record = { type: 'object', creator: 'did:kilt:4pehddkhEanexVTTzWAtrrfo2R7xPnePpuiJLC7shQU894aY', createdAt: new Date('2023-05-01T12:00:00'), - extrinsicHash: '0xexamplehash', description: 'This is some example cType data', block: '123', - attestationsCount: '1', + attestationsCreated: 1, tags: [ { dataValues: { @@ -123,8 +119,7 @@ export const mockCTypes: Record = { }, type: 'object', creator: 'did:kilt:4rrkiRTZgsgxjJDFkLsivqqKTqdUTuxKk3FX3mKFAeMxsR5E', - attestationsCount: '4444', - extrinsicHash: '0xexamplehash', + attestationsCreated: 4444, createdAt: new Date('2023-05-01T12:03:00'), block: '456', isHidden: false, diff --git a/src/utilities/paginate.test.ts b/src/utilities/paginate.test.ts index 1b14cf25..d12c425b 100644 --- a/src/utilities/paginate.test.ts +++ b/src/utilities/paginate.test.ts @@ -14,7 +14,6 @@ vi.mock('../models/ctype', () => ({ findAll: vi.fn(), count: vi.fn(), }, - groupForAttestationsCount: [], })); const baseUrl = process.env.URL as string; diff --git a/src/utilities/scanAttestations.test.ts b/src/utilities/scanAttestations.test.ts deleted file mode 100644 index 317607f4..00000000 --- a/src/utilities/scanAttestations.test.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - Attestation, - Blockchain, - Claim, - ConfigService, - connect, - Credential, - CType, - Did, - type DidUri, - disconnect, - type IAttestation, - type ICType, - type KiltKeyringPair, - type SubmittableExtrinsic, - Utils, -} from '@kiltprotocol/sdk-js'; - -import { CType as CTypeModel } from '../models/ctype'; -import { Attestation as AttestationModel } from '../models/attestation'; -import { endowAccount } from '../../testing/endowAccount'; -import { createDid } from '../../testing/createDid'; -import { createCType } from '../../testing/createCType'; -import { resetDatabase } from '../../testing/resetDatabase'; - -import { configuration } from './configuration'; -import { subScanEventGenerator } from './subScan'; -import { scanAttestations } from './scanAttestations'; - -vi.mock('./subScan'); - -let submitter: KiltKeyringPair; -let did: DidUri; - -let cType: ICType; -let extrinsic: SubmittableExtrinsic; -let attestation: IAttestation; - -async function createAttestation(assertionMethod: KiltKeyringPair) { - const claim = Claim.fromCTypeAndClaimContents( - cType, - { Email: 'test@example.com' }, - did, - ); - const credential = Credential.fromClaim(claim); - const attestation = Attestation.fromCredentialAndDid(credential, did); - - const api = ConfigService.get('api'); - const tx = api.tx.attestation.add( - attestation.claimHash, - attestation.cTypeHash, - null, - ); - - const extrinsic = await Did.authorizeTx( - did, - tx, - async ({ data }) => ({ - signature: assertionMethod.sign(data, { withType: false }), - keyType: assertionMethod.type, - }), - submitter.address, - ); - await Blockchain.signAndSubmitTx(extrinsic, submitter); - - return { attestation, extrinsic }; -} - -function mockAttestationEvent() { - const mockParams = [ - { - type_name: 'AttesterOf', - value: Utils.Crypto.u8aToHex( - Utils.Crypto.decodeAddress(Did.toChain(did)), - ), - }, - { type_name: 'ClaimHashOf', value: attestation.claimHash }, - { type_name: 'CtypeHashOf', value: CType.idToHash(cType.$id) }, - { type_name: 'Option', value: null }, - ]; - const mockParsedParams = { - AttesterOf: Utils.Crypto.u8aToHex( - Utils.Crypto.decodeAddress(Did.toChain(did)), - ), - ClaimHashOf: attestation.claimHash, - CtypeHashOf: CType.idToHash(cType.$id), - 'Option': null, - }; - - vi.mocked(subScanEventGenerator).mockImplementation(async function* () { - yield { - params: mockParams, - parsedParams: mockParsedParams, - block: 123456, - blockTimestampMs: 160273245600, - extrinsicHash: extrinsic.hash.toHex(), - }; - }); -} - -beforeAll(async () => { - await connect(configuration.blockchainEndpoint); - - submitter = Utils.Crypto.makeKeypairFromSeed(undefined, 'sr25519'); - await endowAccount(submitter.address); - - const created = await createDid(); - did = created.did; - - cType = (await createCType(did, created.assertionMethod, submitter)).cType; - - ({ attestation, extrinsic } = await createAttestation( - created.assertionMethod, - )); - - const { sequelize, initializeDatabase } = await import('./sequelize'); - await initializeDatabase(); - - return async function teardown() { - await sequelize.close(); - await disconnect(); - // give the SDK time to log the disconnect message - await new Promise((resolve) => setTimeout(resolve, 1000)); - }; -}, 30_000); - -beforeEach(async () => { - await resetDatabase(); - - const { $id, $schema, title, properties, type } = cType; - await CTypeModel.create({ - id: $id, - schema: $schema, - title, - properties, - type, - creator: did, - createdAt: new Date(), - extrinsicHash: '0xexampleextrinsichash', - }); - - vi.mocked(subScanEventGenerator).mockClear(); -}); - -describe('scanAttestations', () => { - it('should add new attestation to the database', async () => { - mockAttestationEvent(); - - await scanAttestations(); - - expect(subScanEventGenerator).toHaveBeenLastCalledWith( - 'attestation', - 'AttestationCreated', - 0, - ); - const created = await AttestationModel.findByPk(attestation.claimHash); - expect(created).not.toBeNull(); - expect(created?.dataValues.cTypeId).toBe(cType.$id); - }); - - it('should not add a CType if it already exists', async () => { - mockAttestationEvent(); - - await scanAttestations(); - - const latestAttestation = await AttestationModel.findOne({ - order: [['createdAt', 'DESC']], - }); - - const expectedFromBlock = Number(latestAttestation?.dataValues.block); - - vi.mocked(subScanEventGenerator).mockImplementation(async function* () { - // yield nothing - }); - - await scanAttestations(); - - expect(subScanEventGenerator).toHaveBeenCalledTimes(2); - expect(subScanEventGenerator).toHaveBeenLastCalledWith( - 'attestation', - 'AttestationCreated', - expectedFromBlock, - ); - - const count = await AttestationModel.count(); - expect(count).toBe(1); - }); - - it('should correctly upsert an existing CType', async () => { - const { claimHash, cTypeHash, owner, delegationId } = attestation; - const cTypeId = CType.hashToId(cTypeHash); - - await AttestationModel.create({ - claimHash, - cTypeId, - owner, - delegationId, - block: '0', - extrinsicHash: extrinsic.hash.toHex(), - createdAt: new Date(), - }); - - const beforeUpsert = await AttestationModel.findByPk(claimHash); - - if (!beforeUpsert) { - throw new Error('Not found'); - } - - expect(beforeUpsert.dataValues.block).toBe('0'); - - mockAttestationEvent(); - await scanAttestations(); - - await beforeUpsert.reload(); - expect(beforeUpsert.dataValues.block).not.toBeNull(); - }); -}); diff --git a/src/utilities/scanAttestations.ts b/src/utilities/scanAttestations.ts deleted file mode 100644 index 6c27179e..00000000 --- a/src/utilities/scanAttestations.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { CType, Did, type HexString } from '@kiltprotocol/sdk-js'; -import { hexToU8a } from '@polkadot/util'; - -import { Attestation as AttestationModel } from '../models/attestation'; - -import { subScanEventGenerator } from './subScan'; -import { logger } from './logger'; - -function getDidUriFromAccountHex(didAccount: HexString) { - logger.debug('DID as HexString of Account Address: ' + didAccount); - // SubScan returns some AttesterOf values as hex without the "0x" prefix. - // So we first parsed to a Uint8Array via `hexToU8a`, which can handle HexStrings with or without the prefix. - const didU8a = hexToU8a(didAccount); - - const didUri = Did.fromChain(didU8a as Parameters[0]); - logger.debug('Corresponding DID-URI: ' + didUri); - return didUri; -} - -export async function scanAttestations() { - const latestAttestation = await AttestationModel.findOne({ - order: [['createdAt', 'DESC']], - }); - const fromBlock = latestAttestation - ? Number(latestAttestation.dataValues.block) - : 0; - - const eventGenerator = subScanEventGenerator( - 'attestation', - 'AttestationCreated', - fromBlock, - ); - - for await (const event of eventGenerator) { - const { block, blockTimestampMs, extrinsicHash } = event; - const params = event.parsedParams; - - const createdAt = new Date(blockTimestampMs); - const owner = getDidUriFromAccountHex(params.AttesterOf as HexString); - const claimHash = params.ClaimHashOf as HexString; - const cTypeHash = params.CtypeHashOf as HexString; - const cTypeId = CType.hashToId(cTypeHash); - const delegationId = params[ - 'Option' - ] as HexString | null; - - try { - await AttestationModel.upsert({ - claimHash, - cTypeId, - owner, - delegationId, - createdAt, - extrinsicHash, - block: String(block), - }); - } catch (exception) { - if ((exception as Error).name === 'SequelizeForeignKeyConstraintError') { - // Likely a broken CType which we haven’t saved to the database - logger.debug(`Ignoring attestation ${claimHash} for unknown CType`); - continue; - } - throw exception; - } - } -} diff --git a/src/utilities/scanCTypes.test.ts b/src/utilities/scanCTypes.test.ts deleted file mode 100644 index 98cc136d..00000000 --- a/src/utilities/scanCTypes.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { Op } from 'sequelize'; - -import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { - connect, - CType, - type DidUri, - disconnect, - type ICType, - type KiltKeyringPair, - type SubmittableExtrinsic, - Utils, -} from '@kiltprotocol/sdk-js'; - -import { CType as CTypeModel } from '../models/ctype'; -import { endowAccount } from '../../testing/endowAccount'; -import { createDid } from '../../testing/createDid'; -import { createCType } from '../../testing/createCType'; -import { resetDatabase } from '../../testing/resetDatabase'; - -import { scanCTypes } from './scanCTypes'; -import { configuration } from './configuration'; -import { subScanEventGenerator } from './subScan'; - -vi.mock('./subScan'); - -let submitter: KiltKeyringPair; -let did: DidUri; - -let cType: ICType; -let extrinsic: SubmittableExtrinsic; - -function mockCTypeEvent() { - const mockParams = [ - { type_name: 'CtypeCreatorOf', value: '0xexamplecreator' }, - { type_name: 'CtypeHashOf', value: CType.idToHash(cType.$id) }, - ]; - const mockParsedParams = { - CtypeCreatorOf: '0xexamplecreator', - CtypeHashOf: CType.idToHash(cType.$id), - }; - vi.mocked(subScanEventGenerator).mockImplementation(async function* () { - yield { - params: mockParams, - parsedParams: mockParsedParams, - block: 123456, - blockTimestampMs: 160273245600, - extrinsicHash: extrinsic.hash.toHex(), - }; - }); -} - -beforeAll(async () => { - await connect(configuration.blockchainEndpoint); - - submitter = Utils.Crypto.makeKeypairFromSeed(undefined, 'sr25519'); - await endowAccount(submitter.address); - - const created = await createDid(); - did = created.did; - - ({ cType, extrinsic } = await createCType( - did, - created.assertionMethod, - submitter, - )); - - const { sequelize, initializeDatabase } = await import('./sequelize'); - await initializeDatabase(); - - return async function teardown() { - await sequelize.close(); - await disconnect(); - // give the SDK time to log the disconnect message - await new Promise((resolve) => setTimeout(resolve, 1000)); - }; -}, 30_000); - -beforeEach(async () => { - await resetDatabase(); - vi.mocked(subScanEventGenerator).mockClear(); -}); - -describe('scanCTypes', () => { - it('should add new CType to the database', async () => { - mockCTypeEvent(); - - await scanCTypes(); - - expect(subScanEventGenerator).toHaveBeenLastCalledWith( - 'ctype', - 'CTypeCreated', - 0, - ); - const created = await CTypeModel.findByPk(cType.$id); - expect(created).not.toBeNull(); - expect(created?.dataValues.title).toBe(cType.title); - }); - - it('should not add a CType if it already exists', async () => { - mockCTypeEvent(); - - await scanCTypes(); - - const latestCType = await CTypeModel.findOne({ - order: [['createdAt', 'DESC']], - where: { - block: { - [Op.not]: null, - }, - }, - }); - - const expectedFromBlock = Number(latestCType?.dataValues.block); - - vi.mocked(subScanEventGenerator).mockImplementation(async function* () { - // yield nothing - }); - - await scanCTypes(); - - expect(subScanEventGenerator).toHaveBeenCalledTimes(2); - expect(subScanEventGenerator).toHaveBeenLastCalledWith( - 'ctype', - 'CTypeCreated', - expectedFromBlock, - ); - - const count = await CTypeModel.count(); - expect(count).toBe(1); - }); - - it('should correctly upsert an existing CType', async () => { - const { $id, $schema, title, properties, type } = cType; - await CTypeModel.create({ - id: $id, - schema: $schema, - title, - properties, - type, - creator: did, - createdAt: new Date(), - extrinsicHash: extrinsic.hash.toHex(), - }); - - const beforeUpsert = await CTypeModel.findByPk(cType.$id); - - if (!beforeUpsert) { - throw new Error('Not found'); - } - - expect(beforeUpsert.dataValues.block).toBeNull(); - - mockCTypeEvent(); - await scanCTypes(); - - await beforeUpsert.reload(); - expect(beforeUpsert.dataValues.block).not.toBeNull(); - }); -}); diff --git a/src/utilities/scanCTypes.ts b/src/utilities/scanCTypes.ts deleted file mode 100644 index c8c87240..00000000 --- a/src/utilities/scanCTypes.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { CType, type HexString } from '@kiltprotocol/sdk-js'; - -import { Op } from 'sequelize'; - -import { CType as CTypeModel } from '../models/ctype'; - -import { logger } from './logger'; -import { subScanEventGenerator } from './subScan'; - -export async function scanCTypes() { - const latestCType = await CTypeModel.findOne({ - order: [['createdAt', 'DESC']], - where: { - block: { - [Op.not]: null, - }, - }, - }); - - const fromBlock = latestCType ? Number(latestCType.dataValues.block) : 0; - const eventGenerator = subScanEventGenerator( - 'ctype', - 'CTypeCreated', - fromBlock, - ); - - for await (const event of eventGenerator) { - const { blockTimestampMs, extrinsicHash } = event; - const params = event.parsedParams; - const cTypeHash = params.CtypeHashOf as HexString; - - let cTypeDetails: CType.ICTypeDetails; - try { - cTypeDetails = await CType.fetchFromChain(CType.hashToId(cTypeHash)); - } catch (exception) { - logger.error(exception, `Error fetching CType ${cTypeHash}`); - continue; - } - - const { $id, $schema, ...rest } = cTypeDetails.cType; - const { creator, createdAt: block } = cTypeDetails; - - await CTypeModel.upsert({ - id: $id, - schema: $schema, - createdAt: new Date(blockTimestampMs), - creator, - extrinsicHash, - block: block.toString(), - ...rest, - }); - } -} diff --git a/src/utilities/sequelize.ts b/src/utilities/sequelize.ts index 7d130fc7..27106d2e 100644 --- a/src/utilities/sequelize.ts +++ b/src/utilities/sequelize.ts @@ -1,7 +1,6 @@ import { Sequelize } from 'sequelize'; import { initCType } from '../models/ctype'; -import { initAttestation } from '../models/attestation'; import { initTag } from '../models/tag'; import { configuration } from './configuration'; @@ -35,7 +34,6 @@ export function trackDatabaseConnection() { } export async function initializeDatabase() { - initAttestation(sequelize); initTag(sequelize); initCType(sequelize); await sequelize.sync(); diff --git a/src/utilities/subScan.test.ts b/src/utilities/subScan.test.ts deleted file mode 100644 index fcabeb78..00000000 --- a/src/utilities/subScan.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; - -import { got } from 'got'; - -import { ConfigService, type connect } from '@kiltprotocol/sdk-js'; - -import { - getEvents, - subScanEventGenerator, - type EventsListJSON, - type EventsParamsJSON, -} from './subScan'; -import { configuration } from './configuration'; - -const api = { - query: { - system: { - number: () => ({ toNumber: () => 12345 }), - }, - }, -} as unknown as Awaited>; -ConfigService.set({ api }); - -let postResponse: EventsListJSON | EventsParamsJSON; -vi.mock('got', () => ({ - got: { - post: vi.fn().mockReturnValue({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - json: () => postResponse, - }), - }, -})); - -beforeEach(() => { - vi.mocked(got.post).mockClear(); -}); - -const module = 'ctype'; -const eventId = 'CTypeCreated'; - -describe('subScan', () => { - describe('getEvents', () => { - it('should query the subscan API', async () => { - postResponse = { data: { count: 0, events: null } }; - - await getEvents({ - module, - eventId, - fromBlock: 10, - page: 0, - row: 0, - }); - - expect(got.post).toHaveBeenCalledWith( - 'https://example.api.subscan.io/api/v2/scan/events', - { - headers: { 'X-API-Key': configuration.subscan.secret }, - json: { - module, - event_id: eventId, - block_range: '10-100010', - order: 'asc', - page: 0, - row: 0, - finalized: true, - }, - }, - ); - }); - - it('should return the count when no events exist', async () => { - postResponse = { data: { count: 0, events: null } }; - - const cTypeEvents = await getEvents({ - module, - eventId, - fromBlock: 10, - page: 0, - row: 0, - }); - - expect(cTypeEvents.count).toBe(0); - expect(cTypeEvents.events).toBeUndefined(); - }); - }); - - describe('subScanEventGenerator', () => { - it('should iterate through pages in reverse order', async () => { - postResponse = { data: { count: 200, events: [] } }; - - const eventGenerator = subScanEventGenerator(module, eventId, 0); - - for await (const event of eventGenerator) { - expect(event).toBeDefined(); - } - - expect(got.post).toHaveBeenCalledTimes(6); - const { calls } = vi.mocked(got.post).mock; - - // get count - expect(calls[0][1]).toMatchObject({ json: { page: 0, row: 1 } }); - - // get last page - expect(calls[2][1]).toMatchObject({ json: { page: 1, row: 100 } }); - - // get first page - expect(calls[4][1]).toMatchObject({ json: { page: 0, row: 100 } }); - }); - }); - it('should get events in batches if current block is higher than block range', async () => { - const api = { - query: { - system: { - number: () => ({ toNumber: () => 150000 }), - }, - }, - } as unknown as Awaited>; - ConfigService.set({ api }); - - postResponse = { data: { count: 100, events: [] } }; - - const eventGenerator = subScanEventGenerator(module, eventId, 0); - - for await (const event of eventGenerator) { - expect(event).toBeDefined(); - } - - expect(got.post).toHaveBeenCalledTimes(8); - const { calls } = vi.mocked(got.post).mock; - - expect(calls[6][1]).toMatchObject({ - json: { block_range: '100000-200000', page: 0, row: 100 }, - }); - }); -}); diff --git a/src/utilities/subScan.ts b/src/utilities/subScan.ts deleted file mode 100644 index a355a815..00000000 --- a/src/utilities/subScan.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { got } from 'got'; - -import { ConfigService } from '@kiltprotocol/sdk-js'; - -import { configuration } from './configuration'; -import { logger } from './logger'; -import { sleep } from './sleep'; - -const { subscan } = configuration; - -const SUBSCAN_MAX_ROWS = 100; -const QUERY_INTERVAL_MS = 1000; -const BLOCK_RANGE_SIZE = 100_000; - -const subscanAPI = `https://${subscan.network}.api.subscan.io`; -const eventsListURL = `${subscanAPI}/api/v2/scan/events`; -const eventsParamsURL = `${subscanAPI}/api/scan/event/params`; -const headers = { - 'X-API-Key': subscan.secret, -}; - -/** - * Structure of SubScan responses from `/api/v2/scan/events`. - */ -export interface EventsListJSON { - code?: number; - data: { - count: number; - events: Array<{ - block_timestamp: number; // UNIX-time in seconds - event_id: string; - event_index: string; - extrinsic_hash: string; - extrinsic_index: string; - finalized: true; - id: number; - module_id: string; - phase: number; - }> | null; - }; - generated_at?: number; - message?: string; -} - -/** - * Structure of SubScan responses from `/api/scan/event/params`. - */ -export interface EventsParamsJSON { - code: number; - data: Array<{ - event_index: string; - params: Array<{ - name?: string; - type?: string; - type_name: string; - value: unknown; - }>; - }>; - generated_at: number; - message: string; -} - -export async function getEvents({ - fromBlock, - row = SUBSCAN_MAX_ROWS, - eventId, - ...parameters -}: { - module: string; // Pallet name - eventId: string; // Event emitted - fromBlock: number; - page: number; - row?: number; -}) { - const payloadForEventsListRequest = { - ...parameters, - event_id: eventId, - block_range: `${fromBlock}-${fromBlock + BLOCK_RANGE_SIZE}`, - order: 'asc', - row, - finalized: true, - }; - - logger.debug( - 'payloadForEventsListRequest: ' + - JSON.stringify(payloadForEventsListRequest, null, 2), - ); - - const { - data: { count, events }, - } = await got - .post(eventsListURL, { headers, json: payloadForEventsListRequest }) - .json(); - - if (!events) { - return { count }; - } - - const eventIndices = events.map((event) => event.event_index); - - const payloadForEventsParamsRequest = { event_index: eventIndices }; - logger.debug( - 'payloadForEventsParamsRequest: ' + - JSON.stringify(payloadForEventsParamsRequest, null, 2), - ); - - const { data: eventsParameters } = await got - .post(eventsParamsURL, { headers, json: payloadForEventsParamsRequest }) - .json(); - - const parsedEvents = events.map( - // eslint-disable-next-line @typescript-eslint/naming-convention - ({ event_index, block_timestamp, extrinsic_hash }) => { - // Block number - const block = parseInt(event_index.split('-')[0]); - - const params = eventsParameters.find( - (detailed) => detailed.event_index === event_index, - )?.params; - if (!params || params.length === 0) { - throw new Error( - `Parameters could not be retrieved for event with index: ${event_index}`, - ); - } - - return { - block, - blockTimestampMs: block_timestamp * 1000, - params, - extrinsicHash: extrinsic_hash, - }; - }, - ); - - logger.debug('parsedEvents: ' + JSON.stringify(parsedEvents, null, 2)); - - return { count, events: parsedEvents }; -} - -export interface ParsedEvent { - block: number; - blockTimestampMs: number; - params: EventsParamsJSON['data'][number]['params']; - extrinsicHash: string; -} - -/** Extends the `event` with the parameters parsed, - * so that the parameters value extraction is easier and more elegant. - * - * @param event - * @returns the extended event - */ -function parseParams(event: ParsedEvent) { - return { - ...event, - parsedParams: Object.fromEntries( - event.params.map((param) => [param.type_name, param.value]), - ), - }; -} - -export async function* subScanEventGenerator( - module: string, - eventId: string, - startBlock: number, -) { - if (subscan.network === 'NONE') { - return; - } - - const api = ConfigService.get('api'); - - const currentBlock = (await api.query.system.number()).toNumber(); - - // get events in batches until the current block is reached - for ( - let fromBlock = startBlock; - fromBlock < currentBlock; - fromBlock += BLOCK_RANGE_SIZE - ) { - const parameters = { - module, - eventId, - fromBlock, - }; - - const { count } = await getEvents({ ...parameters, page: 0, row: 1 }); - - const blockRange = `${fromBlock} - ${fromBlock + BLOCK_RANGE_SIZE}`; - - if (count === 0) { - logger.debug( - `No new "${eventId}" events found on SubScan in block range ${blockRange}.`, - ); - await sleep(QUERY_INTERVAL_MS); - continue; - } - - logger.debug( - `Found ${count} new "${eventId}" events on SubScan for in block range ${blockRange}.`, - ); - - const pages = Math.ceil(count / SUBSCAN_MAX_ROWS) - 1; - - for (let page = pages; page >= 0; page--) { - const { events } = await getEvents({ ...parameters, page }); - if (!events) { - continue; - } - - logger.debug( - `Loaded page ${page} of "${eventId}" events in block range ${blockRange}.`, - ); - - // Yields the events extended with the parameters comfortably parsed - for (const event of events) { - yield parseParams(event); - } - - await sleep(QUERY_INTERVAL_MS); - } - } -} diff --git a/src/utilities/watchSubScan.ts b/src/utilities/watchSubScan.ts deleted file mode 100644 index e836df02..00000000 --- a/src/utilities/watchSubScan.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { configuration } from './configuration'; -import { scanCTypes } from './scanCTypes'; -import { sleep } from './sleep'; -import { scanAttestations } from './scanAttestations'; - -const { isTest } = configuration; - -const SCAN_INTERVAL_MS = 10 * 60 * 1000; - -export function watchSubScan() { - if (isTest) { - return; - } - (async () => { - while (true) { - await scanCTypes(); - await scanAttestations(); - await sleep(SCAN_INTERVAL_MS); - } - })(); -} diff --git a/testing/globalSetup.ts b/testing/globalSetup.ts index 47d39615..0ffa2031 100644 --- a/testing/globalSetup.ts +++ b/testing/globalSetup.ts @@ -14,8 +14,8 @@ import { const env = { MODE: 'test', - SUBSCAN_NETWORK: 'example', - SECRET_SUBSCAN: 'SECRET_SUBSCAN', + GRAPHQL_ENDPOINT: 'placeholder', + POLKADOT_RPC_ENDPOINT: 'placeholder', BLOCKCHAIN_ENDPOINT: '', DATABASE_URI: '', DID: 'placeholder', diff --git a/testing/resetDatabase.ts b/testing/resetDatabase.ts index 6ac3b6cf..d5dd4213 100644 --- a/testing/resetDatabase.ts +++ b/testing/resetDatabase.ts @@ -1,11 +1,9 @@ import { CType } from '../src/models/ctype'; -import { Attestation } from '../src/models/attestation'; import { Tag } from '../src/models/tag'; import { sequelize } from '../src/utilities/sequelize'; export async function resetDatabase() { await sequelize.sync(); - await Attestation.destroy({ where: {} }); await Tag.destroy({ where: {} }); await CType.destroy({ where: {} }); } diff --git a/testing/setup.ts b/testing/setup.ts index ef3ab5f7..b310bb1b 100644 --- a/testing/setup.ts +++ b/testing/setup.ts @@ -13,9 +13,9 @@ vi.mock('../src/utilities/configuration', () => ({ isTest: true, databaseUri: import.meta.env.DATABASE_URI as string, blockchainEndpoint: import.meta.env.BLOCKCHAIN_ENDPOINT as string, - subscan: { - network: 'example', - secret: 'SECRET_SUBSCAN', + indexer: { + graphqlEndpoint: 'placeholder', + polkadotRPCEndpoint: 'placeholder', }, }, }));