From 659d0d6358968a0fddcb867a4c43d8f294ebe7e5 Mon Sep 17 00:00:00 2001 From: Timothee Legros <62490329+timolegros@users.noreply.github.com> Date: Mon, 14 Aug 2023 20:02:17 +0200 Subject: [PATCH] Kill ChainEntityMeta (#4025) * remove ChainEntityMeta model + routes + messageProcessor * client side entity meta removal + remaining server references * chain-events + common-common entity meta removal * remove entity meta migration CW * fix types * remove queued column from ChainEntities CE side * revert migration create bare-bones ChainEntityMeta table * Remove threadTitle. * trigger CI again * update compound getVotes so it doesn't run out of tokens for tests * update ganache version to latest * trigger CI * trigger CI * merge master fixes * remove entity_meta from threads model on client * remove entity_meta from backend get_bulk_threads --------- Co-authored-by: Jake Naviasky Co-authored-by: Muon Shot <120686579+CowMuon@users.noreply.github.com> --- .../chain-events/chain-testing/package.json | 2 +- .../src/utils/governance/compoundGov.ts | 49 +++++++++--- packages/chain-events/chain-testing/yarn.lock | 24 +++++- .../ChainEventHandlers/entityArchival.ts | 24 +----- .../20230525101148-remove-entity-queued.js | 15 ++++ .../services/database/models/chain_entity.ts | 2 - .../service-tests/EVM/aave.spec.ts | 13 --- .../service-tests/EVM/compoundBravo.spec.ts | 13 --- .../rabbitmq/configuration/rascalConfig.ts | 21 ----- .../src/rabbitmq/rabbitMQConfig.ts | 8 -- .../src/rabbitmq/types/chainEntityCUD.ts | 43 ---------- .../common-common/src/rabbitmq/types/index.ts | 12 +-- .../chain/substrate/democracy_proposal.ts | 9 +-- .../chain/substrate/democracy_referendum.ts | 1 - .../chain/substrate/treasury_proposal.ts | 5 +- .../controllers/server/chain_entities.ts | 42 ++-------- .../client/scripts/models/ChainEntity.ts | 16 +--- .../client/scripts/models/Proposal.ts | 1 - .../client/scripts/models/Thread.ts | 17 +--- .../components/ChainEntitiesSelector/utils.ts | 1 - .../modals/update_proposal_status_modal.tsx | 2 +- .../pages/chain_entity_link_redirect.tsx | 3 +- packages/commonwealth/package.json | 3 - .../CommonwealthConsumer.ts | 8 -- .../messageProcessors/chainEntityCUDQueue.ts | 19 ----- .../get_bulk_threads.ts | 15 ---- packages/commonwealth/server/database.ts | 2 - .../20230525090836-remove-entity-meta.js | 20 +++++ packages/commonwealth/server/models.ts | 2 - packages/commonwealth/server/models/chain.ts | 1 - .../server/models/chain_entity_meta.ts | 65 --------------- packages/commonwealth/server/models/thread.ts | 3 - .../commonwealth/server/routes/deleteChain.ts | 6 -- .../server/routes/fetchEntityTitle.ts | 22 ------ .../server/routes/getEntityMeta.ts | 48 ----------- .../server/routes/updateChainEntityTitle.ts | 55 ------------- .../commonwealth/server/routing/router.ts | 25 +----- .../server/scripts/enforceDataConsistency.ts | 67 ---------------- .../enforceDataConsistency.spec.ts | 41 ---------- packages/token-balance-cache/test/evm.spec.ts | 79 +++++++++++-------- 40 files changed, 163 insertions(+), 641 deletions(-) create mode 100644 packages/chain-events/services/database/migrations/20230525101148-remove-entity-queued.js delete mode 100644 packages/common-common/src/rabbitmq/types/chainEntityCUD.ts delete mode 100644 packages/commonwealth/server/CommonwealthConsumer/messageProcessors/chainEntityCUDQueue.ts create mode 100644 packages/commonwealth/server/migrations/20230525090836-remove-entity-meta.js delete mode 100644 packages/commonwealth/server/models/chain_entity_meta.ts delete mode 100644 packages/commonwealth/server/routes/fetchEntityTitle.ts delete mode 100644 packages/commonwealth/server/routes/getEntityMeta.ts delete mode 100644 packages/commonwealth/server/routes/updateChainEntityTitle.ts delete mode 100644 packages/commonwealth/server/scripts/enforceDataConsistency.ts delete mode 100644 packages/commonwealth/test/integration/enforceDataConsistency.spec.ts diff --git a/packages/chain-events/chain-testing/package.json b/packages/chain-events/chain-testing/package.json index 881ca72090c..65e3a5488bd 100644 --- a/packages/chain-events/chain-testing/package.json +++ b/packages/chain-events/chain-testing/package.json @@ -25,6 +25,6 @@ "web3": "^1.8.2" }, "optionalDependencies": { - "ganache": "^7.7.6" + "ganache": "^7.9.0" } } diff --git a/packages/chain-events/chain-testing/src/utils/governance/compoundGov.ts b/packages/chain-events/chain-testing/src/utils/governance/compoundGov.ts index 71e16dbd1bd..d35b452e7b1 100644 --- a/packages/chain-events/chain-testing/src/utils/governance/compoundGov.ts +++ b/packages/chain-events/chain-testing/src/utils/governance/compoundGov.ts @@ -155,21 +155,52 @@ export class compoundGovernor implements IGovernor { let txReceipt; if (currBalance.lt(tokensNeeded)) { try { + const binanceWalletAddress = + '0xF977814e90dA44bFA03b6295A0616a897441aceC'; + const binanceWalletTokensAvailable = provider.utils.toBN( + await compToken.methods.balanceOf(binanceWalletAddress).call() + ); txReceipt = await compToken.methods - .transfer(accounts, tokensNeeded) + .transfer(accounts, binanceWalletTokensAvailable) .send({ - from: '0xF977814e90dA44bFA03b6295A0616a897441aceC', - gasLimit: 100000, - }); - } catch { - txReceipt = await compToken.methods - .transfer(accounts, tokensNeeded) - .send({ - from: '0xfA9b5f7fDc8AB34AAf3099889475d47febF830D7', + from: binanceWalletAddress, gasLimit: 100000, }); + } catch (e) { + console.error('Failed to transfer tokens from binance wallet', e); + } + + const newBalance = provider.utils.toBN( + await compToken.methods.balanceOf(accounts).call() + ); + + if (newBalance.lt(tokensNeeded)) { + try { + const fundWalletAddress = + '0xfA9b5f7fDc8AB34AAf3099889475d47febF830D7'; + const fundWalletTokensAvailable = provider.utils.toBN( + await compToken.methods.balanceOf(fundWalletAddress).call() + ); + txReceipt = await compToken.methods + .transfer(accounts, fundWalletTokensAvailable) + .send({ + from: fundWalletAddress, + gasLimit: 100000, + }); + } catch (e) { + console.log('Failed to transfer tokens from fund wallet', e); + } } } + + const newBalance = provider.utils.toBN( + await compToken.methods.balanceOf(accounts).call() + ); + if (newBalance.lt(tokensNeeded)) { + console.error('Not enough tokens in binance and fund wallet'); + return; + } + try { txReceipt = await compToken.methods .delegate(accounts) diff --git a/packages/chain-events/chain-testing/yarn.lock b/packages/chain-events/chain-testing/yarn.lock index e13c7a7e225..e22526b3de3 100644 --- a/packages/chain-events/chain-testing/yarn.lock +++ b/packages/chain-events/chain-testing/yarn.lock @@ -254,6 +254,16 @@ dependencies: node-gyp-build "4.4.0" +"@trufflesuite/uws-js-unofficial@20.10.0-unofficial.2": + version "20.10.0-unofficial.2" + resolved "https://registry.yarnpkg.com/@trufflesuite/uws-js-unofficial/-/uws-js-unofficial-20.10.0-unofficial.2.tgz#7ed613ce3260cd5d1773a4d5787a2a106acd1a91" + integrity sha512-oQQlnS3oNeGsgS4K3KCSSavJgSb0W9D5ktZs4FacX9VbM7b+NlhjH96d6/G4fMrz+bc5MXRyco419on0X0dvRA== + dependencies: + ws "8.2.3" + optionalDependencies: + bufferutil "4.0.5" + utf-8-validate "5.0.7" + "@tsconfig/node10@^1.0.7": version "1.0.9" resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" @@ -1283,12 +1293,13 @@ function-bind@^1.1.1: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== -ganache@^7.7.6: - version "7.7.7" - resolved "https://registry.yarnpkg.com/ganache/-/ganache-7.7.7.tgz#19939a86799f0bcb7df02e88082944466394b913" - integrity sha512-kZUuOcgDQBtbxzs4iB3chg1iAc28s2ffdOdzyTTzo4vr9sb843w4PbWd5v1hsIqtcNjurcpLaW8XRp/cw2u++g== +ganache@^7.9.0: + version "7.9.0" + resolved "https://registry.yarnpkg.com/ganache/-/ganache-7.9.0.tgz#561deceb376b1c4e8998ac8e5a842574507d3295" + integrity sha512-KdsTZaAKqDXTNDMKnLzg0ngX8wnZKyVGm7HD03GIyUMVRuXI83s0CUEaGIDWRUWTQP7BE8sDh7QtbW+NoX4zrQ== dependencies: "@trufflesuite/bigint-buffer" "1.1.10" + "@trufflesuite/uws-js-unofficial" "20.10.0-unofficial.2" "@types/bn.js" "^5.1.0" "@types/lru-cache" "5.1.1" "@types/seedrandom" "3.0.1" @@ -2830,6 +2841,11 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== +ws@8.2.3: + version "8.2.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba" + integrity sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA== + ws@^3.0.0: version "3.3.3" resolved "https://registry.yarnpkg.com/ws/-/ws-3.3.3.tgz#f1cf84fe2d5e901ebce94efaece785f187a228f2" diff --git a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts index f075980f1ee..2b130e4b9d4 100644 --- a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts +++ b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts @@ -3,11 +3,7 @@ */ import { addPrefix, factory } from 'common-common/src/logging'; -import type { RmqEntityCUD } from 'common-common/src/rabbitmq/types'; -import { - AbstractRabbitMQController, - RascalPublications, -} from 'common-common/src/rabbitmq/types'; +import { AbstractRabbitMQController } from 'common-common/src/rabbitmq/types'; import type { DB } from '../../database/database'; @@ -83,24 +79,6 @@ export default class extends IEventHandler { completed, }); - const publishData: RmqEntityCUD.RmqMsgType = { - ce_id: dbEntity.id, - chain_id: dbEntity.chain, - author, - entity_type_id: type_id, - cud: 'create', - }; - - await this._rmqController.safePublish( - publishData, - dbEntity.id, - RascalPublications.ChainEntityCUDMain, - { - sequelize: this._models.sequelize, - model: this._models.ChainEntity, - } - ); - if (dbEvent.entity_id !== dbEntity.id) { dbEvent.entity_id = dbEntity.id; await dbEvent.save(); diff --git a/packages/chain-events/services/database/migrations/20230525101148-remove-entity-queued.js b/packages/chain-events/services/database/migrations/20230525101148-remove-entity-queued.js new file mode 100644 index 00000000000..be00e85ac6c --- /dev/null +++ b/packages/chain-events/services/database/migrations/20230525101148-remove-entity-queued.js @@ -0,0 +1,15 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.removeColumn('ChainEntities', 'queued'); + }, + + down: async (queryInterface, Sequelize) => { + await queryInterface.addColumn('ChainEntities', 'queued', { + type: Sequelize.STRING, + defaultValue: 0, + allowNull: false, + }); + }, +}; diff --git a/packages/chain-events/services/database/models/chain_entity.ts b/packages/chain-events/services/database/models/chain_entity.ts index 3e93c95353b..e74bf57fb22 100644 --- a/packages/chain-events/services/database/models/chain_entity.ts +++ b/packages/chain-events/services/database/models/chain_entity.ts @@ -9,7 +9,6 @@ export type ChainEntityAttributes = { chain: string; type: string; type_id: string; - queued: number; author?: string; completed?: boolean; created_at?: Date; @@ -46,7 +45,6 @@ export default ( author: { type: dataTypes.STRING, allowNull: true }, created_at: { type: dataTypes.DATE, allowNull: false }, updated_at: { type: dataTypes.DATE, allowNull: false }, - queued: { type: dataTypes.SMALLINT, allowNull: false, defaultValue: 0 }, }, { tableName: 'ChainEntities', diff --git a/packages/chain-events/test/integration/service-tests/EVM/aave.spec.ts b/packages/chain-events/test/integration/service-tests/EVM/aave.spec.ts index b49f9e5a139..4473277b139 100644 --- a/packages/chain-events/test/integration/service-tests/EVM/aave.spec.ts +++ b/packages/chain-events/test/integration/service-tests/EVM/aave.spec.ts @@ -237,19 +237,6 @@ describe('Integration tests for Aave', () => { expect(relatedEntity.id, 'Incorrect entity id').to.equal( propCreatedEvent.entity_id ); - - expect( - rmq.queuedMessages[RascalSubscriptions.ChainEntityCUDMain].length - ).to.equal(1); - expect( - rmq.queuedMessages[RascalSubscriptions.ChainEntityCUDMain][0] - ).to.deep.equal({ - author: relatedEntity.author, - ce_id: relatedEntity.id, - chain_id, - entity_type_id: relatedEntity.type_id, - cud: 'create', - }); }); it('Should process vote cast events', async () => { diff --git a/packages/chain-events/test/integration/service-tests/EVM/compoundBravo.spec.ts b/packages/chain-events/test/integration/service-tests/EVM/compoundBravo.spec.ts index 073ba01a4b1..d8ba612b603 100644 --- a/packages/chain-events/test/integration/service-tests/EVM/compoundBravo.spec.ts +++ b/packages/chain-events/test/integration/service-tests/EVM/compoundBravo.spec.ts @@ -243,19 +243,6 @@ describe('Integration tests for Compound Bravo', () => { expect(relatedEntity.id, 'Incorrect entity id').to.equal( propCreatedEvent.entity_id ); - - expect( - rmq.queuedMessages[RascalSubscriptions.ChainEntityCUDMain].length - ).to.equal(1); - expect( - rmq.queuedMessages[RascalSubscriptions.ChainEntityCUDMain][0] - ).to.deep.equal({ - author: relatedEntity.author, - ce_id: relatedEntity.id, - chain_id, - entity_type_id: relatedEntity.type_id, - cud: 'create', - }); }); it('Should process vote cast events', async () => { diff --git a/packages/common-common/src/rabbitmq/configuration/rascalConfig.ts b/packages/common-common/src/rabbitmq/configuration/rascalConfig.ts index 5785c02e7ad..38dfda22e98 100644 --- a/packages/common-common/src/rabbitmq/configuration/rascalConfig.ts +++ b/packages/common-common/src/rabbitmq/configuration/rascalConfig.ts @@ -120,12 +120,6 @@ export function getAllRascalConfigs( arguments: queueOptions, }, }, - [RascalQueues.ChainEntityCUDMain]: { - ...queueConfig, - options: { - arguments: queueOptions, - }, - }, [RascalQueues.ChainEventNotificationsCUDMain]: { ...queueConfig, options: { @@ -162,12 +156,6 @@ export function getAllRascalConfigs( destinationType: 'queue', bindingKey: RascalRoutingKeys.ChainEvents, }, - [RascalBindings.ChainEntityCUDMain]: { - source: RascalExchanges.CUD, - destination: RascalQueues.ChainEntityCUDMain, - destinationType: 'queue', - bindingKey: RascalRoutingKeys.ChainEntityCUD, - }, [RascalBindings.ChainEventNotificationsCUD]: { source: RascalExchanges.CUD, destination: RascalQueues.ChainEventNotificationsCUDMain, @@ -200,11 +188,6 @@ export function getAllRascalConfigs( routingKey: RascalRoutingKeys.ChainEvents, ...publicationConfig, }, - [RascalPublications.ChainEntityCUDMain]: { - exchange: RascalExchanges.CUD, - routingKey: RascalRoutingKeys.ChainEntityCUD, - ...publicationConfig, - }, [RascalPublications.ChainEventNotificationsCUDMain]: { exchange: RascalExchanges.CUD, routingKey: RascalRoutingKeys.ChainEventNotificationsCUD, @@ -232,10 +215,6 @@ export function getAllRascalConfigs( queue: RascalQueues.ChainEvents, ...subscriptionConfig, }, - [RascalSubscriptions.ChainEntityCUDMain]: { - queue: RascalQueues.ChainEntityCUDMain, - ...subscriptionConfig, - }, [RascalSubscriptions.ChainEventNotificationsCUDMain]: { queue: RascalQueues.ChainEventNotificationsCUDMain, ...subscriptionConfig, diff --git a/packages/common-common/src/rabbitmq/rabbitMQConfig.ts b/packages/common-common/src/rabbitmq/rabbitMQConfig.ts index 66bcbfa61b4..421ce99a3a2 100644 --- a/packages/common-common/src/rabbitmq/rabbitMQConfig.ts +++ b/packages/common-common/src/rabbitmq/rabbitMQConfig.ts @@ -71,28 +71,24 @@ export function getRabbitMQConfig( RascalExchanges.Discobot, ]); copyConfigs(allQueues, vhostConfig.queues, [ - RascalQueues.ChainEntityCUDMain, RascalQueues.ChainEventNotificationsCUDMain, RascalQueues.ChainEventNotifications, RascalQueues.SnapshotListener, RascalQueues.DiscordListener, ]); copyConfigs(allBindings, vhostConfig.bindings, [ - RascalBindings.ChainEntityCUDMain, RascalBindings.ChainEventNotificationsCUD, RascalBindings.ChainEventNotifications, RascalBindings.SnapshotListener, RascalBindings.DiscordListener, ]); copyConfigs(allPublications, vhostConfig.publications, [ - RascalPublications.ChainEntityCUDMain, RascalPublications.ChainEventNotificationsCUDMain, RascalPublications.ChainEventNotifications, RascalPublications.SnapshotListener, RascalPublications.DiscordListener, ]); copyConfigs(allSubscriptions, vhostConfig.subscriptions, [ - RascalSubscriptions.ChainEntityCUDMain, RascalSubscriptions.ChainEventNotificationsCUDMain, RascalSubscriptions.ChainEventNotifications, RascalSubscriptions.SnapshotListener, @@ -105,22 +101,18 @@ export function getRabbitMQConfig( ]); copyConfigs(allQueues, vhostConfig.queues, [ RascalQueues.ChainEvents, - RascalQueues.ChainEntityCUDMain, RascalQueues.ChainEventNotificationsCUDMain, ]); copyConfigs(allBindings, vhostConfig.bindings, [ RascalBindings.ChainEvents, - RascalBindings.ChainEntityCUDMain, RascalBindings.ChainEventNotificationsCUD, ]); copyConfigs(allPublications, vhostConfig.publications, [ RascalPublications.ChainEvents, - RascalPublications.ChainEntityCUDMain, RascalPublications.ChainEventNotificationsCUDMain, ]); copyConfigs(allSubscriptions, vhostConfig.subscriptions, [ RascalSubscriptions.ChainEvents, - RascalSubscriptions.ChainEntityCUDMain, RascalSubscriptions.ChainEventNotificationsCUDMain, ]); } else if (service === RascalConfigServices.SnapshotService) { diff --git a/packages/common-common/src/rabbitmq/types/chainEntityCUD.ts b/packages/common-common/src/rabbitmq/types/chainEntityCUD.ts deleted file mode 100644 index 6d626c72f46..00000000000 --- a/packages/common-common/src/rabbitmq/types/chainEntityCUD.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { RmqMsgNamespace } from './index'; -import { RmqMsgFormatError } from './index'; - -interface IRmqMsgCreateEntityCUD { - ce_id: number; - chain_id: string; - author?: string; - entity_type_id: string; - cud: 'create'; -} - -export const RmqEntityCUD: RmqMsgNamespace = { - getInvalidFormatError(entity: any): RmqMsgFormatError { - return new RmqMsgFormatError( - `The following entity is improperly formatted: ${JSON.stringify(entity)}` - ); - }, - - isValidMsgFormat(data: any): data is IRmqMsgCreateEntityCUD { - return !!( - typeof data.ce_id === 'number' && - data.chain_id && - typeof data.chain_id === 'string' && - data.entity_type_id && - typeof data.entity_type_id === 'string' && - data.cud === 'create' - ); - }, - - checkMsgFormat(data: any): void { - const valid = this.isValidMsgFormat(data); - if (!valid) { - console.log( - `The following entity is improperly formatted: ${JSON.stringify(data)}` - ); - throw this.getInvalidFormatError(data); - } - }, -}; - -export namespace RmqEntityCUD { - export type RmqMsgType = IRmqMsgCreateEntityCUD; -} diff --git a/packages/common-common/src/rabbitmq/types/index.ts b/packages/common-common/src/rabbitmq/types/index.ts index 3decc9d25d4..7387f8d7813 100644 --- a/packages/common-common/src/rabbitmq/types/index.ts +++ b/packages/common-common/src/rabbitmq/types/index.ts @@ -1,7 +1,5 @@ -import type { RmqEntityCUD } from './chainEntityCUD'; import type { RmqCENotificationCUD } from './chainEventNotificationsCUD'; -export * from './chainEntityCUD'; export * from './chainEventNotificationsCUD'; export * from './chainEvents'; @@ -30,7 +28,6 @@ export class RmqMsgFormatError extends Error { * anywhere, it MUST be one of these types */ export type TRmqMessages = - | RmqEntityCUD.RmqMsgType | RmqCENotificationCUD.RmqMsgType | RmqCWEvent.RmqMsgType | RmqCENotification.RmqMsgType @@ -46,7 +43,6 @@ export interface RmqMsgNamespace { export enum RascalPublications { ChainEvents = 'ChainEventsPublication', - ChainEntityCUDMain = 'ChainEntityCUDMainPublication', ChainEventNotificationsCUDMain = 'ChainEventNotificationsCUDMainPublication', ChainEventNotifications = 'ChainEventNotificationsPublication', SnapshotListener = 'SnapshotListenerPublication', @@ -55,7 +51,6 @@ export enum RascalPublications { export enum RascalSubscriptions { ChainEvents = 'ChainEventsSubscription', - ChainEntityCUDMain = 'ChainEntityCUDMainSubscription', ChainEventNotificationsCUDMain = 'ChainEventNotificationsCUDSubscription', ChainEventNotifications = 'ChainEventNotificationsSubscription', SnapshotListener = 'SnapshotListenerSubscription', @@ -73,7 +68,6 @@ export enum RascalExchanges { export enum RascalQueues { ChainEvents = 'ChainEventsQueueV2', - ChainEntityCUDMain = 'ChainEntityCUDMainQueueV2', ChainEventNotificationsCUDMain = 'ChainEventNotificationsCUDMainQueueV2', ChainEventNotifications = 'ChainEventNotificationsQueueV2', DeadLetter = 'DeadLetterQueue', @@ -83,7 +77,6 @@ export enum RascalQueues { export enum RascalBindings { ChainEvents = 'ChainEventsBinding', - ChainEntityCUDMain = 'ChainEntityCUDMainBinding', ChainEventNotificationsCUD = 'ChainEventNotificationsCUDBinding', ChainEventNotifications = 'ChainEventNotificationsBinding', SnapshotListener = 'SnapshotListenerBinding', @@ -93,7 +86,6 @@ export enum RascalBindings { export enum RascalRoutingKeys { ChainEvents = 'ChainEvents', - ChainEntityCUD = 'ChainEntityCUD', ChainEventNotificationsCUD = 'ChainEventNotificationsCUD', ChainEventNotifications = 'ChainEventNotifications', SnapshotListener = 'SnapshotListener', @@ -101,9 +93,7 @@ export enum RascalRoutingKeys { DiscordListener = 'DiscordListener', } -export type SafeRmqPublishSupported = - | ChainEntityModelStatic - | ChainEventModelStatic; +export type SafeRmqPublishSupported = ChainEventModelStatic; export abstract class AbstractRabbitMQController { protected _initialized = false; diff --git a/packages/commonwealth/client/scripts/controllers/chain/substrate/democracy_proposal.ts b/packages/commonwealth/client/scripts/controllers/chain/substrate/democracy_proposal.ts index 3b82b911a43..c3e45a4d89e 100644 --- a/packages/commonwealth/client/scripts/controllers/chain/substrate/democracy_proposal.ts +++ b/packages/commonwealth/client/scripts/controllers/chain/substrate/democracy_proposal.ts @@ -168,7 +168,6 @@ class SubstrateDemocracyProposal extends Proposal< ); this.hash = eventData.proposalHash; this.createdAt = entity.createdAt; - this.threadTitle = entity.threadTitle; // see if preimage exists and populate data if it does const preimage = this._Proposals.app.chainEntities.getPreimage( @@ -178,11 +177,11 @@ class SubstrateDemocracyProposal extends Proposal< this._method = preimage.method; this._section = preimage.section; this._preimage = preimage; - this.title = entity.title || formatCall(preimage); + this.title = formatCall(preimage); } else { - this.title = - entity.title || - `Proposal ${formatProposalHashShort(eventData.proposalHash)}`; + this.title = `Proposal ${formatProposalHashShort( + eventData.proposalHash + )}`; } entity.chainEvents.forEach((e) => this.update(e)); diff --git a/packages/commonwealth/client/scripts/controllers/chain/substrate/democracy_referendum.ts b/packages/commonwealth/client/scripts/controllers/chain/substrate/democracy_referendum.ts index c1670e29f27..9b3d42244a5 100644 --- a/packages/commonwealth/client/scripts/controllers/chain/substrate/democracy_referendum.ts +++ b/packages/commonwealth/client/scripts/controllers/chain/substrate/democracy_referendum.ts @@ -261,7 +261,6 @@ export class SubstrateDemocracyReferendum extends Proposal< this._threshold = this.data.threshold; this.hash = eventData.proposalHash; this.createdAt = entity.createdAt; - this.threadTitle = entity.threadTitle; // see if associated entity title exists, otherwise try to populate title with preimage const preimage = this._Democracy.app.chainEntities.getPreimage( diff --git a/packages/commonwealth/client/scripts/controllers/chain/substrate/treasury_proposal.ts b/packages/commonwealth/client/scripts/controllers/chain/substrate/treasury_proposal.ts index 5bc3fdde1d1..969932ccd5b 100644 --- a/packages/commonwealth/client/scripts/controllers/chain/substrate/treasury_proposal.ts +++ b/packages/commonwealth/client/scripts/controllers/chain/substrate/treasury_proposal.ts @@ -10,7 +10,7 @@ import type ChainEntity from '../../../models/ChainEntity'; import type ChainEvent from '../../../models/ChainEvent'; import type { ITXModalData } from '../../../models/interfaces'; import Proposal from '../../../models/Proposal'; -import type { ProposalEndTime} from '../../../models/types'; +import type { ProposalEndTime } from '../../../models/types'; import { ProposalStatus, VotingType, VotingUnit } from '../../../models/types'; import type { BinaryVote } from '../../../models/votes'; import type SubstrateAccounts from './account'; @@ -158,9 +158,8 @@ export class SubstrateTreasuryProposal extends Proposal< ? this._Accounts.fromAddress(this.data.proposer) : null; - this.title = entity.title || this.generateTitle(); + this.title = this.generateTitle(); this.createdAt = entity.createdAt; - this.threadTitle = entity.threadTitle; entity.chainEvents.forEach((e) => this.update(e)); diff --git a/packages/commonwealth/client/scripts/controllers/server/chain_entities.ts b/packages/commonwealth/client/scripts/controllers/server/chain_entities.ts index 92abf22b66f..dac09cae9f5 100644 --- a/packages/commonwealth/client/scripts/controllers/server/chain_entities.ts +++ b/packages/commonwealth/client/scripts/controllers/server/chain_entities.ts @@ -83,24 +83,10 @@ class ChainEntityController { .filter((e) => e.type === type); } - private static _formatEntitiesWithMeta( - entities: any[], - entityMetas: any[] - ): ChainEntity[] { + private static _formatEntities(entities: any[]): ChainEntity[] { const data: ChainEntity[] = []; - // save chain-entity metadata to the appropriate chain-entity - const metaMap: Map = new Map( - entityMetas.map((e) => [e.ce_id, { title: e.title }]) - ); - if (Array.isArray(entities)) { - // save the chain-entity objects in the store for (const entityJSON of entities) { - const metaData = metaMap.get(entityJSON.id); - if (metaData) { - entityJSON.title = metaData.title; - } - const entity = ChainEntity.fromJSON(entityJSON); data.push(entity); } @@ -114,17 +100,11 @@ class ChainEntityController { * @param id the chain entity id */ public async getOneEntity(chain: string, id: string): Promise { - const [entities, entityMetas] = await Promise.all([ - getFetch(`${app.serverUrl()}/ce/entities`, { chain, id }), - getFetch(`${app.serverUrl()}/getEntityMeta`, { chain, ce_id: id }), - ]); - const data = ChainEntityController._formatEntitiesWithMeta( - entities, - entityMetas - ); - if (data?.length > 1) { - throw new Error('Found multiple entities with same id!'); - } + const entities = await getFetch(`${app.serverUrl()}/ce/entities`, { + chain, + id, + }); + const data = ChainEntityController._formatEntities(entities); return data ? data[0] : null; } @@ -141,14 +121,8 @@ class ChainEntityController { const options: any = { chain }; // load the chain-entity objects - const [entities, entityMetas] = await Promise.all([ - getFetch(`${app.serverUrl()}/ce/entities`, options), - getFetch(`${app.serverUrl()}/getEntityMeta`, options), - ]); - const data = ChainEntityController._formatEntitiesWithMeta( - entities, - entityMetas - ); + const entities = await getFetch(`${app.serverUrl()}/ce/entities`, options); + const data = ChainEntityController._formatEntities(entities); this._store.set(chain, data); return data; } diff --git a/packages/commonwealth/client/scripts/models/ChainEntity.ts b/packages/commonwealth/client/scripts/models/ChainEntity.ts index b21f5e725a2..f503aa95b73 100644 --- a/packages/commonwealth/client/scripts/models/ChainEntity.ts +++ b/packages/commonwealth/client/scripts/models/ChainEntity.ts @@ -6,19 +6,15 @@ class ChainEntity { public readonly chain: string; public readonly type: IChainEntityKind; public readonly typeId: string; - public readonly author: string; - public readonly threadTitle?: string; public readonly createdAt?: moment.Moment; public readonly completed?: boolean; + public readonly author?: string; // This id is the chain-events service chain-entity id -> equivalent to ce_id in ChainEntityMeta in main service // This id is only available when the chain-entity is loaded from the server and NOT from the chain public readonly id?: number; - // these values cannot be readonly because they are updated when the chain-entity metadata is added to the instance - public title?: string; - private _updatedAt?: moment.Moment; public get updatedAt() { return this._updatedAt; @@ -47,8 +43,6 @@ class ChainEntity { createdAt, updatedAt, id, - threadTitle, - title, author, completed, }: { @@ -58,18 +52,14 @@ class ChainEntity { chainEvents: any[]; createdAt: moment.MomentInput; updatedAt: moment.MomentInput; - threadTitle: string; author: string; id?: number; - title?: string; completed?: boolean; }) { this.id = id; this.chain = chain; this.type = type; this.typeId = typeId; - this.threadTitle = decodeURIComponent(threadTitle); - this.title = title; this.author = author; this.createdAt = moment(createdAt); this.completed = completed; @@ -93,8 +83,6 @@ class ChainEntity { created_at, updated_at, id, - Thread, - title, author, completed, } = json; @@ -107,8 +95,6 @@ class ChainEntity { createdAt: created_at, updatedAt: updated_at, id, - threadTitle: Thread?.title, - title, author, completed, }); diff --git a/packages/commonwealth/client/scripts/models/Proposal.ts b/packages/commonwealth/client/scripts/models/Proposal.ts index a15d655b05d..c3e3b14d9ef 100644 --- a/packages/commonwealth/client/scripts/models/Proposal.ts +++ b/packages/commonwealth/client/scripts/models/Proposal.ts @@ -38,7 +38,6 @@ abstract class Proposal< public createdAt: moment.Moment; public threadId: number; - public threadTitle: string; public abstract title: string; diff --git a/packages/commonwealth/client/scripts/models/Thread.ts b/packages/commonwealth/client/scripts/models/Thread.ts index 01e613afb83..7c0077520fe 100644 --- a/packages/commonwealth/client/scripts/models/Thread.ts +++ b/packages/commonwealth/client/scripts/models/Thread.ts @@ -41,19 +41,8 @@ function processVersionHistory(versionHistory: any[]) { return versionHistoryProcessed; } -function processChainEntities(chain: string, chainEntityMeta: any) { +function processChainEntities(chain: string) { const chainEntitiesProcessed: ChainEntity[] = []; - if (chainEntityMeta) { - for (const meta of chainEntityMeta) { - const full_entity = Array.from(app.chainEntities.store.values()) - .flat() - .filter((e) => e.id === meta.ce_id)[0]; - if (full_entity) { - if (meta.title) full_entity.title = meta.title; - chainEntitiesProcessed.push(full_entity); - } - } - } return chainEntitiesProcessed ? chainEntitiesProcessed.map((ce) => { @@ -193,7 +182,6 @@ export class Thread implements IUniqueId { url, pinned, collaborators, - chain_entity_meta, last_edited, marked_as_spam_at, locked_at, @@ -240,7 +228,6 @@ export class Thread implements IUniqueId { reactionIds: any[]; // TODO: fix type addressesReacted: any[]; //TODO: fix type, reactionType: any[]; // TODO: fix type - chain_entity_meta: any; // TODO: fix type version_history: any[]; // TODO: fix type Address: any; // TODO: fix type discord_meta?: any; @@ -279,7 +266,7 @@ export class Thread implements IUniqueId { this.links = links || []; this.discord_meta = discord_meta; this.versionHistory = processVersionHistory(version_history); - this.chainEntities = processChainEntities(chain, chain_entity_meta); + this.chainEntities = processChainEntities(chain); this.associatedReactions = processAssociatedReactions( reactions, reactionIds, diff --git a/packages/commonwealth/client/scripts/views/components/ChainEntitiesSelector/utils.ts b/packages/commonwealth/client/scripts/views/components/ChainEntitiesSelector/utils.ts index efaa4f5ce63..ca3f04fe21b 100644 --- a/packages/commonwealth/client/scripts/views/components/ChainEntitiesSelector/utils.ts +++ b/packages/commonwealth/client/scripts/views/components/ChainEntitiesSelector/utils.ts @@ -8,7 +8,6 @@ export const filterChainEntities = (ce: ChainEntity, searchTerm: string) => { return ( ce.typeId.toString().toLowerCase().includes(searchTerm.toLowerCase()) || - ce.title?.toString().toLowerCase().includes(searchTerm.toLowerCase()) || chainEntityTypeToProposalName(ce.type) .toLowerCase() .includes(searchTerm.toLowerCase()) diff --git a/packages/commonwealth/client/scripts/views/modals/update_proposal_status_modal.tsx b/packages/commonwealth/client/scripts/views/modals/update_proposal_status_modal.tsx index 4104584ee2d..302ee781d81 100644 --- a/packages/commonwealth/client/scripts/views/modals/update_proposal_status_modal.tsx +++ b/packages/commonwealth/client/scripts/views/modals/update_proposal_status_modal.tsx @@ -76,7 +76,7 @@ export const UpdateProposalStatusModal = ({ Array> >(getInitialSnapshots(thread)); const [tempProposals, setTempProposals] = useState< - Array> + Array> >(getInitialProposals(thread)); const [tempCosmosProposals, setTempCosmosProposals] = useState< Array> diff --git a/packages/commonwealth/client/scripts/views/pages/chain_entity_link_redirect.tsx b/packages/commonwealth/client/scripts/views/pages/chain_entity_link_redirect.tsx index a5317e1fb72..0e3dac2ca6d 100644 --- a/packages/commonwealth/client/scripts/views/pages/chain_entity_link_redirect.tsx +++ b/packages/commonwealth/client/scripts/views/pages/chain_entity_link_redirect.tsx @@ -23,10 +23,9 @@ export default function ChainEntityLinkRedirect({ const entity = await app.chainEntities.getOneEntity(scope, identifier); // 2. query data to construct new link - const { title, type, typeId } = entity; + const { type, typeId } = entity; const newLink = { source: 'proposal', - title, identifier: type === 'proposal' ? `${typeId}` diff --git a/packages/commonwealth/package.json b/packages/commonwealth/package.json index 77cadf325a9..7f32205e2a4 100644 --- a/packages/commonwealth/package.json +++ b/packages/commonwealth/package.json @@ -32,7 +32,6 @@ "test-api": "NODE_ENV=test nyc ts-mocha --project tsconfig.json './test/integration/api/**/*.spec.ts'", "test-suite": "NODE_ENV=test nyc ts-mocha --project tsconfig.json './test/**/*.spec.ts'", "test-consumer": "ts-mocha --project tsconfig.json test/systemTests/consumer.test.ts --timeout 20000", - "test-scripts": "ts-mocha --project tsconfig.json test/integration/enforceDataConsistency.spec.ts", "start": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.json server.ts", "start-consumer": "ts-node --project ./tsconfig.consumer.json server/CommonwealthConsumer/CommonwealthConsumer.ts run-as-script", "start-prerender": "ts-node --project tsconfig.json server/scripts/runPrerenderService.ts", @@ -62,8 +61,6 @@ "start-containers": "chmod +rx ./scripts/start-docker-containers.sh && ./scripts/start-docker-containers.sh", "stop-containers": "chmod +rx ./scripts/stop-docker-containers.sh && ./scripts/stop-docker-containers.sh", "dump-db-local": " pg_dump -U commonwealth --verbose --no-privileges --no-owner -f local_save.dump", - "sync-entities": "ts-node server/scripts/enforceDataConsistency.ts run-as-script $(heroku config:get DATABASE_URL -a chain-events)", - "sync-entities-local": "ts-node server/scripts/enforceDataConsistency.ts run-as-script 'postgresql://commonwealth:edgeware@localhost/commonwealth_chain_events'", "compress-images": "npx ts-node -T ./scripts/compressImages.ts", "gen-e2e": "npx playwright codegen", "e2e-start-server": "ETH_RPC=e2e-test yarn start", diff --git a/packages/commonwealth/server/CommonwealthConsumer/CommonwealthConsumer.ts b/packages/commonwealth/server/CommonwealthConsumer/CommonwealthConsumer.ts index 2e8430d2536..eb9c25207ee 100644 --- a/packages/commonwealth/server/CommonwealthConsumer/CommonwealthConsumer.ts +++ b/packages/commonwealth/server/CommonwealthConsumer/CommonwealthConsumer.ts @@ -10,7 +10,6 @@ import { RascalSubscriptions } from 'common-common/src/rabbitmq/types'; import Rollbar from 'rollbar'; import { RABBITMQ_URI, ROLLBAR_ENV, ROLLBAR_SERVER_TOKEN } from '../config'; import models from '../database'; -import { processChainEntityCUD } from './messageProcessors/chainEntityCUDQueue'; import { processChainEventNotificationsCUD } from './messageProcessors/chainEventNotificationsCUDQueue'; import { processSnapshotMessage } from './messageProcessors/snapshotConsumer'; import { RascalConfigServices } from 'common-common/src/rabbitmq/rabbitMQConfig'; @@ -52,12 +51,6 @@ export async function setupCommonwealthConsumer(): Promise { log, }; - const chainEntityCUDProcessorRmqSub: RabbitMQSubscription = { - messageProcessor: processChainEntityCUD, - subscriptionName: RascalSubscriptions.ChainEntityCUDMain, - msgProcessorContext: context, - }; - const ceNotifsCUDProcessorRmqSub: RabbitMQSubscription = { messageProcessor: processChainEventNotificationsCUD, subscriptionName: RascalSubscriptions.ChainEventNotificationsCUDMain, @@ -71,7 +64,6 @@ export async function setupCommonwealthConsumer(): Promise { }; const subscriptions: RabbitMQSubscription[] = [ - chainEntityCUDProcessorRmqSub, ceNotifsCUDProcessorRmqSub, snapshotEventProcessorRmqSub, ]; diff --git a/packages/commonwealth/server/CommonwealthConsumer/messageProcessors/chainEntityCUDQueue.ts b/packages/commonwealth/server/CommonwealthConsumer/messageProcessors/chainEntityCUDQueue.ts deleted file mode 100644 index 11efd76963a..00000000000 --- a/packages/commonwealth/server/CommonwealthConsumer/messageProcessors/chainEntityCUDQueue.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { RmqEntityCUD } from 'common-common/src/rabbitmq/types/chainEntityCUD'; -import type { DB } from '../../models'; - -export type Ithis = { - models: DB; -}; - -export async function processChainEntityCUD( - this: Ithis, - data: RmqEntityCUD.RmqMsgType -) { - RmqEntityCUD.checkMsgFormat(data); - await this.models.ChainEntityMeta.create({ - ce_id: data.ce_id, - chain: data.chain_id, - author: data.author, - type_id: data.entity_type_id, - }); -} diff --git a/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts b/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts index 642e866dabc..7169b4f919d 100644 --- a/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts +++ b/packages/commonwealth/server/controllers/server_threads_methods/get_bulk_threads.ts @@ -170,25 +170,11 @@ export async function __getBulkThreads( throw new ServerError('Could not fetch threads'); } - const processLinks = async (thread) => { - let chain_entity_meta = []; - if (thread.links) { - const ces = thread.links.filter((item) => item.source === 'proposal'); - if (ces.length > 0) { - chain_entity_meta = ces.map((ce: Link) => { - return { ce_id: parseInt(ce.identifier), title: ce.title }; - }); - } - } - return { chain_entity_meta }; - }; - // transform thread response let threads = responseThreads.map(async (t) => { const collaborators = JSON.parse(t.collaborators[0]).address?.length ? t.collaborators.map((c) => JSON.parse(c)) : []; - const { chain_entity_meta } = await processLinks(t); const last_edited = getLastEdited(t); @@ -209,7 +195,6 @@ export async function __getBulkThreads( locked_at: t.thread_locked, links: t.links, collaborators, - chain_entity_meta, has_poll: t.has_poll, last_commented_on: t.last_commented_on, plaintext: t.plaintext, diff --git a/packages/commonwealth/server/database.ts b/packages/commonwealth/server/database.ts index 5cffc8a2371..ac43eac31b3 100644 --- a/packages/commonwealth/server/database.ts +++ b/packages/commonwealth/server/database.ts @@ -5,7 +5,6 @@ import type { DB, Models } from './models'; import AddressFactory from './models/address'; import BanFactory from './models/ban'; import ChainFactory from './models/chain'; -import ChainEntityMetaFactory from './models/chain_entity_meta'; import ChainNodeFactory from './models/chain_node'; import CollaborationFactory from './models/collaboration'; import CommentFactory from './models/comment'; @@ -71,7 +70,6 @@ const models: Models = { Ban: BanFactory(sequelize, DataTypes), Chain: ChainFactory(sequelize, DataTypes), ChainNode: ChainNodeFactory(sequelize, DataTypes), - ChainEntityMeta: ChainEntityMetaFactory(sequelize, DataTypes), Collaboration: CollaborationFactory(sequelize, DataTypes), Contract: ContractFactory(sequelize, DataTypes), ContractAbi: ContractAbiFactory(sequelize, DataTypes), diff --git a/packages/commonwealth/server/migrations/20230525090836-remove-entity-meta.js b/packages/commonwealth/server/migrations/20230525090836-remove-entity-meta.js new file mode 100644 index 00000000000..401081bbb3f --- /dev/null +++ b/packages/commonwealth/server/migrations/20230525090836-remove-entity-meta.js @@ -0,0 +1,20 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.dropTable('ChainEntityMeta'); + }, + + down: async (queryInterface, Sequelize) => { + // can't restore data but can create bare-bones table + await queryInterface.createTable('ChainEntityMeta', { + id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true }, + ce_id: { type: Sequelize.INTEGER, allowNull: false, unique: true }, + title: { type: Sequelize.STRING, allowNull: true }, + chain: { type: Sequelize.STRING, allowNull: false }, + author: { type: Sequelize.STRING, allowNull: true }, + type_id: { type: Sequelize.STRING, allowNull: true }, + project_chain: { type: Sequelize.STRING, allowNull: true }, + }); + }, +}; diff --git a/packages/commonwealth/server/models.ts b/packages/commonwealth/server/models.ts index afce7ffa239..bd0aa3ac511 100644 --- a/packages/commonwealth/server/models.ts +++ b/packages/commonwealth/server/models.ts @@ -3,7 +3,6 @@ import type { Sequelize } from 'sequelize'; import type { AddressModelStatic } from './models/address'; import type { BanModelStatic } from './models/ban'; import type { ChainModelStatic } from './models/chain'; -import type { ChainEntityMetaModelStatic } from './models/chain_entity_meta'; import type { ChainNodeModelStatic } from './models/chain_node'; import type { CollaborationModelStatic } from './models/collaboration'; import type { CommentModelStatic } from './models/comment'; @@ -41,7 +40,6 @@ export type Models = { Address: AddressModelStatic; Ban: BanModelStatic; Chain: ChainModelStatic; - ChainEntityMeta: ChainEntityMetaModelStatic; ChainNode: ChainNodeModelStatic; Contract: ContractModelStatic; ContractAbi: ContractAbiModelStatic; diff --git a/packages/commonwealth/server/models/chain.ts b/packages/commonwealth/server/models/chain.ts index 9b5ebc4bc56..bdb4c2677d2 100644 --- a/packages/commonwealth/server/models/chain.ts +++ b/packages/commonwealth/server/models/chain.ts @@ -166,7 +166,6 @@ export default ( models.Chain.belongsToMany(models.Contract, { through: models.CommunityContract, }); - models.Chain.hasMany(models.ChainEntityMeta, { foreignKey: 'chain' }); }; return Chain; diff --git a/packages/commonwealth/server/models/chain_entity_meta.ts b/packages/commonwealth/server/models/chain_entity_meta.ts deleted file mode 100644 index 9f1a0bf5837..00000000000 --- a/packages/commonwealth/server/models/chain_entity_meta.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type * as Sequelize from 'sequelize'; -import type { DataTypes } from 'sequelize'; -import type { ChainAttributes } from './chain'; -import type { ModelInstance, ModelStatic } from './types'; - -export type ChainEntityMetaAttributes = { - id: number; // sequelize auto-generated primary key id --- NEVER USE DIRECTLY - // this is the primary key id from the chain-events service - // (used to match chain-entity-meta with chain-entities from chain-events) - ce_id: number; - title?: string; - chain: string; - author?: string; - type_id?: string; - project_chain?: string; - - Chain?: ChainAttributes; - ProjectChain?: ChainAttributes; -}; - -export type ChainEntityMetaInstance = ModelInstance; - -export type ChainEntityMetaModelStatic = ModelStatic; - -export default ( - sequelize: Sequelize.Sequelize, - dataTypes: typeof DataTypes -): ChainEntityMetaModelStatic => { - const ChainEntityMeta = sequelize.define< - ChainEntityMetaInstance, - ChainEntityMetaAttributes - >( - 'ChainEntityMeta', - { - id: { type: dataTypes.INTEGER, primaryKey: true, autoIncrement: true }, - ce_id: { type: dataTypes.INTEGER, allowNull: false, unique: true }, - title: { type: dataTypes.STRING, allowNull: true }, - chain: { type: dataTypes.STRING, allowNull: false }, - author: { type: dataTypes.STRING, allowNull: true }, - type_id: { type: dataTypes.STRING, allowNull: true }, - project_chain: { type: dataTypes.STRING, allowNull: true }, - }, - { - tableName: 'ChainEntityMeta', - timestamps: false, - underscored: true, - paranoid: false, - indexes: [{ fields: ['id'] }], - } - ); - - ChainEntityMeta.associate = (models) => { - models.ChainEntityMeta.belongsTo(models.Chain, { - foreignKey: 'chain', - targetKey: 'id', - }); - models.ChainEntityMeta.belongsTo(models.Chain, { - foreignKey: 'project_chain', - targetKey: 'id', - as: 'ProjectChain', - }); - }; - - return ChainEntityMeta; -}; diff --git a/packages/commonwealth/server/models/thread.ts b/packages/commonwealth/server/models/thread.ts index 16c3ae6e6a9..901f896984b 100644 --- a/packages/commonwealth/server/models/thread.ts +++ b/packages/commonwealth/server/models/thread.ts @@ -183,9 +183,6 @@ export default ( models.Thread.hasMany(models.Notification, { foreignKey: 'thread_id', }); - models.Thread.hasOne(models.ChainEntityMeta, { - foreignKey: 'id', - }); }; return Thread; diff --git a/packages/commonwealth/server/routes/deleteChain.ts b/packages/commonwealth/server/routes/deleteChain.ts index 62c70071c1d..55ae929f5f4 100644 --- a/packages/commonwealth/server/routes/deleteChain.ts +++ b/packages/commonwealth/server/routes/deleteChain.ts @@ -62,12 +62,6 @@ const deleteChain = async ( await new Promise(async (resolve, reject) => { try { await models.sequelize.transaction(async (t) => { - // TODO: need a parallel API call to chain-events to destroy chain-entities there too - await models.ChainEntityMeta.destroy({ - where: { chain: chain.id }, - transaction: t, - }); - await models.User.update( { selected_chain_id: null, diff --git a/packages/commonwealth/server/routes/fetchEntityTitle.ts b/packages/commonwealth/server/routes/fetchEntityTitle.ts deleted file mode 100644 index 520193483e1..00000000000 --- a/packages/commonwealth/server/routes/fetchEntityTitle.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { Request, Response } from 'express'; -import type { DB } from '../models'; - -export const Errors = { - NoEntity: 'Cannot find entity', -}; - -const fetchEntityTitle = async (models: DB, req: Request, res: Response) => { - const { chain_entity_id } = req.query; - - const entityMeta = await models.ChainEntityMeta.findOne({ - where: { - ce_id: chain_entity_id, - }, - }); - if (!entityMeta) - return res.json({ status: 'Failure', message: Errors.NoEntity }); - - return res.json({ status: 'Success', result: entityMeta.title || '' }); -}; - -export default fetchEntityTitle; diff --git a/packages/commonwealth/server/routes/getEntityMeta.ts b/packages/commonwealth/server/routes/getEntityMeta.ts deleted file mode 100644 index d80dcfd7655..00000000000 --- a/packages/commonwealth/server/routes/getEntityMeta.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { AppError } from 'common-common/src/errors'; -import type { NextFunction, Request, Response } from 'express'; -import type { WhereOptions } from 'sequelize/types'; -import type { ChainEntityMetaAttributes } from 'server/models/chain_entity_meta'; -import type { DB } from '../models'; - -export const Errors = { - NeedChain: 'Must provide a chain to fetch entities from', - InvalidChain: 'Invalid chain', -}; - -const getEntityMeta = async ( - models: DB, - req: Request, - res: Response, - next: NextFunction -) => { - if (!req.query.chain) { - return next(new AppError(Errors.NeedChain)); - } - - const chain = await models.Chain.findOne({ - where: { id: req.query.chain }, - }); - - if (!chain) { - return next(new AppError(Errors.InvalidChain)); - } - - const entityMetaWhereOptions: WhereOptions = { - chain: chain.id, - }; - if (req.query.id) { - entityMetaWhereOptions.id = req.query.id; - } - if (req.query.type_id) { - entityMetaWhereOptions.type_id = req.query.type_id; - } - const entityMeta = await models.ChainEntityMeta.findAll({ - where: entityMetaWhereOptions, - }); - return res.json({ - status: 'Success', - result: entityMeta.map((e) => e.toJSON()), - }); -}; - -export default getEntityMeta; diff --git a/packages/commonwealth/server/routes/updateChainEntityTitle.ts b/packages/commonwealth/server/routes/updateChainEntityTitle.ts deleted file mode 100644 index 0528a0d285d..00000000000 --- a/packages/commonwealth/server/routes/updateChainEntityTitle.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { AppError } from 'common-common/src/errors'; -import type { NextFunction, Request, Response } from 'express'; -import { Op } from 'sequelize'; -import type { DB } from '../models'; -import { findAllRoles } from '../util/roles'; - -export const Errors = { - NoEntity: 'Cannot find entity', - NotAdminOrOwner: 'Not an admin or owner of this entity', -}; - -const updateChainEntityTitle = async ( - models: DB, - req: Request, - res: Response, - next: NextFunction -) => { - const chain = req.chain; - - const { title, chain_entity_id } = req.body; - const entity = await models.ChainEntityMeta.findOne({ - where: { - ce_id: chain_entity_id, - }, - }); - if (!entity) return next(new AppError(Errors.NoEntity)); - const userOwnedAddressObjects = (await req.user.getAddresses()).filter( - (addr) => !!addr.verified - ); - const userOwnedAddresses = userOwnedAddressObjects.map( - (addr) => addr.address - ); - const userOwnedAddressIds = userOwnedAddressObjects.map((addr) => addr.id); - - if (!userOwnedAddresses.includes(entity.author)) { - const roles = await findAllRoles( - models, - { where: { address_id: { [Op.in]: userOwnedAddressIds } } }, - chain.id, - ['admin', 'moderator'] - ); - // If address does not belong to entity chain, return error - const role = roles.find((r) => { - return r.chain_id === entity.chain; - }); - if (!role) return next(new AppError(Errors.NotAdminOrOwner)); - } - - entity.title = title; - await entity.save(); - - return res.json({ status: 'Success', result: entity.toJSON() }); -}; - -export default updateChainEntityTitle; diff --git a/packages/commonwealth/server/routing/router.ts b/packages/commonwealth/server/routing/router.ts index bfc65f15f15..e53c7bb81d4 100644 --- a/packages/commonwealth/server/routing/router.ts +++ b/packages/commonwealth/server/routing/router.ts @@ -77,8 +77,6 @@ import updateThreadPrivacy from '../routes/updateThreadPrivacy'; import updateThreadPinned from '../routes/updateThreadPinned'; import updateVote from '../routes/updateVote'; import viewVotes from '../routes/viewVotes'; -import fetchEntityTitle from '../routes/fetchEntityTitle'; -import updateChainEntityTitle from '../routes/updateChainEntityTitle'; import addEditors, { addEditorValidation } from '../routes/addEditors'; import deleteEditors from '../routes/deleteEditors'; import deleteChain from '../routes/deleteChain'; @@ -108,7 +106,6 @@ import updateChainPriority from '../routes/updateChainPriority'; import startSsoLogin from '../routes/startSsoLogin'; import finishSsoLogin from '../routes/finishSsoLogin'; -import getEntityMeta from '../routes/getEntityMeta'; import getTokenForum from '../routes/getTokenForum'; import tokenBalance from '../routes/tokenBalance'; import bulkBalances from '../routes/bulkBalances'; @@ -518,12 +515,7 @@ function setupRouter( databaseValidationService.validateChain, viewVotes.bind(this, models) ); - registerRoute( - router, - 'get', - '/fetchEntityTitle', - fetchEntityTitle.bind(this, models) - ); + registerRoute( router, 'post', @@ -606,14 +598,6 @@ function setupRouter( deleteCommunityContractTemplateMetadata.bind(this, models) ); - registerRoute( - router, - 'post', - '/updateChainEntityTitle', - passport.authenticate('jwt', { session: false }), - databaseValidationService.validateChain, - updateChainEntityTitle.bind(this, models) - ); registerRoute( router, 'post', @@ -1335,13 +1319,6 @@ function setupRouter( // logout registerRoute(router, 'get', '/logout', logout.bind(this, models)); - registerRoute( - router, - 'get', - '/getEntityMeta', - getEntityMeta.bind(this, models) - ); - // snapshotAPI registerRoute( router, diff --git a/packages/commonwealth/server/scripts/enforceDataConsistency.ts b/packages/commonwealth/server/scripts/enforceDataConsistency.ts deleted file mode 100644 index 6624707aeb6..00000000000 --- a/packages/commonwealth/server/scripts/enforceDataConsistency.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { QueryTypes } from 'sequelize'; -import models from '../database'; - -/** - * This script is meant as the worst case scenario recovery tool or as a method of ensuring consistency without a trace - * of a doubt. From the script we can fully rehydrate any cross-server data that was lost. This script can be used in - * emergencies and/or as part of the regular release phase. - * - * To run this file as a script 2 arguments are needed. Example: - * `ts-node enforceDataConsistency.ts run-as-script [Chain-event DB URI]` - * - * WARNING: - * Requires having the dbLink extension created/enabled by a superuser on each of the involved databases - * `create extension dblink;` - * This script should be run while the queues are empty + the chain-event listeners are offline to avoid duplicate - * data errors. For example if this script inserts a chain-event-type that is also in a queue, the queue processor will - * throw when it attempts to insert and that message will be retried and eventually dead-lettered unnecessarily. - * - * @param ce_db_uri {string} The URI of the chain-events database to sync with - * @param enforceEntities {boolean} A boolean indicating whether chain-event-entities should be synced - */ -export async function enforceDataConsistency( - ce_db_uri: string, - enforceEntities = true -) { - // if the function is called with run-as-script i.e. yarn runEnforceDataConsistency ensure that CONFIRM=true is passed - if (process.argv[2] === 'run-as-script' && process.env.CONFIRM != 'true') { - console.warn( - 'This script makes changes to the database specified by the given database URI. If you are sure ' + - "you want to do this run this script again with the env var 'CONFIRM=true'" - ); - process.exit(0); - } - const chainEntitySyncQuery = ` - WITH existingCeIds AS (SELECT ce_id FROM "ChainEntityMeta") - INSERT INTO "ChainEntityMeta" (ce_id, chain, author, type_id) - SELECT "AllChainEntities".id as ce_id, chain, author, type_id - FROM dblink('${ce_db_uri}', 'SELECT id, chain, author, type_id FROM "ChainEntities";') as - "AllChainEntities"(id int, chain varchar(255), author varchar(255), type_id varchar(255)) - WHERE "AllChainEntities".id NOT IN (SELECT * FROM existingCeIds) - RETURNING ce_id; - `; - - await models.sequelize.transaction(async (t) => { - if (enforceEntities) { - const result = await models.sequelize.query(chainEntitySyncQuery, { - type: QueryTypes.INSERT, - raw: true, - transaction: t, - }); - console.log('ChainEventEntities synced:', result); - } - }); -} - -// enables running the enforceDataConsistency function as a standalone script -if (process.argv[2] == 'run-as-script') { - enforceDataConsistency(process.argv[3]) - .then(() => { - console.log('Successfully synced the databases'); - process.exit(0); - }) - .catch((e) => { - console.error(e); - process.exit(1); - }); -} diff --git a/packages/commonwealth/test/integration/enforceDataConsistency.spec.ts b/packages/commonwealth/test/integration/enforceDataConsistency.spec.ts deleted file mode 100644 index 8053ae9c005..00000000000 --- a/packages/commonwealth/test/integration/enforceDataConsistency.spec.ts +++ /dev/null @@ -1,41 +0,0 @@ -import chai from 'chai'; -import { DATABASE_URI as CE_DB_URI } from 'chain-events/services/config'; -import ceModels from 'chain-events/services/database/database'; -import { QueryTypes } from 'sequelize'; -import cwModels from '../../server/database'; -import { enforceDataConsistency } from '../../server/scripts/enforceDataConsistency'; - -const { assert } = chai; - -describe.skip('Tests for enforceDataConsistency script', () => { - it('Should copy chain-entities from the CE db to the main db', async () => { - const author = '0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266'; - const { id } = await ceModels.ChainEntity.create({ - chain: 'edgeware', - type: 'test-type', - type_id: '1', - created_at: new Date(), - updated_at: new Date(), - author, - completed: false, - queued: -1, - }); - - // run consistency script - await enforceDataConsistency(CE_DB_URI, true); - - // ensure type id has been transferred to cw db - const newResults = await cwModels.ChainEntityMeta.findAll({ - where: { ce_id: id }, - }); - assert.equal(newResults.length, 1); - assert.equal(newResults[0].author, author); - - await cwModels.ChainEntityMeta.destroy({ - where: { ce_id: id }, - }); - await ceModels.ChainEntity.destroy({ - where: { id }, - }); - }); -}); diff --git a/packages/token-balance-cache/test/evm.spec.ts b/packages/token-balance-cache/test/evm.spec.ts index b9e068b4324..1f58ca78ae3 100644 --- a/packages/token-balance-cache/test/evm.spec.ts +++ b/packages/token-balance-cache/test/evm.spec.ts @@ -1,44 +1,55 @@ import { assert, expect, use as chaiUse } from 'chai'; import { BalanceType } from 'common-common/src/types'; import { IChainNode } from '../src/types'; -import {TokenBalanceCache} from '../src/tbc'; +import { TokenBalanceCache } from '../src/tbc'; import { default as evmBalanceProvider } from '../src/providers/ethToken'; -import {ChainTesting} from '../../chain-events/chain-testing/sdk/chainTesting'; +import { ChainTesting } from '../../chain-events/chain-testing/sdk/chainTesting'; async function mockNodesProvider(): Promise { - return [ - { - id: 1, - url: 'http://127.0.0.1:8545', - eth_chain_id: 5555, - balance_type: BalanceType.Ethereum, - name: 'eth-token', - }, - ]; - } -const providers = [new evmBalanceProvider()] + return [ + { + id: 1, + url: 'http://127.0.0.1:8545', + eth_chain_id: 5555, + balance_type: BalanceType.Ethereum, + name: 'eth-token', + }, + ]; +} + +const providers = [new evmBalanceProvider()]; const tbc = new TokenBalanceCache(0, 0, providers, mockNodesProvider); -const testSDK = new ChainTesting("http://127.0.0.1:3000"); +const testSDK = new ChainTesting('http://127.0.0.1:3000'); describe('EVM Token BP unit tests', () => { - it('should return correct ERC20 balance', async () => { - await tbc.start(); - let amt = 10; - const accounts = await testSDK.getAccounts() - const token = "0x92D6C1e31e14520e676a687F0a93788B716BEff5" - await testSDK.getErc20(token, accounts[0],amt.toString()); - const balance = await tbc.getBalancesForAddresses(1, [accounts[0]], 'eth-token', {contractType: 'erc20', tokenAddress: token}); - amt *= 1e18; - assert.equal(balance.balances[accounts[0]], amt.toString()); - }); - - it('should return correct ERC721 balance', async () => { - const accounts = await testSDK.getAccounts() - const nft = await testSDK.deployNFT() - await nft.mint('137', 0); - const token = nft.address; - const balance = await tbc.getBalancesForAddresses(1, [accounts[0]], 'eth-token', {contractType: 'erc721', tokenAddress: token}); - assert.equal(balance.balances[accounts[0]], "1"); - }); -}); \ No newline at end of file + it('should return the correct ERC20 balance', async () => { + await tbc.start(); + let amt = 10; + const accounts = await testSDK.getAccounts(); + const token = '0x92D6C1e31e14520e676a687F0a93788B716BEff5'; + await testSDK.getErc20(token, accounts[0], amt.toString()); + const balance = await tbc.getBalancesForAddresses( + 1, + [accounts[0]], + 'eth-token', + { contractType: 'erc20', tokenAddress: token } + ); + amt *= 1e18; + assert.equal(balance.balances[accounts[0]], amt.toString()); + }); + + it('should return the correct ERC721 balance', async () => { + const accounts = await testSDK.getAccounts(); + const nft = await testSDK.deployNFT(); + await nft.mint('137', 0); + const token = nft.address; + const balance = await tbc.getBalancesForAddresses( + 1, + [accounts[0]], + 'eth-token', + { contractType: 'erc721', tokenAddress: token } + ); + assert.equal(balance.balances[accounts[0]], '1'); + }); +});