feat(persistence-fabric): rewrite the plugin

- Rewrite fabric persistence plugin to match persistence ethereum plugin methods and behavior. - Change DB schema for persistence. - Add tests and sample data to the new plugin. Add them to project CI. Depends on hyperledger-cacti#3298 Signed-off-by: Michal Bajer <michal.bajer@fujitsu.com>
outSH · Jul 18, 2024 · aacae5e · aacae5e
1 parent 497ea32
commit aacae5e
Show file tree

Hide file tree

Showing 34 changed files with 4,645 additions and 1,669 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
diff --git a/packages/cactus-plugin-persistence-fabric/CHANGELOG.md b/packages/cactus-plugin-persistence-fabric/CHANGELOG.md
diff --git a/packages/cactus-plugin-persistence-fabric/Dockerfile b/packages/cactus-plugin-persistence-fabric/Dockerfile
@@ -0,0 +1,14 @@
+FROM node:16
+
+ENV PACKAGE_PATH=/opt/cactus-plugin-persistence-fabric
+
+WORKDIR ${PACKAGE_PATH}
+
+# CMake is required by one of npm dependencies (install other packages in this step as well in the future)
+RUN apt-get update && apt-get install -y cmake && rm -rf /var/lib/apt/lists/*
+
+COPY ./dist/yarn.lock ./package.json ./
+RUN yarn install --production --ignore-engines --non-interactive --cache-folder ./.yarnCache && \
+    rm -rf ./.yarnCache
+
+COPY ./dist ./dist
diff --git a/packages/cactus-plugin-persistence-fabric/README.md b/packages/cactus-plugin-persistence-fabric/README.md
@@ -1,9 +1,7 @@
 # `@hyperledger/cactus-plugin-persistence-fabric`
 
-This plugin allows `Cacti` to persist Fabric Block general data and basic information about transactions into some storage (currently to a `PostgreSQL` database, but this concept can be extended further).
+This plugin allows `Cacti` to persist Hyperledger Fabric data into some storage (currently to a `PostgreSQL` database, but this concept can be extended further).
 Data in the database can later be analyzed and viewed in a GUI tool.
-GUI tool is in project root directory of Cacti project in GUI folder. cacti/packages/cacti-ledger-browser
-
 
 ## Summary
 
@@ -17,9 +15,9 @@ GUI tool is in project root directory of Cacti project in GUI folder. cacti/pack
 
 ## Remarks
 
-- This plugin was only tested with small test ledgers. Running it to synchronize with old ledgers will take a lot of time.
+- This plugin was only tested with small Fabric ledgers. Running it to archive and monitor large ledgers is not recommended yet.
 - For now, the database schema is not considered public and can change over time (i.e., writing own application that reads data directly from the database is discouraged).
-- All the methods must be called directly on the plugin instance for now.
+- Only `status` endpoint is available, all the methods must be called directly on the plugin instance for now.
 
 ## Getting Started
 
@@ -35,65 +33,88 @@ yarn run configure
 
 ### Usage
 
-Instantiate a new `PluginPersistenceFabrickBlock` instance:
+Instantiate a new `PluginPersistenceFabric` instance:
 
 ```typescript
-import { PluginPersistenceFabric } from "../../../main/typescript/plugin-fabric-persistence-block";
+import { PluginPersistenceFabric } from "@hyperledger/cactus-plugin-persistence-fabric";
 import { v4 as uuidv4 } from "uuid";
 
-PluginInstance = new PluginPersistenceFabric({
-  gatewayOptions,
+const persistencePlugin = new PluginPersistenceFabric({
   apiClient,
-  logLevel: testLogLevel,
-  instanceId: uuidv4(),
-  connectionString:
-    "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",
+  logLevel: "info",
+  instanceId,
+  connectionString: "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",,
+  channelName: "mychannel",
+  gatewayOptions: {
+    identity: signingCredential.keychainRef,
+    wallet: {
+      keychain: signingCredential,
+    },
+  },
 });
 
 // Initialize the connection to the DB
-PluginInstance.onPluginInit();
+await persistencePlugin.onPluginInit();
 ```
 
-Alternatively, import `PluginPersistenceFabric` from the plugin package and use it to create a plugin.
+Alternatively, import `PluginFactoryLedgerPersistence` from the plugin package and use it to create a plugin.
 
 ```typescript
-import { PluginPersistenceFabric } from "@hyperledger/plugin-fabric-persistence-block";
+import { PluginFactoryLedgerPersistence } from "@hyperledger/cactus-plugin-persistence-fabric";
 import { PluginImportType } from "@hyperledger/cactus-core-api";
 import { v4 as uuidv4 } from "uuid";
 
-const factory = new PluginFactoryPersistenceFabricBlock({
+const factory = new PluginFactoryLedgerPersistence({
   pluginImportType: PluginImportType.Local,
 });
 
 const persistencePlugin = await factory.create({
-  instanceId: uuidv4(),
-  apiClient: new SocketIOApiClient(apiConfigOptions),
+  apiClient,
   logLevel: "info",
-  connectionString:
-    "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",
+  instanceId,
+  connectionString: "postgresql://postgres:your-super-secret-and-long-postgres-password@localhost:5432/postgres",,
+  channelName: "mychannel",
+  gatewayOptions: {
+    identity: signingCredential.keychainRef,
+    wallet: {
+      keychain: signingCredential,
+    },
+  },
 });
 
 // Initialize the connection to the DB
-persistencePlugin.onPluginInit();
+await persistencePlugin.onPluginInit();
 ```
 
-onPluginInit it also creates database structure if this is first time run
+You can use the persistent plugin to synchronize ledger state with the database. Here is a sample script that starts monitoring for new blocks:
 
-// Start synchronization with ledger.
-// To synchronize ledger
-
-you individually
+```typescript
+// Start monitoring new blocks.
+// Entire ledger is synchronized first with the DB (`syncAll` is called) so this operation can take a while on large ledgers!
+persistencePlugin.startMonitor((err) => {
+  reject(err);
+});
 
 // Show current status of the plugin
-PluginInstance.getStatus();
-
+persistencePlugin.getStatus();
 ```
 
+> See [plugin integration tests](./src/test/typescript/integration) for complete usage examples.
+
+### Building/running the container image locally
 
+In the Cacti project root say:
+
+```sh
+DOCKER_BUILDKIT=1 docker build ./packages/cactus-plugin-persistence-fabric/ -f ./packages/cactus-plugin-persistence-fabric/Dockerfile -t cactus-plugin-persistence-fabric
 ```
 
 ## Endpoints
 
+### StatusV1 (`/api/v1/plugins/@hyperledger/cactus-plugin-persistence-fabric/status`)
+
+- Returns status of the plugin (latest block read, failed blocks, is monitor running, etc...)
+
 ### Plugin Methods
 
 - Most of the plugin functionalities are currently not available through OpenAPI interface, please use direct method calls instead.
@@ -110,42 +131,28 @@ PluginInstance.getStatus();
 
 - Get status report of this instance of persistence plugin.
 
-#### `constinousBlocksSynchronization`
-
-- Start the block synchronization process. New blocks from the ledger will be parsed and pushed to the database.
-
-#### `continueBlocksSynchronization`
-
-- Start the block synchronization process. New blocks from the ledger will be parsed and pushed to the database. Should be used periodically.
+#### `startMonitor`
 
-#### `changeSynchronization`
+- Start the block monitoring process. New blocks from the ledger will be parsed and pushed to the database.
 
-- Stop the block synchronization process.
+#### `stopMonitor`
 
-#### `whichBlocksAreMissingInDdSimple`
+- Stop the block monitoring process.
 
-- Walk through all the blocks that could not be synchronized with the DB for some reasons and list them
+#### `syncFailedBlocks`
 
-#### `synchronizeOnlyMissedBlocks`
+- Walk through all the blocks that could not be synchronized with the DB for some reasons and try pushing them again.
 
-- Walk through all the blocks that are listed as not be synchronized with the DB for some reasons and try push them into DB from ledger.
-- can try many times to use this
+#### `syncAll`
 
-### `setLastBlockConsidered`
-
-- set the last block in ledger which we consider valid by our party and synchronize only to this point in ledger
-  If some blocks above this number are already in database they will not be removed.
-
-#### `initialBlocksSynchronization`
-
-- Synchronize entire first N number of blocks of ledger state with the database. It is a good start and easy to check if everything is correctly set.
+- Synchronize entire ledger state with the database.
 
 ## Running the tests
 
-To run all the tests for this persistence fabric plugin to ensure it's working correctly execute the following from the root of the `cactus` project:
+To run all the tests for this persistence plugin to ensure it's working correctly execute the following from the root of the `cactus` project:
 
 ```sh
-npx jest cactus-plugin-fabric-persistence-block
+npx jest cactus-plugin-persistence-fabric
 ```
 
 ## Contributing
@@ -156,23 +163,19 @@ Please review [CONTRIBUTING.md](../../CONTRIBUTING.md) to get started.
 
 ### Quick plugin project walkthrough
 
-#### ./src/main/json/contract_abi
-
-- Contains reference token ABIs used to call and identify token transfers.
-
 #### `./src/main/json/openapi.json`
 
 - Contains OpenAPI definition.
 
 #### `./src/main/sql/schema.sql`
 
-- Database schema for Ethereum data.
+- Database schema for Fabric data.
 
 #### `./src/main/typescript/web-services`
 
-- Folder that contains web service endpoint definitions if present.
+- Folder that contains web service endpoint definitions.
 
-#### `./plugin-fabric-persistence-block`
+#### `./plugin-persistence-fabric`
 
 - Main persistent plugin logic file
 

diff --git a/packages/cactus-plugin-persistence-fabric/package.json b/packages/cactus-plugin-persistence-fabric/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@hyperledger/cactus-plugin-persistence-fabric",
   "version": "2.0.0-rc.2",
-  "description": "A web service plugin that provides management capabilities on a Cacti Plugin Fabric Block Persistence as a scrapper of blocks.",
+  "description": "Persistence plugin for Fabric ledgers to store data into a database.",
   "keywords": [
     "Hyperledger",
     "Cacti",
@@ -25,69 +25,55 @@
   },
   "contributors": [
     {
-      "name": "Barnaba Pawelczak",
-      "email": "barnaba.pawelczak@fujitsu.com",
-      "url": "https://fujitsu.com"
+      "name": "Please add yourself to the list of contributors",
+      "email": "your.name@example.com",
+      "url": "https://example.com"
     },
     {
       "name": "Michal Bajer",
       "email": "michal.bajer@fujitsu.com",
-      "url": "https://fujitsu.com"
-    },
-    {
-      "name": "Eryk Baranowski",
-      "email": "Eryk.Baranowski@fujitsu.com",
-      "url": "https://fujitsu.com"
+      "url": "https://www.fujitsu.com/global/"
     },
     {
       "name": "Tomasz Awramski",
       "email": "tomasz.awramski@fujitsu.com",
-      "url": "https://fujitsu.com"
+      "url": "https://www.fujitsu.com/global/"
     }
   ],
   "main": "dist/lib/main/typescript/index.js",
   "module": "dist/lib/main/typescript/index.js",
-  "browser": "dist/cactus-plugin-persistence-fabric.web.umd.js",
   "types": "dist/lib/main/typescript/index.d.ts",
   "files": [
     "dist/*"
   ],
   "scripts": {
-    "build": "npm run build-ts && npm run build:dev:backend:postbuild",
+    "build": "npm run build-ts",
     "build-ts": "tsc",
-    "build:dev:backend:postbuild": "npm run copy-yarn-lock",
     "codegen": "run-p 'codegen:*'",
     "codegen:openapi": "npm run generate-sdk",
-    "copy-yarn-lock": "cp -af ../../yarn.lock ./dist/yarn.lock",
-    "generate-sdk": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g typescript-axios -o ./src/main/typescript/generated/openapi/typescript-axios/ --reserved-words-mappings protected=protected --ignore-file-override ../../openapi-generator-ignore",
-    "watch": "npm-watch",
-    "webpack": "npm-run-all webpack:dev",
-    "webpack:dev": "npm-run-all webpack:dev:node webpack:dev:web",
-    "webpack:dev:node": "webpack --env=dev --target=node --config ../../webpack.config.js",
-    "webpack:dev:web": "webpack --env=dev --target=web --config ../../webpack.config.js"
+    "generate-sdk": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g typescript-axios -o ./src/main/typescript/generated/openapi/typescript-axios/ --reserved-words-mappings protected=protected --ignore-file-override ../../openapi-generator-ignore"
   },
   "dependencies": {
     "@hyperledger/cactus-common": "2.0.0-rc.2",
     "@hyperledger/cactus-core": "2.0.0-rc.2",
     "@hyperledger/cactus-core-api": "2.0.0-rc.2",
-    "@hyperledger/cactus-plugin-keychain-memory": "2.0.0-rc.2",
     "@hyperledger/cactus-plugin-ledger-connector-fabric": "2.0.0-rc.2",
+    "async-mutex": "0.4.0",
     "axios": "1.7.2",
-    "fabric-protos": "2.2.20",
-    "js-sha256": "0.9.0",
     "pg": "8.8.0",
-    "safe-stable-stringify": "2.4.3",
-    "socket.io": "4.6.2"
+    "run-time-error-cjs": "1.4.0",
+    "uuid": "10.0.0"
   },
   "devDependencies": {
+    "@hyperledger/cactus-plugin-keychain-memory": "2.0.0-rc.2",
     "@hyperledger/cactus-test-tooling": "2.0.0-rc.2",
+    "@openapitools/openapi-generator-cli": "2.7.0",
     "@types/express": "4.17.21",
     "@types/pg": "8.6.5",
-    "@types/uuid": "10.0.0",
     "body-parser": "1.20.2",
     "express": "4.19.2",
-    "fabric-network": "2.2.20",
-    "uuid": "10.0.0"
+    "jest-extended": "4.0.1",
+    "socket.io": "4.6.2"
   },
   "engines": {
     "node": ">=18",
@@ -96,13 +82,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "browserMinified": "dist/cactus-plugin-persistence-fabric.web.umd.min.js",
-  "mainMinified": "dist/cactus-plugin-persistence-fabric.node.umd.min.js",
-  "watch": {
-    "codegen:openapi": {
-      "patterns": [
-        "./src/main/json/openapi.json"
-      ]
-    }
-  }
+  "watch": {}
 }