From 56b0203ab55c20e932fd69adde7789fabefecd0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20=C5=BBuk?= Date: Wed, 21 Aug 2024 15:21:19 +0200 Subject: [PATCH 1/3] Endpoint for fetching congress spendings as CSV --- config.development.js | 4 +- config.js | 1 + config.production.js | 4 +- config.staging.js | 2 +- package-lock.json | 11 ++++ package.json | 1 + src/routers/api.js | 27 ++++++++- src/utils/explorer.js | 2 +- src/utils/spendings.js | 128 +++++++++++++++++++++++++++++++++++++++++ 9 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 src/utils/spendings.js diff --git a/config.development.js b/config.development.js index 127d1f0..3883583 100644 --- a/config.development.js +++ b/config.development.js @@ -8,6 +8,6 @@ module.exports = { }, CENTRALIZED_API_URL: "http://localhost:8010", IS_LIVE: false, - EXPLORER_GRAPHQL: "http://localhost:3000", - ONBOARDER_PHRASE: 'REPLACE ME WITH AUTO LLD ONBOARDER ACCOUNT PHRASE' + ONBOARDER_PHRASE: 'REPLACE ME WITH AUTO LLD ONBOARDER ACCOUNT PHRASE', + EXPLORER_API_URL: "http://localhost:3000", }; diff --git a/config.js b/config.js index e9dd7df..8c0edea 100644 --- a/config.js +++ b/config.js @@ -14,6 +14,7 @@ const config = { LAND_NFTs_ID: 0, ONBOARDER_PHRASE: 'REPLACE ME WITH AUTO LLD ONBOARDER ACCOUNT PHRASE', CENTRALIZED_API_URL: "http://localhost:8010", + EXPLORER_API_URL: "http://localhost:3000", }; try { diff --git a/config.production.js b/config.production.js index fbb391f..ce76120 100644 --- a/config.production.js +++ b/config.production.js @@ -8,6 +8,6 @@ module.exports = { }, IS_LIVE: true, CENTRALIZED_API_URL: "https://api.liberland.org", - EXPLORER_GRAPHQL: "https://archive.mainnet.liberland.org", - ONBOARDER_PHRASE: 'REPLACE ME WITH AUTO LLD ONBOARDER ACCOUNT PHRASE' + ONBOARDER_PHRASE: 'REPLACE ME WITH AUTO LLD ONBOARDER ACCOUNT PHRASE', + EXPLORER_API_URL: "https://archive.mainnet.liberland.org", }; diff --git a/config.staging.js b/config.staging.js index 49d7b42..02aec1a 100644 --- a/config.staging.js +++ b/config.staging.js @@ -8,6 +8,6 @@ module.exports = { }, IS_LIVE: true, CENTRALIZED_API_URL: "https://staging.api.liberland.org", - EXPLORER_GRAPHQL: "https://archive.testchain.liberland.org/graphql", ONBOARDER_PHRASE: "REPLACE ME WITH AUTO LLD ONBOARDER ACCOUNT PHRASE", + EXPLORER_API_URL: "https://archive.testchain.liberland.org/graphql", }; diff --git a/package-lock.json b/package-lock.json index 891ffbc..352816e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@polkadot/util": "*11.1.2", "axios": "^1.6.2", "cors": "^2.8.5", + "csv-stringify": "^6.5.1", "debug": "^4.3.4", "ejs": "^3.1.9", "express": "^4.18.2", @@ -1385,6 +1386,11 @@ "node": ">= 8" } }, + "node_modules/csv-stringify": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.1.tgz", + "integrity": "sha512-+9lpZfwpLntpTIEpFbwQyWuW/hmI/eHuJZD1XzeZpfZTqkf1fyvBbBLXTJJMsBuuS11uTShMqPwzx4A6ffXgRQ==" + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -5956,6 +5962,11 @@ "which": "^2.0.1" } }, + "csv-stringify": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.1.tgz", + "integrity": "sha512-+9lpZfwpLntpTIEpFbwQyWuW/hmI/eHuJZD1XzeZpfZTqkf1fyvBbBLXTJJMsBuuS11uTShMqPwzx4A6ffXgRQ==" + }, "data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", diff --git a/package.json b/package.json index 10b8512..f5c5204 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@polkadot/util": "*11.1.2", "axios": "^1.6.2", "cors": "^2.8.5", + "csv-stringify": "^6.5.1", "debug": "^4.3.4", "ejs": "^3.1.9", "express": "^4.18.2", diff --git a/src/routers/api.js b/src/routers/api.js index c34c6a4..8e163ec 100644 --- a/src/routers/api.js +++ b/src/routers/api.js @@ -6,8 +6,10 @@ const { ApiPromise, WsProvider, Keyring } = require("@polkadot/api"); const axios = require ('axios'); const {BN, BN_ONE, BN_ZERO, BN_MILLION} = require("@polkadot/util") const config = require("../../config"); -const generateCertificate = require('./generate-certificate'); +const generateCertificate = require('./generate-certificate') +const { fetchAllCongressSpendings } = require('../utils/spendings'); const { getLastWeekEraPaidEvents } = require("../utils/explorer"); +const { stringify } = require('csv-stringify/sync'); const provider = new WsProvider(config.RPC_NODE_URL); @@ -364,4 +366,25 @@ router.get( }) ); -module.exports = router; +router.get( + "/congress_spendings", + wrap(async (req, res) => { + try { + const allSpendings = await fetchAllCongressSpendings(); + const csv = stringify( + [ + ["Timestamp", "Block Number", "Recipient", "Asset", "Value", "Remark"], + ...allSpendings.map(v => [v.block.timestamp, v.block.number, v.toId, v.asset, v.value, v.remark]) + ] + ); + res + .set('Content-Disposition', 'attachment; filename="congress-spendings.csv"') + .status(200) + .send(csv) + } catch(e) { + res.status(400).json({ error: e.message }) + } + }) +) + +module.exports = router; \ No newline at end of file diff --git a/src/utils/explorer.js b/src/utils/explorer.js index f7445d5..5376a8f 100644 --- a/src/utils/explorer.js +++ b/src/utils/explorer.js @@ -21,7 +21,7 @@ query EraPaidEvents { `; const getApi = () => axios.create({ - baseURL: config.EXPLORER_GRAPHQL, + baseURL: config.EXPLORER_API_URL, }); diff --git a/src/utils/spendings.js b/src/utils/spendings.js new file mode 100644 index 0000000..4f10b7d --- /dev/null +++ b/src/utils/spendings.js @@ -0,0 +1,128 @@ +const config = require("../../config"); + +const CONGRESS_ADDRESS = "5EYCAe5g8CDuMsTief7QBxfvzDFEfws6ueXTUhsbx5V81nGH"; + +async function queryAllPages(query, variables, key) { + let allData = []; + let after = undefined; + while (true) { + const result = await fetch(config.EXPLORER_API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ query, variables: { after, ...variables } }), + }); + const data = await result.json(); + allData.push(data.data[key].nodes); + if (data.data[key].pageInfo.hasNextPage) { + after = data.data[key].pageInfo.endCursor; + } else { + break; + } + } + return allData; +} + +async function getLLMSpendings() { + const data = await queryAllPages( + ` + query LLM($after: Cursor, $userId: String) { + merits(first: 50, after: $after, filter: { fromId: { equalTo: $userId } }) { + nodes { + id + toId + value + remark + block { + number + timestamp + } + } + pageInfo { + hasNextPage, + endCursor + } + } + } + `, + { userId: CONGRESS_ADDRESS }, + "merits" + ); + return data.flat().map((v) => ({ asset: "LLM", ...v })); +} + +async function getAssetsSpendings() { + const data = await queryAllPages( + ` + query Assets($after: Cursor, $userId: String) { + assetTransfers(first: 50, after: $after, filter: { asset: { notEqualTo: "1" }, fromId: { equalTo: $userId } }) { + nodes { + id + asset + toId + value + remark + block { + number + timestamp + } + } + pageInfo { + hasNextPage, + endCursor + } + } + } + `, + { userId: CONGRESS_ADDRESS }, + "assetTransfers" + ); + return data.flat(); +} + +async function getLLDSpendings() { + const data = await queryAllPages( + ` + query LLD($after: Cursor, $userId: String) { + transfers(first: 50, after: $after, filter: { fromId: { equalTo: $userId } }) { + nodes { + id + toId + value + remark + block { + number + timestamp + } + } + pageInfo { + hasNextPage, + endCursor + } + } + } + `, + { userId: CONGRESS_ADDRESS }, + "transfers" + ); + return data.flat().map((v) => ({ asset: "LLD", ...v })); +} + +async function fetchAllCongressSpendings() { + const allSpendings = [ + await getLLDSpendings(), + await getLLMSpendings(), + await getAssetsSpendings(), + ] + .flat() + .sort((a, b) => + parseInt(a.block.number) > parseInt(b.block.number) ? -1 : 1 + ); + + return allSpendings; +} + +module.exports = { + fetchAllCongressSpendings, +}; From d11ffeb6c47cea8402c8d9d545e3ee1cff14935a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20=C5=BBuk?= Date: Fri, 27 Sep 2024 12:13:25 +0200 Subject: [PATCH 2/3] Parse new format of remarks --- src/routers/api.js | 93 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 14 deletions(-) diff --git a/src/routers/api.js b/src/routers/api.js index 8e163ec..1953df3 100644 --- a/src/routers/api.js +++ b/src/routers/api.js @@ -4,12 +4,13 @@ const router = require("express").Router(); const wrap = require("express-async-handler"); const { ApiPromise, WsProvider, Keyring } = require("@polkadot/api"); const axios = require ('axios'); -const {BN, BN_ONE, BN_ZERO, BN_MILLION} = require("@polkadot/util") +const {BN, BN_ONE, BN_ZERO, BN_MILLION, hexToU8a} = require("@polkadot/util") const config = require("../../config"); const generateCertificate = require('./generate-certificate') const { fetchAllCongressSpendings } = require('../utils/spendings'); const { getLastWeekEraPaidEvents } = require("../utils/explorer"); const { stringify } = require('csv-stringify/sync'); +const pako = require('pako'); const provider = new WsProvider(config.RPC_NODE_URL); @@ -29,6 +30,16 @@ const apiPromise = ApiPromise.create({ name: "Text", purpose: "Text", }, + RemarkInfo: { + category: 'Text', + project: 'Text', + supplier: 'Text', + description: 'Text', + finalDestination: 'Text', + amountInUSDAtDateOfPayment: 'u64', + date: 'u64', + currency: 'Text', + }, }, }); @@ -367,24 +378,78 @@ router.get( ); router.get( - "/congress_spendings", + "/congress-spendings", wrap(async (req, res) => { try { const allSpendings = await fetchAllCongressSpendings(); - const csv = stringify( + const api = await apiPromise; + const csv = stringify([ [ - ["Timestamp", "Block Number", "Recipient", "Asset", "Value", "Remark"], - ...allSpendings.map(v => [v.block.timestamp, v.block.number, v.toId, v.asset, v.value, v.remark]) - ] - ); - res - .set('Content-Disposition', 'attachment; filename="congress-spendings.csv"') + "Timestamp", + "Block Number", + "Recipient", + "Asset", + "Value", + "Category", + "Project", + "Supplier", + "Description", + "Final Destination", + "Amount In USD At Date Of Payment", + "Date", + "Currency", + "Text Remark", + "Raw Remark", + ], + ...allSpendings.map((v) => { + let parsedRemark; + let textRemark; + try { + if (v.remark) { + const compressedData = hexToU8a(v.remark); + const decompressed = pako.inflate(compressedData); + parsedRemark = api + .createType("RemarkInfo", decompressed) + .toJSON(); + parsedRemark.date = new Date( + parsedRemark.date + ).toISOString(); + } + } catch (e) { + textRemark = Buffer.from( + v.remark.substring(2), + "hex" + ).toString("utf-8"); + } + return [ + v.block.timestamp, + v.block.number, + v.toId, + v.asset, + v.value, + parsedRemark?.category ?? "-", + parsedRemark?.project ?? "-", + parsedRemark?.supplier ?? "-", + parsedRemark?.description ?? "-", + parsedRemark?.finalDestination ?? "-", + parsedRemark?.amountInUSDAtDateOfPayment ?? "-", + parsedRemark?.date ?? "-", + parsedRemark?.currency ?? "-", + textRemark ?? "-", + v.remark, + ]; + }), + ]); + res.set( + "Content-Disposition", + 'attachment; filename="congress-spendings.csv"' + ) .status(200) - .send(csv) - } catch(e) { - res.status(400).json({ error: e.message }) + .send(csv); + } catch (e) { + res.status(400).json({ error: e.message }); } }) -) +); -module.exports = router; \ No newline at end of file +module.exports = router; From e927d4fdd1bf36522ddfbeffafe9a61568fd7bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20=C5=BBuk?= Date: Tue, 1 Oct 2024 11:55:34 +0200 Subject: [PATCH 3/3] Update to new changes in main --- src/routers/api.js | 3 +- src/utils/explorer.js | 161 ++++++++++++++++++++++++++++++++++++----- src/utils/spendings.js | 128 -------------------------------- 3 files changed, 144 insertions(+), 148 deletions(-) delete mode 100644 src/utils/spendings.js diff --git a/src/routers/api.js b/src/routers/api.js index 1953df3..81d7bbb 100644 --- a/src/routers/api.js +++ b/src/routers/api.js @@ -7,8 +7,7 @@ const axios = require ('axios'); const {BN, BN_ONE, BN_ZERO, BN_MILLION, hexToU8a} = require("@polkadot/util") const config = require("../../config"); const generateCertificate = require('./generate-certificate') -const { fetchAllCongressSpendings } = require('../utils/spendings'); -const { getLastWeekEraPaidEvents } = require("../utils/explorer"); +const { getLastWeekEraPaidEvents, fetchAllCongressSpendings } = require("../utils/explorer"); const { stringify } = require('csv-stringify/sync'); const pako = require('pako'); diff --git a/src/utils/explorer.js b/src/utils/explorer.js index 5376a8f..594c45f 100644 --- a/src/utils/explorer.js +++ b/src/utils/explorer.js @@ -3,35 +3,160 @@ const axios = require("axios"); const config = require("../../config"); +const CONGRESS_ADDRESS = "5EYCAe5g8CDuMsTief7QBxfvzDFEfws6ueXTUhsbx5V81nGH"; + const eraPaidEventsQuery = ` query EraPaidEvents { - events( - orderBy: BLOCK_NUMBER_DESC, - first: 28, - filter: { - method: { equalTo: "EraPaid" }, - section: { equalTo: "staking" } - } - ) { - nodes { - data - } - } + events( + orderBy: BLOCK_NUMBER_DESC, + first: 28, + filter: { + method: { equalTo: "EraPaid" }, + section: { equalTo: "staking" } + } + ) { + nodes { + data + } + } } `; const getApi = () => axios.create({ - baseURL: config.EXPLORER_API_URL, + baseURL: config.EXPLORER_API_URL, }); const getLastWeekEraPaidEvents = async () => { - const { data } = await getApi().post('', { - query: eraPaidEventsQuery - }); - return data.data.events.nodes.map(v => v.data); + const { data } = await getApi().post('', { + query: eraPaidEventsQuery + }); + return data.data.events.nodes.map(v => v.data); }; + +async function queryAllPages(query, variables, key) { + let allData = []; + let after = undefined; + while (true) { + const result = await fetch(config.EXPLORER_API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ query, variables: { after, ...variables } }), + }); + const data = await result.json(); + allData.push(data.data[key].nodes); + if (data.data[key].pageInfo.hasNextPage) { + after = data.data[key].pageInfo.endCursor; + } else { + break; + } + } + return allData; +} + +async function getLLMSpendings() { + const data = await queryAllPages( + ` + query LLM($after: Cursor, $userId: String) { + merits(first: 50, after: $after, filter: { fromId: { equalTo: $userId } }) { + nodes { + id + toId + value + remark + block { + number + timestamp + } + } + pageInfo { + hasNextPage, + endCursor + } + } + } + `, + { userId: CONGRESS_ADDRESS }, + "merits" + ); + return data.flat().map((v) => ({ asset: "LLM", ...v })); +} + +async function getAssetsSpendings() { + const data = await queryAllPages( + ` + query Assets($after: Cursor, $userId: String) { + assetTransfers(first: 50, after: $after, filter: { asset: { notEqualTo: "1" }, fromId: { equalTo: $userId } }) { + nodes { + id + asset + toId + value + remark + block { + number + timestamp + } + } + pageInfo { + hasNextPage, + endCursor + } + } + } + `, + { userId: CONGRESS_ADDRESS }, + "assetTransfers" + ); + return data.flat(); +} + +async function getLLDSpendings() { + const data = await queryAllPages( + ` + query LLD($after: Cursor, $userId: String) { + transfers(first: 50, after: $after, filter: { fromId: { equalTo: $userId } }) { + nodes { + id + toId + value + remark + block { + number + timestamp + } + } + pageInfo { + hasNextPage, + endCursor + } + } + } + `, + { userId: CONGRESS_ADDRESS }, + "transfers" + ); + return data.flat().map((v) => ({ asset: "LLD", ...v })); +} + +async function fetchAllCongressSpendings() { + const allSpendings = [ + await getLLDSpendings(), + await getLLMSpendings(), + await getAssetsSpendings(), + ] + .flat() + .sort((a, b) => + parseInt(a.block.number) > parseInt(b.block.number) ? -1 : 1 + ); + + return allSpendings; +} + module.exports = { - getLastWeekEraPaidEvents, + fetchAllCongressSpendings, + getLastWeekEraPaidEvents, } \ No newline at end of file diff --git a/src/utils/spendings.js b/src/utils/spendings.js deleted file mode 100644 index 4f10b7d..0000000 --- a/src/utils/spendings.js +++ /dev/null @@ -1,128 +0,0 @@ -const config = require("../../config"); - -const CONGRESS_ADDRESS = "5EYCAe5g8CDuMsTief7QBxfvzDFEfws6ueXTUhsbx5V81nGH"; - -async function queryAllPages(query, variables, key) { - let allData = []; - let after = undefined; - while (true) { - const result = await fetch(config.EXPLORER_API_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ query, variables: { after, ...variables } }), - }); - const data = await result.json(); - allData.push(data.data[key].nodes); - if (data.data[key].pageInfo.hasNextPage) { - after = data.data[key].pageInfo.endCursor; - } else { - break; - } - } - return allData; -} - -async function getLLMSpendings() { - const data = await queryAllPages( - ` - query LLM($after: Cursor, $userId: String) { - merits(first: 50, after: $after, filter: { fromId: { equalTo: $userId } }) { - nodes { - id - toId - value - remark - block { - number - timestamp - } - } - pageInfo { - hasNextPage, - endCursor - } - } - } - `, - { userId: CONGRESS_ADDRESS }, - "merits" - ); - return data.flat().map((v) => ({ asset: "LLM", ...v })); -} - -async function getAssetsSpendings() { - const data = await queryAllPages( - ` - query Assets($after: Cursor, $userId: String) { - assetTransfers(first: 50, after: $after, filter: { asset: { notEqualTo: "1" }, fromId: { equalTo: $userId } }) { - nodes { - id - asset - toId - value - remark - block { - number - timestamp - } - } - pageInfo { - hasNextPage, - endCursor - } - } - } - `, - { userId: CONGRESS_ADDRESS }, - "assetTransfers" - ); - return data.flat(); -} - -async function getLLDSpendings() { - const data = await queryAllPages( - ` - query LLD($after: Cursor, $userId: String) { - transfers(first: 50, after: $after, filter: { fromId: { equalTo: $userId } }) { - nodes { - id - toId - value - remark - block { - number - timestamp - } - } - pageInfo { - hasNextPage, - endCursor - } - } - } - `, - { userId: CONGRESS_ADDRESS }, - "transfers" - ); - return data.flat().map((v) => ({ asset: "LLD", ...v })); -} - -async function fetchAllCongressSpendings() { - const allSpendings = [ - await getLLDSpendings(), - await getLLMSpendings(), - await getAssetsSpendings(), - ] - .flat() - .sort((a, b) => - parseInt(a.block.number) > parseInt(b.block.number) ? -1 : 1 - ); - - return allSpendings; -} - -module.exports = { - fetchAllCongressSpendings, -};