From 506dfc43af54e3220afe1c860e7297c996838df7 Mon Sep 17 00:00:00 2001 From: arobsn <87387688+arobsn@users.noreply.github.com> Date: Fri, 9 Aug 2024 08:56:10 -0300 Subject: [PATCH] query Addresses/ErgoTrees by chunks of 20 --- .changeset/unlucky-numbers-sort.md | 5 + .../ergo-graphql/ergoGraphQLProvider.spec.ts | 13 +- .../src/ergo-graphql/ergoGraphQLProvider.ts | 204 +++++++++--------- 3 files changed, 110 insertions(+), 112 deletions(-) create mode 100644 .changeset/unlucky-numbers-sort.md diff --git a/.changeset/unlucky-numbers-sort.md b/.changeset/unlucky-numbers-sort.md new file mode 100644 index 00000000..99bfb100 --- /dev/null +++ b/.changeset/unlucky-numbers-sort.md @@ -0,0 +1,5 @@ +--- +"@fleet-sdk/blockchain-providers": patch +--- + +[`ErgoGraphQLProvider`] Query Addresses/ErgoTrees by chunks of 20 diff --git a/packages/blockchain-providers/src/ergo-graphql/ergoGraphQLProvider.spec.ts b/packages/blockchain-providers/src/ergo-graphql/ergoGraphQLProvider.spec.ts index 6f9b7d8e..e3594895 100644 --- a/packages/blockchain-providers/src/ergo-graphql/ergoGraphQLProvider.spec.ts +++ b/packages/blockchain-providers/src/ergo-graphql/ergoGraphQLProvider.spec.ts @@ -63,7 +63,7 @@ describe("ergo-graphql provider", () => { await _client.getBoxes({ where: { - boxIds: ["boxId_1", "boxId_2"], + boxId: "boxId_1", ergoTrees: ["contract", "another_contract"] } }); @@ -71,7 +71,7 @@ describe("ergo-graphql provider", () => { call = JSON.parse(fetchSpy.mock.lastCall?.[1]?.body as string); expect(call.variables).to.be.deep.equal({ spent: false, - boxIds: ["boxId_1", "boxId_2"], + boxIds: ["boxId_1"], ergoTrees: ["contract", "another_contract"], skip: 0, take: 50 @@ -90,7 +90,6 @@ describe("ergo-graphql provider", () => { .setBigIntMapper((v) => Number(v)) .getBoxes({ where: { - boxIds: ["boxId_0", "boxId_1"], boxId: "boxId_0", ergoTrees: ["ergoTree_0", "ergoTree_1", "ergoTree_1"], ergoTree: "ergoTree_2" @@ -103,7 +102,7 @@ describe("ergo-graphql provider", () => { const call = JSON.parse(fetchSpy.mock.lastCall?.[1]?.body as string); expect(call.variables).to.be.deep.equal({ spent: false, - boxIds: ["boxId_0", "boxId_1"], + boxIds: ["boxId_0"], ergoTrees: ["ergoTree_0", "ergoTree_1", "ergoTree_2"], skip: 0, take: 50 @@ -451,7 +450,6 @@ describe("ergo-graphql provider", () => { const response = await _client.getConfirmedTransactions({ where: { transactionId: "txId", - transactionIds: ["txId_1", "txId_2", "txId_1"], address: _addresses[0].encode(), addresses: [_addresses[0], _addresses[1]], ergoTree: _addresses[1].ergoTree, @@ -469,7 +467,7 @@ describe("ergo-graphql provider", () => { let callBody = JSON.parse(fetchSpy.mock.lastCall?.[1]?.body as string); expect(callBody.query).to.be.equal(CONF_TX_QUERY); expect(callBody.variables).to.be.deep.equal({ - transactionIds: ["txId_1", "txId_2", "txId"], + transactionIds: ["txId"], addresses: [ _addresses[0].encode(), _addresses[1].encode(), @@ -565,8 +563,6 @@ describe("ergo-graphql provider", () => { const response = await _client.getUnconfirmedTransactions({ where: { - transactionId: "txId", - transactionIds: ["txId_1", "txId_2", "txId_1"], address: _addresses[0].encode(), addresses: [_addresses[0], _addresses[1]], ergoTree: _addresses[1].ergoTree, @@ -582,7 +578,6 @@ describe("ergo-graphql provider", () => { let callBody = JSON.parse(fetchSpy.mock.lastCall?.[1]?.body as string); expect(callBody.query).to.be.equal(UNCONF_TX_QUERY); expect(callBody.variables).to.be.deep.equal({ - transactionIds: ["txId_1", "txId_2", "txId"], addresses: [ _addresses[0].encode(), _addresses[1].encode(), diff --git a/packages/blockchain-providers/src/ergo-graphql/ergoGraphQLProvider.ts b/packages/blockchain-providers/src/ergo-graphql/ergoGraphQLProvider.ts index 4c3546cf..8ef04b8d 100644 --- a/packages/blockchain-providers/src/ergo-graphql/ergoGraphQLProvider.ts +++ b/packages/blockchain-providers/src/ergo-graphql/ergoGraphQLProvider.ts @@ -56,15 +56,9 @@ import { UNCONF_BOXES_QUERY, UNCONF_TX_QUERY } from "./queries"; - -type GraphQLThrowableOptions = GraphQLRequestOptions & { throwOnNonNetworkErrors: true }; -type OP = GraphQLOperation, V>; -type BiMapper = (value: string) => T; +import { chunk } from "packages/common/src"; export type GraphQLBoxWhere = BoxWhere & { - /** Base16-encoded BoxIds */ - boxIds?: HexString[]; - /** Base16-encoded ErgoTrees */ ergoTrees?: HexString[]; @@ -73,13 +67,11 @@ export type GraphQLBoxWhere = BoxWhere & { }; export type GraphQLConfirmedTransactionWhere = ConfirmedTransactionWhere & { - transactionIds?: HexString[]; addresses?: (Base58String | ErgoAddress)[]; ergoTrees?: HexString[]; }; export type GraphQLUnconfirmedTransactionWhere = UnconfirmedTransactionWhere & { - transactionIds?: HexString[]; addresses?: (Base58String | ErgoAddress)[]; ergoTrees?: HexString[]; }; @@ -100,7 +92,12 @@ type CheckTransactionResponse = { checkTransaction: string }; type TransactionSubmissionResponse = { submitTransaction: string }; type SignedTxArgsResp = { signedTransaction: SignedTransaction }; +type GraphQLThrowableOptions = GraphQLRequestOptions & { throwOnNonNetworkErrors: true }; +type OP = GraphQLOperation, V>; +type BiMapper = (value: string) => T; + const PAGE_SIZE = 50; +const MAX_ARGS = 20; export class ErgoGraphQLProvider implements IBlockchainProvider { #options: GraphQLThrowableOptions; @@ -116,15 +113,14 @@ export class ErgoGraphQLProvider implements IBlockchainProvider { #getHeaders!: OP; constructor(url: string); - constructor(url: ErgoGraphQLRequestOptions); + constructor(options: ErgoGraphQLRequestOptions); constructor(optOrUrl: ErgoGraphQLRequestOptions | string) { + this.#biMapper = (value) => BigInt(value) as I; this.#options = { ...(isRequestParam(optOrUrl) ? optOrUrl : { url: optOrUrl }), throwOnNonNetworkErrors: true }; - this.#biMapper = (value) => BigInt(value) as I; - this.#getConfirmedBoxes = this.createOperation(CONF_BOXES_QUERY); this.#getUnconfirmedBoxes = this.createOperation(UNCONF_BOXES_QUERY); this.#getAllBoxes = this.createOperation(ALL_BOXES_QUERY); @@ -137,10 +133,10 @@ export class ErgoGraphQLProvider implements IBlockchainProvider { #fetchBoxes(args: QueryBoxesArgs, inclConf: boolean, inclUnconf: boolean) { return inclConf && inclUnconf - ? this.#getAllBoxes(args, this.#options.url) + ? this.#getAllBoxes(args) : inclUnconf - ? this.#getUnconfirmedBoxes(args, this.#options.url) - : this.#getConfirmedBoxes(args, this.#options.url); + ? this.#getUnconfirmedBoxes(args) + : this.#getConfirmedBoxes(args); } setUrl(url: string): ErgoGraphQLProvider { @@ -161,55 +157,57 @@ export class ErgoGraphQLProvider implements IBlockchainProvider { const notBeingSpent = (box: GQLBox) => !box.beingSpent; const returnedBoxIds = new Set(); const { where, from } = query; - const args = buildGqlBoxQueryArgs(where); + const queries = buildGqlBoxQueries(where); + const isMempoolAware = from !== "blockchain"; - let inclChain = from !== "mempool"; - let inclPool = from !== "blockchain"; - const isMempoolAware = inclPool; + for (const query of queries) { + let inclChain = from !== "mempool"; + let inclPool = from !== "blockchain"; - do { - const { data } = await this.#fetchBoxes(args, inclChain, inclPool); - let boxes: ChainProviderBox[] = []; + while (inclChain || inclPool) { + const { data } = await this.#fetchBoxes(query, inclChain, inclPool); + let boxes: ChainProviderBox[] = []; - if (inclChain && hasConfirmed(data)) { - if (some(data.boxes)) { - const confirmedBoxes = ( - isMempoolAware ? data.boxes.filter(notBeingSpent) : data.boxes - ).map((b) => mapConfirmedBox(b, this.#biMapper)); + if (inclChain && hasConfirmed(data)) { + if (some(data.boxes)) { + const confirmedBoxes = ( + isMempoolAware ? data.boxes.filter(notBeingSpent) : data.boxes + ).map((b) => mapConfirmedBox(b, this.#biMapper)); - boxes = boxes.concat(confirmedBoxes); - } - - inclChain = data.boxes.length === PAGE_SIZE; - } + boxes = boxes.concat(confirmedBoxes); + } - if (isMempoolAware && hasMempool(data)) { - if (some(data.mempool.boxes)) { - const mempoolBoxes = data.mempool.boxes - .filter(notBeingSpent) - .map((b) => mapUnconfirmedBox(b, this.#biMapper)); - boxes = boxes.concat(mempoolBoxes); + inclChain = data.boxes.length === PAGE_SIZE; } - inclPool = data.mempool.boxes.length === PAGE_SIZE; - } + if (isMempoolAware && hasMempool(data)) { + if (some(data.mempool.boxes)) { + const mempoolBoxes = data.mempool.boxes + .filter(notBeingSpent) + .map((b) => mapUnconfirmedBox(b, this.#biMapper)); + boxes = boxes.concat(mempoolBoxes); + } - if (some(boxes)) { - // boxes can be moved from the mempool to the blockchain while streaming, - // so we need to filter out boxes that have already been returned. - if (boxes.some((box) => returnedBoxIds.has(box.boxId))) { - boxes = boxes.filter((b) => !returnedBoxIds.has(b.boxId)); + inclPool = data.mempool.boxes.length === PAGE_SIZE; } if (some(boxes)) { - boxes = uniqBy(boxes, (box) => box.boxId); - for (const box of boxes) returnedBoxIds.add(box.boxId); - yield boxes; + // boxes can be moved from the mempool to the blockchain while streaming, + // so we need to filter out boxes that have already been returned. + if (boxes.some((box) => returnedBoxIds.has(box.boxId))) { + boxes = boxes.filter((b) => !returnedBoxIds.has(b.boxId)); + } + + if (some(boxes)) { + boxes = uniqBy(boxes, (box) => box.boxId); + for (const box of boxes) returnedBoxIds.add(box.boxId); + yield boxes; + } } - } - if (inclChain || inclPool) args.skip += PAGE_SIZE; - } while (inclChain || inclPool); + if (inclChain || inclPool) query.skip += PAGE_SIZE; + } + } } async getBoxes(query: GraphQLBoxQuery): Promise[]> { @@ -221,19 +219,21 @@ export class ErgoGraphQLProvider implements IBlockchainProvider { async *streamUnconfirmedTransactions( query: TransactionQuery ): AsyncIterable[]> { - const args = buildGqlUnconfirmedTxQueryArgs(query.where); - - let keepFetching = true; - while (keepFetching) { - const response = await this.#getUnconfirmedTransactions(args); - if (some(response.data?.mempool?.transactions)) { - yield response.data.mempool.transactions.map((t) => - mapUnconfirmedTransaction(t, this.#biMapper) - ); - } + const queries = buildGqlUnconfirmedTxQueries(query.where); + + for (const query of queries) { + let keepFetching = true; + while (keepFetching) { + const response = await this.#getUnconfirmedTransactions(query); + if (some(response.data?.mempool?.transactions)) { + yield response.data.mempool.transactions.map((t) => + mapUnconfirmedTransaction(t, this.#biMapper) + ); + } - keepFetching = response.data?.mempool?.transactions?.length === PAGE_SIZE; - if (keepFetching) args.skip += PAGE_SIZE; + keepFetching = response.data?.mempool?.transactions?.length === PAGE_SIZE; + if (keepFetching) query.skip += PAGE_SIZE; + } } } @@ -251,19 +251,21 @@ export class ErgoGraphQLProvider implements IBlockchainProvider { async *streamConfirmedTransactions( query: TransactionQuery ): AsyncIterable[]> { - const args = buildGqlConfirmedTxQueryArgs(query.where); - - let keepFetching = true; - while (keepFetching) { - const response = await this.#getConfirmedTransactions(args); - if (some(response.data?.transactions)) { - yield response.data.transactions.map((t) => - mapConfirmedTransaction(t, this.#biMapper) - ); - } + const queries = buildGqlConfirmedTxQueries(query.where); + + for (const query of queries) { + let keepFetching = true; + while (keepFetching) { + const response = await this.#getConfirmedTransactions(query); + if (some(response.data?.transactions)) { + yield response.data.transactions.map((t) => + mapConfirmedTransaction(t, this.#biMapper) + ); + } - keepFetching = response.data?.transactions?.length === PAGE_SIZE; - if (keepFetching) args.skip += PAGE_SIZE; + keepFetching = response.data?.transactions?.length === PAGE_SIZE; + if (keepFetching) query.skip += PAGE_SIZE; + } } } @@ -279,7 +281,7 @@ export class ErgoGraphQLProvider implements IBlockchainProvider { } async getHeaders(query: HeaderQuery): Promise { - const response = await this.#getHeaders(query, this.#options.url); + const response = await this.#getHeaders(query); return ( response.data?.blockHeaders.map((header) => ({ @@ -335,32 +337,28 @@ export class ErgoGraphQLProvider implements IBlockchainProvider { } } -function buildGqlBoxQueryArgs(where: GraphQLBoxWhere) { - const args = { +function buildGqlBoxQueries(where: GraphQLBoxWhere) { + const ergoTrees = uniq( + [ + merge(where.ergoTrees, where.ergoTree) ?? [], + merge(where.addresses, where.address)?.map((a) => + typeof a === "string" ? ErgoAddress.decode(a).ergoTree : a.ergoTree + ) ?? [] + ].flat() + ); + + return chunk(ergoTrees, MAX_ARGS).map((chunk) => ({ spent: false, - boxIds: merge(where.boxIds, where.boxId), - ergoTrees: merge(where.ergoTrees, where.ergoTree), + boxIds: where.boxId ? [where.boxId] : undefined, + ergoTrees: chunk, ergoTreeTemplateHash: where.templateHash, tokenId: where.tokenId, skip: 0, take: PAGE_SIZE - } satisfies QueryBoxesArgs; - - const addresses = merge(where.addresses, where.address); - if (some(addresses)) { - const trees = addresses.map((address) => - typeof address === "string" - ? ErgoAddress.decode(address).ergoTree - : address.ergoTree - ); - - args.ergoTrees = uniq(some(args.ergoTrees) ? args.ergoTrees.concat(trees) : trees); - } - - return args; + })); } -function buildGqlUnconfirmedTxQueryArgs(where: GraphQLConfirmedTransactionWhere) { +function buildGqlUnconfirmedTxQueries(where: GraphQLConfirmedTransactionWhere) { const addresses = uniq( [ merge(where.addresses, where.address)?.map((address): string => @@ -372,21 +370,21 @@ function buildGqlUnconfirmedTxQueryArgs(where: GraphQLConfirmedTransactionWhere) ].flat() ); - return { - addresses: addresses.length ? addresses : undefined, - transactionIds: merge(where.transactionIds, where.transactionId), + return chunk(addresses, MAX_ARGS).map((chunk) => ({ + addresses: chunk.length ? chunk : undefined, + transactionIds: where.transactionId ? [where.transactionId] : undefined, skip: 0, take: PAGE_SIZE - }; + })); } -function buildGqlConfirmedTxQueryArgs(where: GraphQLConfirmedTransactionWhere) { - return { - ...buildGqlUnconfirmedTxQueryArgs(where), +function buildGqlConfirmedTxQueries(where: GraphQLConfirmedTransactionWhere) { + return buildGqlUnconfirmedTxQueries(where).map((query) => ({ + ...query, headerId: where.headerId, minHeight: where.minHeight, onlyRelevantOutputs: where.onlyRelevantOutputs - }; + })); } function merge(array?: T[], el?: T) {