diff --git a/.env-example b/.env-example index 89bbe72..d80a92b 100644 --- a/.env-example +++ b/.env-example @@ -1,11 +1,10 @@ -# Substreams endpoint -SUBSTREAMS_API_TOKEN=... -SUBSTREAMS_ENDPOINT=https://eth.firehose.pinax.network:9000 +# Get Substreams API Key +# https://app.pinax.network +# https://app.streamingfast.io/ +SUBSTREAMS_API_KEY=... +SUBSTREAMS_ENDPOINT=https://eth.substreams.pinax.network:443 -# Substreams package -MANIFEST=https://github.com/pinax-network/subtivity-substreams/releases/download/v0.2.3/subtivity-ethereum-v0.2.3.spkg -MODULE_NAME=map_block_stats -START_BLOCK=100000 -STOP_BLOCK=100010 -VERBOSE=true -FINAL_BLOCKS_ONLY=false \ No newline at end of file +# SPKG +MANIFEST=https://github.com/pinax-network/substreams/releases/download/blocks-v0.1.0/blocks-v0.1.0.spkg +MODULE_NAME=map_blocks +START_BLOCK=-10 diff --git a/LICENSE-MIT b/LICENSE similarity index 97% rename from LICENSE-MIT rename to LICENSE index 6af0e03..7946f1b 100644 --- a/LICENSE-MIT +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2023 Pinax +Copyright (c) 2024 Pinax Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated diff --git a/LICENSE-APACHE b/LICENSE-APACHE deleted file mode 100644 index 413916e..0000000 --- a/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2023 Pinax - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md index dfe3ff6..c56c42e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ - [**Substreams** documentation](https://substreams.streamingfast.io) - [Subtreams sink project template Github repo](https://github.com/pinax-network/substreams-sink-template) +## Get Substreams API Key +- https://app.pinax.network +- https://app.streamingfast.io/ + ## 🚀 Quick start ### Installation @@ -64,9 +68,25 @@ Options: ### Example +**.env** + +```env +# Get Substreams API Key +# https://app.pinax.network +# https://app.streamingfast.io/ +SUBSTREAMS_API_KEY=... +SUBSTREAMS_ENDPOINT=https://eth.substreams.pinax.network:443 + +# SPKG +MANIFEST=https://github.com/pinax-network/substreams/releases/download/blocks-v0.1.0/blocks-v0.1.0.spkg +MODULE_NAME=map_blocks +START_BLOCK=-10 +``` + +**example.js** ```js import pkg from "./package.json" assert { type: "json" }; -import { commander, setup, prometheus, http, logger } from "./dist/index.js"; +import { commander, setup, prometheus, http, logger } from "substreams-sink"; // Setup CLI using Commander const program = commander.program(pkg); @@ -80,6 +100,14 @@ command.action(async options => { // Setup sink for Block Emitter const { emitter } = await setup(options); + emitter.on("session", (session) => { + console.log(session); + }); + + emitter.on("progress", (progress) => { + console.log(progress); + }); + // Stream Blocks emitter.on("anyMessage", (message, cursor, clock) => { customCounter?.inc(1); @@ -91,9 +119,13 @@ command.action(async options => { // Setup HTTP server & Prometheus metrics http.listen(options); - // Start streaming - await emitter.start(); - http.server.close(); + // Start the stream + emitter.start(); + + emitter.on("close", () => { + http.server.close(); + console.log("✅ finished"); + }) }) program.parse(); ``` diff --git a/example.env b/example.env deleted file mode 100644 index ea0b4da..0000000 --- a/example.env +++ /dev/null @@ -1,3 +0,0 @@ -MANIFEST=https://github.com/pinax-network/subtivity-substreams/releases/download/v0.2.3/subtivity-ethereum-v0.2.3.spkg -MODULE_NAME=map_block_stats -VERBOSE=true \ No newline at end of file diff --git a/example.js b/example.js index bde4e2c..59b08ba 100644 --- a/example.js +++ b/example.js @@ -12,7 +12,14 @@ const customCounter = prometheus.registerCounter("custom_counter"); command.action(async options => { // Setup sink for Block Emitter const { emitter } = await setup(options); - console.log("setup") + + emitter.on("session", (session) => { + console.log(session); + }); + + emitter.on("progress", (progress) => { + console.log(progress); + }); // Stream Blocks emitter.on("anyMessage", (message, cursor, clock) => { @@ -25,9 +32,12 @@ command.action(async options => { // Setup HTTP server & Prometheus metrics http.listen(options); - // Start streaming + // Start the stream emitter.start(); - http.server.close(); - console.log("✅ finished"); + + emitter.on("close", () => { + http.server.close(); + console.log("✅ finished"); + }) }) program.parse(); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 37f18d1..f5f4ab7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,9 @@ "version": "0.13.8", "license": "MIT OR Apache-2.0", "dependencies": { - "@substreams/core": "^0.12.0", - "@substreams/manifest": "^0.11.0", - "@substreams/node": "^0.5.3", + "@substreams/core": "^0.15.1", + "@substreams/manifest": "^0.14.1", + "@substreams/node": "^0.6.2", "commander": "latest", "dotenv": "latest", "prom-client": "latest", @@ -145,9 +145,9 @@ } }, "node_modules/@bufbuild/protobuf": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.7.0.tgz", - "integrity": "sha512-jIsRadRsyxf6ERBU1auY2c1k3doFdqh15F4HRZs4BELVuBtpN+3ipkXqcsWE+rD+EQNigeR29SfQ+ES6UX/jGg==" + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.7.2.tgz", + "integrity": "sha512-i5GE2Dk5ekdlK1TR7SugY4LWRrKSfb5T1Qn4unpIMbfxoeGKERKQ59HG3iYewacGD10SR7UzevfPnh6my4tNmQ==" }, "node_modules/@connectrpc/connect": { "version": "1.3.0", @@ -182,12 +182,12 @@ } }, "node_modules/@effect/schema": { - "version": "0.60.7", - "resolved": "https://registry.npmjs.org/@effect/schema/-/schema-0.60.7.tgz", - "integrity": "sha512-YSRGyOeR6b932VM4SoR7QjcXPPgfylbE0hY2F3lPGz5rPIG3jpp5yIt58RBcItkQ1tSdGsuPpfLpLWBOhx4VJA==", + "version": "0.62.3", + "resolved": "https://registry.npmjs.org/@effect/schema/-/schema-0.62.3.tgz", + "integrity": "sha512-kstVDXQGarDoOlb05tsszRDPrnudW+oHJ5ZJtCs8TOfR6WJOih1c24LTUSgAiyccENi/5lFC6CMFHcwroU8NDQ==", "peer": true, "peerDependencies": { - "effect": "^2.2.0", + "effect": "^2.3.3", "fast-check": "^3.13.2" } }, @@ -208,33 +208,33 @@ } }, "node_modules/@substreams/core": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@substreams/core/-/core-0.12.0.tgz", - "integrity": "sha512-QgwcorVlBYjJ0znSiMBXFj3BRWIeq+ZDVJkGRrYttQ4c15Bqcmk4ALKW5e6C8flx+EVug297o7fVBbMaOyVclA==", + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/@substreams/core/-/core-0.15.1.tgz", + "integrity": "sha512-zxk/KzSCaR3C+1La9I6tfLp2SJGEARxol8Mwa/wMNOsXUvsZ57Fhm515IYwoUO8HSdi/L+OwOpzjvb4NTJwSKA==", "peerDependencies": { - "@bufbuild/protobuf": "^1.6.0", + "@bufbuild/protobuf": "^1.7.2", "@connectrpc/connect": "^1.3.0" } }, "node_modules/@substreams/manifest": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@substreams/manifest/-/manifest-0.11.0.tgz", - "integrity": "sha512-AOuwGG/zKocjK+n3kD05BWEpt4iH/5R7eWp5BCSWx4vV494Vqd+QcbIwGuQIG32rVHmqo9FtV7jRFsTcmqL0rQ==", + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/@substreams/manifest/-/manifest-0.14.1.tgz", + "integrity": "sha512-NKxNmiVki41wPGmGewQBkpzDc6HOMkK2JIZR4mPczSA3tRIxwiTcK027X5Na3mpaxJawbzAOHGSuJNbS+IBYJQ==", "dependencies": { - "@substreams/core": "0.12.0" + "@substreams/core": "0.15.1" }, "peerDependencies": { - "@bufbuild/buf": "^1.28.1", - "@bufbuild/protobuf": "^1.6.0", - "@effect/schema": "^0.60.2", - "effect": "^2.1.0", + "@bufbuild/buf": "^1.29.0", + "@bufbuild/protobuf": "^1.7.2", + "@effect/schema": "^0.62.0", + "effect": "^2.3.0", "yaml": "^2.3.4" } }, "node_modules/@substreams/node": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/@substreams/node/-/node-0.5.3.tgz", - "integrity": "sha512-3wUUD1N7A9Fw7DFRK43TfNBNPXxuNTh+yiuPiCqJEnEhA7xhmzNvnewgp7aoHdWXE4FZFYuZYCX1wrg5FU3LMw==", + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@substreams/node/-/node-0.6.2.tgz", + "integrity": "sha512-as/LbCOy4YM9UGFZ4eIfNJircDPnm2BYUCk4Fc1U6SakLGBG1Glbnc6c63PIxxonO2zpxu1yYPLu/BXdFCjXWQ==", "dependencies": { "@bufbuild/protobuf": "latest", "@connectrpc/connect": "latest", @@ -248,9 +248,9 @@ } }, "node_modules/@types/node": { - "version": "20.11.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.6.tgz", - "integrity": "sha512-+EOokTnksGVgip2PbYbr3xnR7kZigh4LbybAfBAw5BpnQ+FqBYUsvCEjYd70IXKlbohQ64mzEYmMtlWUY8q//Q==", + "version": "20.11.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.17.tgz", + "integrity": "sha512-QmgQZGWu1Yw9TDyAP9ZzpFJKynYNeOvwMJmaxABfieQoVoiVOS6MN1WSpqpRcbeA5+RW82kraAVxCCJg+780Qw==", "dev": true, "dependencies": { "undici-types": "~5.26.4" @@ -262,28 +262,28 @@ "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" }, "node_modules/commander": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", - "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.0.0.tgz", + "integrity": "sha512-MwVNWlYjDTtOjX5PiD7o5pK0UrFU/OYgcJfjjK4RaHZETNtjJqrZa9Y9ds88+A+f+d5lv+561eZ+yCKoS3gbAA==", "engines": { - "node": ">=16" + "node": ">=18" } }, "node_modules/dotenv": { - "version": "16.4.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.1.tgz", - "integrity": "sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==", + "version": "16.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.2.tgz", + "integrity": "sha512-rZSSFxke7d9nYQ5NeMIwp5PP+f8wXgKNljpOb7KtH6SKW1cEqcXAz9VSJYVLKe7Jhup/gUYOkaeSVyK8GJ+nBg==", "engines": { "node": ">=12" }, "funding": { - "url": "https://github.com/motdotla/dotenv?sponsor=1" + "url": "https://dotenvx.com" } }, "node_modules/effect": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/effect/-/effect-2.2.2.tgz", - "integrity": "sha512-hY/37Ssd2Zfn0r09vDe9tSYPVS5HZBbmW4DMBO7OIIvRADgp5cnWfLhY6pVy+Bm5DXV9luFycjDcB1731WGyTA==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/effect/-/effect-2.3.3.tgz", + "integrity": "sha512-ZmcaQqedKhQT2Zb1X+sCCIXJgz0MAJ/F9rXe53WsjjU6pgtJlgRwGChk+E51uH/8iz/hP99JiPrltuIutes9yw==", "peer": true }, "node_modules/eventemitter3": { @@ -292,9 +292,9 @@ "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" }, "node_modules/fast-check": { - "version": "3.15.0", - "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.15.0.tgz", - "integrity": "sha512-iBz6c+EXL6+nI931x/sbZs1JYTZtLG6Cko0ouS8LRTikhDR7+wZk4TYzdRavlnByBs2G6+nuuJ7NYL9QplNt8Q==", + "version": "3.15.1", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.15.1.tgz", + "integrity": "sha512-GutOXZ+SCxGaFWfHe0Pbeq8PrkpGtPxA9/hdkI3s9YzqeMlrq5RdJ+QfYZ/S93jMX+tAyqgW0z5c9ppD+vkGUw==", "funding": [ { "type": "individual", @@ -374,9 +374,9 @@ } }, "node_modules/undici": { - "version": "5.28.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz", - "integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==", + "version": "5.28.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.3.tgz", + "integrity": "sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==", "dependencies": { "@fastify/busboy": "^2.0.0" }, diff --git a/package.json b/package.json index 4cc21f6..923effd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "substreams-sink", - "version": "0.13.8", + "version": "0.14.0", "description": "Substreams Sink", "type": "module", "exports": "./dist/index.js", @@ -22,7 +22,7 @@ "src" ], "engines": { - "node": ">=18" + "node": ">=20" }, "contributors": [ { @@ -38,15 +38,15 @@ "email": "francois@pinax.network" } ], - "license": "MIT OR Apache-2.0", + "license": "MIT", "scripts": { "build": "tsc", "prepublishOnly": "npm run build" }, "dependencies": { - "@substreams/core": "^0.12.0", - "@substreams/manifest": "^0.11.0", - "@substreams/node": "^0.5.3", + "@substreams/core": "^0.15.1", + "@substreams/manifest": "^0.14.1", + "@substreams/node": "^0.6.2", "commander": "latest", "dotenv": "latest", "prom-client": "latest", @@ -56,4 +56,4 @@ "@types/node": "latest", "typescript": "latest" } -} \ No newline at end of file +} diff --git a/src/commander.ts b/src/commander.ts index 0203aff..1c31be2 100644 --- a/src/commander.ts +++ b/src/commander.ts @@ -1,6 +1,6 @@ import "dotenv/config"; import { Command, Option } from "commander"; -import { DEFAULT_CURSOR_PATH, DEFAULT_INACTIVITY_SECONDS, DEFAULT_PARAMS, DEFAULT_SUBSTREAMS_API_TOKEN, DEFAULT_AUTH_ISSUE_URL, DEFAULT_VERBOSE, DEFAULT_HOSTNAME, DEFAULT_PORT, DEFAULT_METRICS_LABELS, DEFAULT_COLLECT_DEFAULT_METRICS, DEFAULT_START_BLOCK, DEFAULT_DELAY_BEFORE_START, DEFAULT_HEADERS, DEFAULT_PRODUCTION_MODE, DEFAULT_FINAL_BLOCKS_ONLY } from "./config.js"; +import { DEFAULT_CURSOR_PATH, DEFAULT_INACTIVITY_SECONDS, DEFAULT_PARAMS, DEFAULT_VERBOSE, DEFAULT_HOSTNAME, DEFAULT_PORT, DEFAULT_METRICS_LABELS, DEFAULT_COLLECT_DEFAULT_METRICS, DEFAULT_START_BLOCK, DEFAULT_DELAY_BEFORE_START, DEFAULT_HEADERS, DEFAULT_PRODUCTION_MODE, DEFAULT_FINAL_BLOCKS_ONLY } from "./config.js"; import { list } from "./list.js"; import { logger } from "./logger.js"; @@ -18,8 +18,8 @@ export interface RunOptions { params: string[]; startBlock: string; stopBlock: string; - substreamsApiToken: string; - authIssueUrl: string; + substreamsApiKey: string; + substreamsApiToken: string; // Deprecated delayBeforeStart: number; cursorPath: string; httpCursorAuth: string; @@ -88,12 +88,12 @@ export function run(program: Command, pkg: Package) { .addOption(new Option("-s --start-block ", "Start block to stream from (defaults to -1, which means the initialBlock of the first module you are streaming)").default(DEFAULT_START_BLOCK).env("START_BLOCK")) .addOption(new Option("-t --stop-block ", "Stop block to end stream at, inclusively").env("STOP_BLOCK")) .addOption(new Option("-p, --params ", "Set a params for parameterizable modules. Can be specified multiple times. (ex: -p module1=valA -p module2=valX&valY)").default(DEFAULT_PARAMS).env("PARAMS")) // Make sure params are parsed correctly when using env - .addOption(new Option("--substreams-api-token ", "API token for the substream endpoint or API key if '--auth-issue-url' is specified").default(DEFAULT_SUBSTREAMS_API_TOKEN).env("SUBSTREAMS_API_TOKEN")) - .addOption(new Option("--auth-issue-url ", "URL used to issue a token").default(DEFAULT_AUTH_ISSUE_URL).env("AUTH_ISSUE_URL")) + .addOption(new Option("--substreams-api-key ", "API key for the Substream endpoint").env("SUBSTREAMS_API_KEY")) + .addOption(new Option("--substreams-api-token ", "(DEPRECATED) API token for the Substream endpoint").hideHelp(true).env("SUBSTREAMS_API_TOKEN")) .addOption(new Option("--delay-before-start ", "Delay (ms) before starting Substreams").default(DEFAULT_DELAY_BEFORE_START).env("DELAY_BEFORE_START")) .addOption(new Option("--cursor-path ", "File path or URL to cursor lock file").default(DEFAULT_CURSOR_PATH).env("CURSOR_PATH")) .addOption(new Option("--http-cursor-auth ", "Basic auth credentials for http cursor (ex: username:password)").env("HTTP_CURSOR_AUTH").argParser(handleHttpCursorAuth)) - .addOption(new Option("--production-mode ", "Enable production mode, allows cached substreams data if available").default(DEFAULT_PRODUCTION_MODE).env("PRODUCTION_MODE")) + .addOption(new Option("--production-mode ", "Enable production mode, allows cached Substreams data if available").default(DEFAULT_PRODUCTION_MODE).env("PRODUCTION_MODE")) .addOption(new Option("--inactivity-seconds ", "If set, the sink will stop when inactive for over a certain amount of seconds").default(DEFAULT_INACTIVITY_SECONDS).env("INACTIVITY_SECONDS")) .addOption(new Option("--hostname ", "The process will listen on this hostname for any HTTP and Prometheus metrics requests").default(DEFAULT_HOSTNAME).env("HOSTNAME")) .addOption(new Option("--port ", "The process will listen on this port for any HTTP and Prometheus metrics requests").default(DEFAULT_PORT).env("PORT")) diff --git a/src/config.ts b/src/config.ts index 1e0b3bb..5e52f19 100644 --- a/src/config.ts +++ b/src/config.ts @@ -9,8 +9,6 @@ export const DEFAULT_DELAY_BEFORE_START = 0; export const DEFAULT_METRICS_LABELS = {}; export const DEFAULT_COLLECT_DEFAULT_METRICS = "false"; export const DEFAULT_START_BLOCK = "-1"; -export const DEFAULT_SUBSTREAMS_API_TOKEN = ""; export const DEFAULT_PARAMS = []; export const DEFAULT_HEADERS = new Headers(); -export const DEFAULT_AUTH_ISSUE_URL = "https://auth.pinax.network/v1/auth/issue"; export const DEFAULT_FINAL_BLOCKS_ONLY = "false"; \ No newline at end of file diff --git a/src/parseAuthorization.ts b/src/parseAuthorization.ts deleted file mode 100644 index 09ee1f7..0000000 --- a/src/parseAuthorization.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { authIssue } from "@substreams/core"; - -// Supports 48 characters long Pinax API key -export async function parseAuthorization(authorization: string, url?: string) { - // issue token if includes server_ prefix or is 48 characters long - if (authorization.includes("server_") || authorization.length === 48) { - const { token } = await authIssue(authorization, url); - return token; - } - - // no action if Substreams API token is provided - return authorization; -} \ No newline at end of file diff --git a/src/prometheus.ts b/src/prometheus.ts index 0545a8e..ad41a2a 100644 --- a/src/prometheus.ts +++ b/src/prometheus.ts @@ -53,7 +53,7 @@ const substreams_sink_data_message = registerCounter("substreams_sink_data_messa const substreams_sink_data_message_size_bytes = registerCounter("substreams_sink_data_message_size_bytes", "The total size of in bytes of all data message received"); const substreams_sink_undo_message = registerCounter("substreams_sink_undo_message", "The number of block undo message received"); // const substreams_sink_unknown_message = registerCounter("substreams_sink_unknown_message", "The number of unknown message received"); -// const substreams_sink_progress_message = registerCounter("substreams_sink_progress_message", "The number of progress message received", ["module"]); +export const substreams_sink_progress_message = registerCounter("substreams_sink_progress_message", "The number of progress message received", ["module"]); // Gauges // const trace_id = registerGauge("trace_id", "Substreams session trace id", ["trace_id"]); @@ -80,6 +80,10 @@ function updateBlockDataMetrics(block: BlockScopedData) { substreams_sink_backprocessing_completion?.set(1); } +function updateTotalBytesRead(bytes: number) { + // substreams_sink_message_size_bytes?.inc(bytes); +} + export function onPrometheusMetrics(emitter: BlockEmitter) { emitter.on("session", handleSession); emitter.on("undo", () => substreams_sink_undo_message?.inc(1)); diff --git a/src/setup.ts b/src/setup.ts index 7520dd5..08a71c1 100644 --- a/src/setup.ts +++ b/src/setup.ts @@ -1,5 +1,6 @@ import { createRegistry, createRequest, createModuleHashHex } from "@substreams/core"; -import { BlockEmitter, createDefaultTransport } from "@substreams/node"; +import { createNodeTransport } from "@substreams/node/createNodeTransport"; +import { BlockEmitter } from "@substreams/node"; import { readPackage } from "@substreams/manifest"; import { setTimeout } from "timers/promises"; import type { RunOptions } from "./commander.js"; @@ -10,7 +11,6 @@ import { logger } from "./logger.js"; import { onInactivitySeconds } from "./inactivitySeconds.js"; import { applyParams } from "./applyParams.js"; import { health } from "./health.js"; -import { parseAuthorization } from "./parseAuthorization.js"; export async function setup(options: RunOptions) { // Configure logging with TSLog @@ -21,10 +21,9 @@ export async function setup(options: RunOptions) { const substreamPackage = await readPackage(manifest); if (!substreamPackage.modules) throw new Error("No modules found in substream package"); - // auth API token - // https://app.streamingfast.io/ - const token = await parseAuthorization(options.substreamsApiToken, options.authIssueUrl); + // Substreams endpoint let baseUrl = options.substreamsEndpoint; + const token = options.substreamsApiToken ?? options.substreamsApiKey; // append https if not present if (baseUrl.match(/http/) === null) { @@ -43,7 +42,7 @@ export async function setup(options: RunOptions) { const finalBlocksOnly = String(options.finalBlocksOnly) === "true"; // Adding default headers - headers.set("User-Agent", "substreams-sink"); + headers.set("X-User-Agent", "substreams-sink"); // Health check health(); @@ -59,7 +58,7 @@ export async function setup(options: RunOptions) { // Connect Transport const startCursor = await cursor.readCursor(cursorPath, httpCursorAuth); const registry = createRegistry(substreamPackage); - const transport = createDefaultTransport(baseUrl, token, registry, headers); + const transport = createNodeTransport(baseUrl, token, registry, headers); const request = createRequest({ substreamPackage, outputModule,