diff --git a/.github/actions/build-lspd/action.yaml b/.github/actions/build-lspd/action.yaml new file mode 100644 index 00000000..4e8d9c1c --- /dev/null +++ b/.github/actions/build-lspd/action.yaml @@ -0,0 +1,23 @@ +name: 'Build LSPD' +description: 'Build LSPD and upload the build artifacts.' +runs: + using: 'composite' + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build LSPD + run: | + go get github.com/breez/lspd + go get github.com/breez/lspd/cln_plugin + go build . + go build -o lspd_plugin ./cln_plugin/cmd + shell: bash + + - name: Upload build artifacts + uses: actions/upload-artifact@v3 + with: + name: build-artifacts + path: | + ./lspd + ./lspd_plugin diff --git a/.github/actions/process-test-state/action.yaml b/.github/actions/process-test-state/action.yaml new file mode 100644 index 00000000..66cd1f9a --- /dev/null +++ b/.github/actions/process-test-state/action.yaml @@ -0,0 +1,36 @@ +name: "Process Test State" +description: "Check, tar and upload test state" +inputs: + artifact-name: + description: "Name of the artifact" + required: true + default: "test_state_artifact" + test-state-path: + description: "Path of the test state directory" + required: true + default: "/home/runner/test_state" +runs: + using: "composite" + steps: + - name: Check if test_state directory exists + id: check-test-state + run: | + if [ -d "${{ inputs.test-state-path }}" ]; then + echo "exists=true" >> $GITHUB_ENV + else + echo "exists=false" >> $GITHUB_ENV + fi + shell: bash + + - name: Tar state + run: | + find ${{ inputs.test-state-path }} -type f -o -type d | tar -czf ${{ inputs.test-state-path }}.tar.gz -T - + shell: bash + if: env.exists == 'true' + + - name: Upload test_state as artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.test-state-path }}.tar.gz + if: env.exists == 'true' diff --git a/.github/actions/setup-bitcoin/action.yaml b/.github/actions/setup-bitcoin/action.yaml new file mode 100644 index 00000000..f07d87a0 --- /dev/null +++ b/.github/actions/setup-bitcoin/action.yaml @@ -0,0 +1,42 @@ +name: 'Setup Bitcoin Core' +description: 'Download and install Bitcoin Core' +inputs: + bitcoin-version: + description: 'Version of Bitcoin Core' + required: true +runs: + using: 'composite' + steps: + - name: Cache Bitcoin Core + id: cache-bitcoin + uses: actions/cache@v3 + with: + path: | + ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoind + ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoin-cli + key: bitcoin-core-${{ inputs.bitcoin-version }} + + - name: Setup dependencies + if: steps.cache-bitcoin.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -y axel + shell: bash + + - name: Download and install Bitcoin Core + if: steps.cache-bitcoin.outputs.cache-hit != 'true' + run: | + mkdir -p ~/bitcoin-core-${{ inputs.bitcoin-version }} + cd ~/bitcoin-core-${{ inputs.bitcoin-version }} + axel https://bitcoin.org/bin/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}-x86_64-linux-gnu.tar.gz + tar -xzf bitcoin-${{ inputs.bitcoin-version }}-x86_64-linux-gnu.tar.gz + rm bitcoin-${{ inputs.bitcoin-version }}-x86_64-linux-gnu.tar.gz + sudo cp ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoind /usr/bin/ + sudo cp ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoin-cli /usr/bin/ + shell: bash + + - name: Copy Binaries + run: | + sudo cp ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoind /usr/bin/ + sudo cp ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoin-cli /usr/bin/ + shell: bash diff --git a/.github/actions/setup-clightning/action.yaml b/.github/actions/setup-clightning/action.yaml new file mode 100644 index 00000000..fe4f5f1e --- /dev/null +++ b/.github/actions/setup-clightning/action.yaml @@ -0,0 +1,109 @@ +name: 'Setup Core Lightning' +description: 'Set up Core Lightning on the runner' + +inputs: + checkout-version: + description: 'v23.05.1' + required: true + default: 'v23.05.1' + +runs: + using: 'composite' + steps: + - name: Cache Core Lightning + id: cache-core-lightning + uses: actions/cache@v3 + with: + path: | + lightning_git/lightningd/lightning_hsmd + lightning_git/lightningd/lightning_gossipd + lightning_git/lightningd/lightning_openingd + lightning_git/lightningd/lightning_dualopend + lightning_git/lightningd/lightning_channeld + lightning_git/lightningd/lightning_closingd + lightning_git/lightningd/lightning_onchaind + lightning_git/lightningd/lightning_connectd + lightning_git/lightningd/lightning_websocketd + lightning_git/lightningd/lightningd + lightning_git/plugins/offers + lightning_git/plugins/topology + lightning_git/plugins/spenderp + lightning_git/plugins/test/run-route-overlong + lightning_git/plugins/test/run-funder_policy + lightning_git/plugins/pay + lightning_git/plugins/bkpr/test/run-bkpr_db + lightning_git/plugins/bkpr/test/run-recorder + lightning_git/plugins/funder + lightning_git/plugins/bookkeeper + lightning_git/plugins/txprepare + lightning_git/plugins/keysend + lightning_git/plugins/fetchinvoice + lightning_git/plugins/bcli + lightning_git/plugins/cln-grpc + lightning_git/plugins/commando + lightning_git/plugins/autoclean + lightning_git/plugins/chanbackup + lightning_git/plugins/sql + lightning_git/cli/lightning-cli + lightning_git/devtools/bolt11-cli + lightning_git/devtools/decodemsg + lightning_git/devtools/onion + lightning_git/devtools/dump-gossipstore + lightning_git/devtools/gossipwith + lightning_git/devtools/create-gossipstore + lightning_git/devtools/mkcommit + lightning_git/devtools/mkfunding + lightning_git/devtools/mkclose + lightning_git/devtools/mkgossip + key: core-lightning-${{ inputs.checkout-version }} + + - name: Setup Python 3.8 + if: steps.cache-core-lightning.outputs.cache-hit != 'true' + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Setup Rust + if: steps.cache-core-lightning.outputs.cache-hit != 'true' + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + + - name: Install dependencies + if: steps.cache-core-lightning.outputs.cache-hit != 'true' + run: | + sudo apt-get install -y autoconf automake build-essential git libtool libgmp-dev libsqlite3-dev python3 python3-pip net-tools zlib1g-dev libsodium-dev gettext valgrind libpq-dev shellcheck cppcheck libsecp256k1-dev jq + sudo apt-get remove -y protobuf-compiler + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.12.0/protoc-3.12.0-linux-x86_64.zip + sudo unzip -o protoc-3.12.0-linux-x86_64.zip -d /usr/local bin/protoc + sudo unzip -o protoc-3.12.0-linux-x86_64.zip -d /usr/local 'include/*' + rm -f protoc-3.12.0-linux-x86_64.zip + sudo chmod 755 /usr/local/bin/protoc + shell: bash + + - name: Install Python dependencies + if: steps.cache-core-lightning.outputs.cache-hit != 'true' + run: | + pip3 install --upgrade pip + pip3 install poetry mako + pip install grpcio-tools + shell: bash + + - name: Checkout and build lightning + if: steps.cache-core-lightning.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: ElementsProject/lightning + ref: ${{ inputs.checkout-version }} + path: lightning_git + + - name: Build Lightning + if: steps.cache-core-lightning.outputs.cache-hit != 'true' + run: | + cd lightning_git + ./configure --enable-developer --enable-rust + poetry install + poetry run make -j `nproc` + shell: bash diff --git a/.github/actions/setup-itest/action.yaml b/.github/actions/setup-itest/action.yaml new file mode 100644 index 00000000..43c85042 --- /dev/null +++ b/.github/actions/setup-itest/action.yaml @@ -0,0 +1,16 @@ +name: 'Cache itest' +description: 'Fetch LSPD Integration Test and cache the go directory' +runs: + using: 'composite' + steps: + - name: Cache itest + id: cache-itest + uses: actions/cache@v3 + with: + path: | + ~/go + key: itest + - name: Get LSPD Integration Test + if: steps.cache-itest.outputs.cache-hit != 'true' + run: go get github.com/breez/lspd/itest + shell: bash \ No newline at end of file diff --git a/.github/actions/setup-lnd-client/action.yaml b/.github/actions/setup-lnd-client/action.yaml new file mode 100644 index 00000000..accbbb62 --- /dev/null +++ b/.github/actions/setup-lnd-client/action.yaml @@ -0,0 +1,45 @@ +name: 'Setup LND Client' +description: 'Set up LND for the Client on the runner' + +inputs: + client-ref: + description: 'The Git reference for the Client version of LND' + required: true + default: 'v0.16.2-breez' + + go-version: + description: 'The Go version for building LND' + required: true + default: ^1.19 + +runs: + using: 'composite' + steps: + - name: Cache LND client + id: cache-lnd-client + uses: actions/cache@v3 + with: + path: | + ~/go_lnd_client/bin/lnd + key: go_lnd_client-${{ inputs.client-ref }}-${{ inputs.go-version }} + + - name: Set up Go 1.x + if: steps.cache-lnd-client.outputs.cache-hit != 'true' + uses: actions/setup-go@v4 + with: + go-version: ${{ inputs.go-version }} + + - name: Checkout LND for Client + if: steps.cache-lnd-client.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: breez/lnd + ref: ${{ inputs.client-ref }} + path: lnd_client + + - name: Build LND for Client + if: steps.cache-lnd-client.outputs.cache-hit != 'true' + run: | + cd lnd_client + env GOPATH=~/go_lnd_client make install + shell: bash diff --git a/.github/actions/setup-lnd-lsp/action.yaml b/.github/actions/setup-lnd-lsp/action.yaml new file mode 100644 index 00000000..80740108 --- /dev/null +++ b/.github/actions/setup-lnd-lsp/action.yaml @@ -0,0 +1,45 @@ +name: 'Setup LND LSP' +description: 'Set up LND for LSP on the runner' + +inputs: + lsp-ref: + description: 'The Git reference for the LSP version of LND' + required: true + default: 'breez-node-v0.16.2-beta' + + go-version: + description: 'The Go version for building LND' + required: true + default: ^1.19 + +runs: + using: 'composite' + steps: + - name: Cache LND LSP + id: cache-lnd-lsp + uses: actions/cache@v3 + with: + path: | + ~/go_lnd_lsp/bin/lnd + key: go_lnd_lsp-${{ inputs.lsp-ref }}-${{ inputs.go-version }} + + - name: Set up Go 1.x + if: steps.cache-lnd-lsp.outputs.cache-hit != 'true' + uses: actions/setup-go@v4 + with: + go-version: ${{ inputs.go-version }} + + - name: Checkout LND for LSP + if: steps.cache-lnd-lsp.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: breez/lnd + ref: ${{ inputs.lsp-ref }} + path: lnd_lsp + + - name: Build LND for LSP + if: steps.cache-lnd-lsp.outputs.cache-hit != 'true' + run: | + cd lnd_lsp + env GOPATH=~/go_lnd_lsp make install tags='submarineswaprpc chanreservedynamic routerrpc walletrpc chainrpc signrpc invoicesrpc' + shell: bash diff --git a/.github/actions/test-lspd/action.yaml b/.github/actions/test-lspd/action.yaml new file mode 100644 index 00000000..bd09fe8d --- /dev/null +++ b/.github/actions/test-lspd/action.yaml @@ -0,0 +1,144 @@ +name: 'Test LSPD' +description: 'Downloads artifacts, sets permissions, caches and runs a specified test and processes the result as an artifact.' +inputs: + TESTRE: + description: 'Test regular expression to run.' + required: true + artifact-name: + description: 'Artifact name for the test state.' + required: true + bitcoin-version: + description: 'Bitcoin version.' + required: true + LSP_REF: + description: 'LSP reference.' + required: true + CLIENT_REF: + description: 'Client reference.' + required: true + GO_VERSION: + description: 'Go version.' + required: true + CLN_VERSION: + description: 'Core Lightning version.' + required: true +runs: + using: 'composite' + steps: + - name: Download build artifacts + uses: actions/download-artifact@v3 + with: + name: build-artifacts + + - name: Set permissions + run: | + chmod 755 lspd + chmod 755 lspd_plugin + shell: bash + + - name: Cache LND client + uses: actions/cache@v3 + with: + path: ~/go_lnd_client/bin/lnd + key: go_lnd_client-${{ inputs.CLIENT_REF }}-${{ inputs.GO_VERSION }} + + - name: Cache LND LSP + uses: actions/cache@v3 + with: + path: ~/go_lnd_lsp/bin/lnd + key: go_lnd_lsp-${{ inputs.LSP_REF }}-${{ inputs.GO_VERSION }} + + - name: Cache Core Lightning + uses: actions/cache@v3 + with: + path: | + lightning_git/lightningd/lightning_hsmd + lightning_git/lightningd/lightning_gossipd + lightning_git/lightningd/lightning_openingd + lightning_git/lightningd/lightning_dualopend + lightning_git/lightningd/lightning_channeld + lightning_git/lightningd/lightning_closingd + lightning_git/lightningd/lightning_onchaind + lightning_git/lightningd/lightning_connectd + lightning_git/lightningd/lightning_websocketd + lightning_git/lightningd/lightningd + lightning_git/plugins/offers + lightning_git/plugins/topology + lightning_git/plugins/spenderp + lightning_git/plugins/test/run-route-overlong + lightning_git/plugins/test/run-funder_policy + lightning_git/plugins/pay + lightning_git/plugins/bkpr/test/run-bkpr_db + lightning_git/plugins/bkpr/test/run-recorder + lightning_git/plugins/funder + lightning_git/plugins/bookkeeper + lightning_git/plugins/txprepare + lightning_git/plugins/keysend + lightning_git/plugins/fetchinvoice + lightning_git/plugins/bcli + lightning_git/plugins/cln-grpc + lightning_git/plugins/commando + lightning_git/plugins/autoclean + lightning_git/plugins/chanbackup + lightning_git/plugins/sql + lightning_git/cli/lightning-cli + lightning_git/devtools/bolt11-cli + lightning_git/devtools/decodemsg + lightning_git/devtools/onion + lightning_git/devtools/dump-gossipstore + lightning_git/devtools/gossipwith + lightning_git/devtools/create-gossipstore + lightning_git/devtools/mkcommit + lightning_git/devtools/mkfunding + lightning_git/devtools/mkclose + lightning_git/devtools/mkgossip + key: core-lightning-${{ inputs.CLN_VERSION }} + + - name: Cache Bitcoin Core + uses: actions/cache@v3 + with: + path: | + ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoind + ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoin-cli + key: bitcoin-core-${{ inputs.bitcoin-version }} + + - name: Cache itest + id: cache-itest + uses: actions/cache@v3 + with: + path: | + ~/go + key: itest + + - name: Test LSPD + run: | + go get github.com/breez/lspd/itest + go test -timeout 45m -v \ + ./itest \ + -test.run \ + ${{ inputs.TESTRE }} \ + --bitcoindexec ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoind \ + --bitcoincliexec ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoin-cli \ + --lightningdexec ${{ github.workspace }}/lightning_git/lightningd/lightningd \ + --lndexec ~/go_lnd_lsp/bin/lnd \ + --lndmobileexec ~/go_lnd_client/bin/lnd \ + --clnpluginexec ${{ github.workspace }}/lspd_plugin \ + --lspdexec ${{ github.workspace }}/lspd \ + --lspdmigrationsdir ${{ github.workspace }}/postgresql/migrations \ + --preservelogs \ + --testdir /home/runner/test_state || echo "step_failed=true" >> $GITHUB_ENV + shell: bash + + - name: Process test state + uses: ./.github/actions/process-test-state + with: + artifact-name: ${{ inputs.artifact-name }} + test-state-path: /home/runner/test_state + + - name: Fail the workflow if the tests failed + run: | + if [[ "${{ env.step_failed }}" == "true" ]] + then + exit 1 + fi + shell: bash diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml new file mode 100644 index 00000000..f531e3e5 --- /dev/null +++ b/.github/workflows/integration_tests.yaml @@ -0,0 +1,123 @@ +name: integration tests +on: [push] +env: + BITCOIN_VERSION: '22.0' + LSP_REF: 'breez-node-v0.16.4-beta' + CLIENT_REF: 'v0.16.4-beta-breez' + GO_VERSION: '^1.19' + CLN_VERSION: 'v23.05.1' +jobs: + + setup-bitcoin-core: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Bitcoin Core + if: steps.cache-bitcoin.outputs.cache-hit != 'true' + uses: ./.github/actions/setup-bitcoin + with: + bitcoin-version: ${{ env.BITCOIN_VERSION }} + + setup-lnd-lsp: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up LND LSP + if: steps.cache-lnd-lsp.outputs.cache-hit != 'true' + uses: ./.github/actions/setup-lnd-lsp + with: + lsp-ref: ${{ env.LSP_REF }} + go-version: ${{ env.GO_VERSION }} + + setup-lnd-client: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up LND client + if: steps.cache-lnd-client.outputs.cache-hit != 'true' + uses: ./.github/actions/setup-lnd-client + with: + client-ref: ${{ env.CLIENT_REF }} + go-version: ${{ env.GO_VERSION }} + + setup-cln: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Core Lightning + uses: ./.github/actions/setup-clightning + with: + checkout-version: ${{ env.CLN_VERSION }} + + build-lspd: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Build LSPD and Upload Artifacts + uses: ./.github/actions/build-lspd + + setup-itest: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup itest + uses: ./.github/actions/setup-itest + + + run-test: + runs-on: ubuntu-22.04 + needs: + - setup-itest + - setup-bitcoin-core + - setup-lnd-client + - setup-lnd-lsp + - setup-cln + - build-lspd + name: test ${{ matrix.implementation }} ${{ matrix.test }} + strategy: + max-parallel: 6 + matrix: + test: [ + testOpenZeroConfChannelOnReceive, + testOpenZeroConfSingleHtlc, + testZeroReserve, + testFailureBobOffline, + testNoBalance, + testRegularForward, + testProbing, + testInvalidCltv, + registerPaymentWithTag, + testOpenZeroConfUtxo, + testDynamicFeeFlow, + testOfflineNotificationPaymentRegistered, + testOfflineNotificationRegularForward, + testOfflineNotificationZeroConfChannel, + ] + implementation: [ + LND, + CLN + ] + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run and Process Test State + uses: ./.github/actions/test-lspd + with: + TESTRE: "TestLspd/${{ matrix.implementation }}-lspd:_${{ matrix.test }}" + artifact-name: TestLspd-${{ matrix.implementation }}-lspd_${{ matrix.test }} + bitcoin-version: ${{ env.BITCOIN_VERSION }} + LSP_REF: ${{ env.LSP_REF }} + CLIENT_REF: ${{ env.CLIENT_REF }} + GO_VERSION: ${{ env.GO_VERSION }} + CLN_VERSION: ${{ env.CLN_VERSION }} diff --git a/.gitignore b/.gitignore index 7a25686d..c9e8d7cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +.vscode go.sum lspd +lspd_plugin