Add GitHub Actions for Bitcoin, c-lightning, and LND setup and integr…

…ation tests - A new GitHub Actions workflow for running integration tests has been added. This workflow is defined in .github/workflows/integration_tests.yaml. - New GitHub Actions for setting up Bitcoin, c-lightning, and LND environments have been implemented. - The actions use caching for quick builds. - The .gitignore file has been updated.
breez · Jul 12, 2023 · 2cdc772 · 2cdc772
1 parent 70d6afc
commit 2cdc772
Show file tree

Hide file tree

Showing 7 changed files with 335 additions and 2 deletions.
diff --git a/.github/actions/setup-bitcoin/action.yaml b/.github/actions/setup-bitcoin/action.yaml
@@ -0,0 +1,42 @@
+name: 'Setup Bitcoin Core'
+description: 'Download and install Bitcoin Core'
+inputs:
+  bitcoin-version:
+    description: 'Version of Bitcoin Core'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Cache Bitcoin Core
+      id: cache-bitcoin
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoind
+          ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoin-cli
+        key: bitcoin-core-${{ inputs.bitcoin-version }}
+
+    - name: Setup dependencies
+      if: steps.cache-bitcoin.outputs.cache-hit != 'true'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y axel
+      shell: bash
+
+    - name: Download and install Bitcoin Core
+      if: steps.cache-bitcoin.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p ~/bitcoin-core-${{ inputs.bitcoin-version }}
+        cd ~/bitcoin-core-${{ inputs.bitcoin-version }}
+        axel https://bitcoin.org/bin/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}-x86_64-linux-gnu.tar.gz
+        tar -xzf bitcoin-${{ inputs.bitcoin-version }}-x86_64-linux-gnu.tar.gz
+        rm bitcoin-${{ inputs.bitcoin-version }}-x86_64-linux-gnu.tar.gz
+        sudo cp ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoind /usr/bin/
+        sudo cp ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoin-cli /usr/bin/
+      shell: bash
+
+    - name: Copy Binaries
+      run: |
+        sudo cp ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoind /usr/bin/
+        sudo cp ~/bitcoin-core-${{ inputs.bitcoin-version }}/bitcoin-${{ inputs.bitcoin-version }}/bin/bitcoin-cli /usr/bin/
+      shell: bash
diff --git a/.github/actions/setup-clightning/action.yaml b/.github/actions/setup-clightning/action.yaml
@@ -0,0 +1,109 @@
+name: 'Setup Core Lightning'
+description: 'Set up Core Lightning on the runner'
+
+inputs:
+  checkout-version:
+    description: 'v23.05.1'
+    required: true
+    default: 'v23.05.1'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Cache Core Lightning
+      id: cache-core-lightning
+      uses: actions/cache@v2
+      with:
+        path: |
+          lightning_git/lightningd/lightning_hsmd
+          lightning_git/lightningd/lightning_gossipd
+          lightning_git/lightningd/lightning_openingd
+          lightning_git/lightningd/lightning_dualopend
+          lightning_git/lightningd/lightning_channeld
+          lightning_git/lightningd/lightning_closingd
+          lightning_git/lightningd/lightning_onchaind
+          lightning_git/lightningd/lightning_connectd
+          lightning_git/lightningd/lightning_websocketd
+          lightning_git/lightningd/lightningd
+          lightning_git/plugins/offers
+          lightning_git/plugins/topology
+          lightning_git/plugins/spenderp
+          lightning_git/plugins/test/run-route-overlong
+          lightning_git/plugins/test/run-funder_policy
+          lightning_git/plugins/pay
+          lightning_git/plugins/bkpr/test/run-bkpr_db
+          lightning_git/plugins/bkpr/test/run-recorder
+          lightning_git/plugins/funder
+          lightning_git/plugins/bookkeeper
+          lightning_git/plugins/txprepare
+          lightning_git/plugins/keysend
+          lightning_git/plugins/fetchinvoice
+          lightning_git/plugins/bcli
+          lightning_git/plugins/cln-grpc
+          lightning_git/plugins/commando
+          lightning_git/plugins/autoclean
+          lightning_git/plugins/chanbackup
+          lightning_git/plugins/sql
+          lightning_git/cli/lightning-cli
+          lightning_git/devtools/bolt11-cli
+          lightning_git/devtools/decodemsg
+          lightning_git/devtools/onion
+          lightning_git/devtools/dump-gossipstore
+          lightning_git/devtools/gossipwith
+          lightning_git/devtools/create-gossipstore
+          lightning_git/devtools/mkcommit
+          lightning_git/devtools/mkfunding
+          lightning_git/devtools/mkclose
+          lightning_git/devtools/mkgossip
+        key: core-lightning-${{ inputs.checkout-version }}
+
+    - name: Setup Python 3.8
+      if: steps.cache-core-lightning.outputs.cache-hit != 'true'
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Setup Rust
+      if: steps.cache-core-lightning.outputs.cache-hit != 'true'
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: true
+
+    - name: Install dependencies
+      if: steps.cache-core-lightning.outputs.cache-hit != 'true'
+      run: |
+        sudo apt-get install -y autoconf automake build-essential git libtool libgmp-dev libsqlite3-dev python3 python3-pip net-tools zlib1g-dev libsodium-dev gettext valgrind libpq-dev shellcheck cppcheck libsecp256k1-dev jq 
+        sudo apt-get remove -y protobuf-compiler
+        curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.12.0/protoc-3.12.0-linux-x86_64.zip
+        sudo unzip -o protoc-3.12.0-linux-x86_64.zip -d /usr/local bin/protoc
+        sudo unzip -o protoc-3.12.0-linux-x86_64.zip -d /usr/local 'include/*'
+        rm -f protoc-3.12.0-linux-x86_64.zip
+        sudo chmod 755 /usr/local/bin/protoc
+      shell: bash
+
+    - name: Install Python dependencies
+      if: steps.cache-core-lightning.outputs.cache-hit != 'true'
+      run: |
+        pip3 install --upgrade pip
+        pip3 install poetry mako
+        pip install grpcio-tools
+      shell: bash
+
+    - name: Checkout and build lightning
+      if: steps.cache-core-lightning.outputs.cache-hit != 'true'
+      uses: actions/checkout@v2
+      with:
+        repository: ElementsProject/lightning
+        ref: ${{ inputs.checkout-version }}
+        path: lightning_git
+
+    - name: Build Lightning
+      if: steps.cache-core-lightning.outputs.cache-hit != 'true'
+      run: |
+        cd lightning_git
+        ./configure --enable-developer --enable-rust
+        poetry install
+        poetry run make -j `nproc`
+      shell: bash
diff --git a/.github/actions/setup-lnd-client/action.yaml b/.github/actions/setup-lnd-client/action.yaml
@@ -0,0 +1,45 @@
+name: 'Setup LND Client'
+description: 'Set up LND for the Client on the runner'
+
+inputs:
+  client-ref:
+    description: 'The Git reference for the Client version of LND'
+    required: true
+    default: 'v0.16.2-breez'
+
+  go-version:
+    description: 'The Go version for building LND'
+    required: true
+    default: ^1.19
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Cache LND client
+      id: cache-lnd-client
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/go_lnd_client/bin/lnd
+        key: go_lnd_client-${{ inputs.client-ref }}-${{ inputs.go-version }}
+
+    - name: Set up Go 1.x
+      if: steps.cache-lnd-client.outputs.cache-hit != 'true'
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ inputs.go-version }}
+
+    - name: Checkout LND for Client
+      if: steps.cache-lnd-client.outputs.cache-hit != 'true'
+      uses: actions/checkout@v2
+      with:
+        repository: breez/lnd
+        ref: ${{ inputs.client-ref }}
+        path: lnd_client
+
+    - name: Build LND for Client
+      if: steps.cache-lnd-client.outputs.cache-hit != 'true'
+      run: |
+        cd lnd_client
+        env GOPATH=~/go_lnd_client make install
+      shell: bash
diff --git a/.github/actions/setup-lnd-lsp/action.yaml b/.github/actions/setup-lnd-lsp/action.yaml
@@ -0,0 +1,45 @@
+name: 'Setup LND LSP'
+description: 'Set up LND for LSP on the runner'
+
+inputs:
+  lsp-ref:
+    description: 'The Git reference for the LSP version of LND'
+    required: true
+    default: 'breez-node-v0.16.2-beta'
+
+  go-version:
+    description: 'The Go version for building LND'
+    required: true
+    default: ^1.19
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Cache LND LSP
+      id: cache-lnd-lsp
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/go_lnd_lsp/bin/lnd
+        key: go_lnd_lsp-${{ inputs.lsp-ref }}-${{ inputs.go-version }}
+
+    - name: Set up Go 1.x
+      if: steps.cache-lnd-lsp.outputs.cache-hit != 'true'
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ inputs.go-version }}
+
+    - name: Checkout LND for LSP
+      if: steps.cache-lnd-lsp.outputs.cache-hit != 'true'
+      uses: actions/checkout@v2
+      with:
+        repository: breez/lnd
+        ref: ${{ inputs.lsp-ref }}
+        path: lnd_lsp
+
+    - name: Build LND for LSP
+      if: steps.cache-lnd-lsp.outputs.cache-hit != 'true'
+      run: |
+        cd lnd_lsp
+        env GOPATH=~/go_lnd_lsp make install tags='chanreservedynamic'
+      shell: bash
diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml
@@ -0,0 +1,84 @@
+name: integration tests
+on: [push]
+jobs:
+  setup:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Bitcoin Core
+        if: steps.cache-bitcoin.outputs.cache-hit != 'true'
+        uses: ./.github/actions/setup-bitcoin
+        with:
+          bitcoin-version: '22.0'
+
+      - name: Set up LND LSP
+        if: steps.cache-lnd-lsp.outputs.cache-hit != 'true'
+        uses: ./.github/actions/setup-lnd-lsp
+        with:
+          lsp-ref: 'breez-node-v0.16.2-beta'
+          go-version: ^1.19
+
+      - name: Set up LND client
+        if: steps.cache-lnd-client.outputs.cache-hit != 'true'
+        uses: ./.github/actions/setup-lnd-client
+        with:
+          client-ref: 'v0.16.2-breez'
+          go-version: ^1.19
+
+      - name: Set up Core Lightning
+        uses: ./.github/actions/setup-clightning
+        with:
+          checkout-version: 'v23.05.1'
+
+      - name: Build LSPD
+        run: |
+          go get github.com/breez/lspd
+          go get github.com/breez/lspd/cln_plugin
+          go build .
+          go build -o lspd_plugin ./cln_plugin/cmd
+        shell: bash
+
+      - name: Test LSPD
+        run: |
+          go get github.com/breez/lspd/itest
+          TESTRE="TestLspd"
+          SKIP_TESTOFFLINENOTIFZEROCONF=1 \
+          go test -timeout 45m -v \
+            ./itest \
+            -test.run \
+            ${TESTRE} \
+            --bitcoindexec /usr/bin/bitcoind \
+            --bitcoincliexec /usr/bin/bitcoin-cli \
+            --lightningdexec ${GITHUB_WORKSPACE}/lightning_git/lightningd/lightningd \
+            --lndexec ~/go_lnd_lsp/bin/lnd \
+            --lndmobileexec ~/go_lnd_client/bin/lnd \
+            --clnpluginexec ${GITHUB_WORKSPACE}/lspd_plugin \
+            --lspdexec ${GITHUB_WORKSPACE}/lspd \
+            --lspdmigrationsdir ${GITHUB_WORKSPACE}/postgresql/migrations \
+            --testdir ~/test_state
+        shell: bash
+
+      - name: Check if test_state directory exists
+        id: check-test-state
+        run: |
+          if [ -d "~/test_state" ]; then
+            echo "::set-output name=exists::true"
+          else
+            echo "::set-output name=exists::false"
+          fi
+        shell: bash
+
+      - name: Tar state
+        run: |
+          find ~/test_state -type f -o -type d | tar -czf ~/test_state.tar.gz -T -
+        shell: bash
+        if: steps.check-test-state.outputs.exists == 'true'
+
+      - name: Upload test_state as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: test_state_artifact
+          path: ~/test_state.tar.gz
+        if: steps.check-test-state.outputs.exists == 'true'
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,4 @@
+.vscode
 go.sum
 lspd
+lspd_plugin
diff --git a/itest/lspd_test.go b/itest/lspd_test.go
@@ -3,6 +3,7 @@ package itest
 import (
 	"fmt"
 	"log"
+	"os"
 	"testing"
 	"time"
 
@@ -44,6 +45,9 @@ func runTests(
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run(fmt.Sprintf("%s: %s", prefix, testCase.name), func(t *testing.T) {
+			if testCase.skipTest {
+				t.Skip("Skipping this test case due to skipTest flag.")
+			}
 			runTest(t, testCase, prefix, lspFunc, clientFunc)
 		})
 	}
@@ -102,6 +106,7 @@ type testCase struct {
 	name          string
 	test          func(t *testParams)
 	skipCreateLsp bool
+	skipTest      bool
 	timeout       time.Duration
 }
 
@@ -160,7 +165,8 @@ var allTestCases = []*testCase{
 		test: testOfflineNotificationRegularForward,
 	},
 	{
-		name: "testOfflineNotificationZeroConfChannel",
-		test: testOfflineNotificationZeroConfChannel,
+		name:     "testOfflineNotificationZeroConfChannel",
+		test:     testOfflineNotificationZeroConfChannel,
+		skipTest: os.Getenv("SKIP_TESTOFFLINENOTIFZEROCONF") != "",
 	},
 }