test: more readable logging for smoke tests (#5099)

## Describe your changes

Moving the smoke test invocations out of the process-compose wrapper,
preferring running raw `cargo test` invocations instead. Doing so
ensures that the test output is readable, both locally, and crucially in
CI.

### Screenshot before

![smoke-test-before](https://github.com/user-attachments/assets/219b8518-8525-463e-8465-2c16f787d3f6)

### Screenshot after

![smoke-logs-after](https://github.com/user-attachments/assets/b7721d89-c969-4870-8d33-b9aa8dbc6ac9)


## Issue ticket number and link

No guiding issue, just trying to improve the testing setup
opportunistically. We did have inscrutable smoke test failures on #5063
& #5081, both of which were failures of the new pindexer integration
tests (#5057), but it was hard to see the specific failure at a glance,
which slowed down development.

## Testing and review

Check the CI logs for the smoke test job. Are they readable? Consider
running `just smoke` locally and confirm the same.

## Checklist before requesting a review

- [x] I have added guiding text to explain how a reviewer should test
these changes.

- [x] If this code contains consensus-breaking changes, I have added the
"consensus-breaking" label. Otherwise, I declare my belief that there
are not consensus-breaking changes, for the following reason:

  > tests-only, no code changes

.github/workflows/smoke.yml

@@ -39,10 +39,6 @@ jobs:
       - name: run the smoke-test suite
         run: nix develop --command just smoke
 
-      - name: Display smoke-test logs
-        if: always()
-        run: cat deployments/logs/smoke-*.log
-
   pmonitor:
     runs-on: buildjet-16vcpu-ubuntu-2204
     steps:

crates/bin/pcli/tests/network_integration.rs

@@ -23,6 +23,7 @@ use predicates::prelude::*;
 use regex::Regex;
 use serde_json::Value;
 use tempfile::{tempdir, NamedTempFile, TempDir};
+use url::Url;
 
 use penumbra_sdk_keys::test_keys::{ADDRESS_0_STR, ADDRESS_1_STR, SEED_PHRASE};
 use penumbra_sdk_proto::core::transaction::v1::TransactionView as ProtoTransactionView;
@@ -39,9 +40,9 @@ const TIMEOUT_COMMAND_SECONDS: u64 = 20;
 // The "unbonding_delay" value is specified in blocks, and in the smoke tests,
 // block time is set to ~500ms, so we'll take the total number of blocks
 // that must elapse and sleep half that many seconds.
-static UNBONDING_DURATION: Lazy<Duration> = Lazy::new(|| {
+static UNBONDING_DELAY: Lazy<Duration> = Lazy::new(|| {
     let blocks: f64 = std::env::var("UNBONDING_DELAY")
-        .unwrap_or("100".to_string())
+        .unwrap_or("50".to_string())
         .parse()
         .unwrap();
     // 0.5 -> 0.6 for comfort, since 500ms is only an estimate.
@@ -52,12 +53,19 @@ static UNBONDING_DURATION: Lazy<Duration> = Lazy::new(|| {
 fn load_wallet_into_tmpdir() -> TempDir {
     let tmpdir = tempdir().unwrap();
 
+    let grpc_url: Url = std::env::var("PENUMBRA_NODE_PD_URL")
+        .unwrap_or_else(|_| "http://127.0.0.1:8080".to_owned())
+        .parse()
+        .expect("failed to parse PENUMBRA_NODE_PD_URL");
+
     let mut setup_cmd = Command::cargo_bin("pcli").unwrap();
     setup_cmd
         .args([
             "--home",
             tmpdir.path().to_str().unwrap(),
             "init",
+            "--grpc-url",
+            &grpc_url.to_string(),
             "soft-kms",
             "import-phrase",
         ])
@@ -341,7 +349,7 @@ fn delegate_and_undelegate() {
     tracing::info!("undelegation succeeded, wait an epoch before claiming.");
 
     // Wait for the epoch duration.
-    thread::sleep(*UNBONDING_DURATION);
+    thread::sleep(*UNBONDING_DELAY);
     tracing::info!("epoch passed, claiming now");
     let mut undelegate_claim_cmd = Command::cargo_bin("pcli").unwrap();
     undelegate_claim_cmd
@@ -649,7 +657,7 @@ fn swap() {
     swap_cmd.assert().success();
 
     // Sleep to allow the outputs from the swap to be processed.
-    thread::sleep(*UNBONDING_DURATION);
+    thread::sleep(*UNBONDING_DELAY);
     let mut balance_cmd = Command::cargo_bin("pcli").unwrap();
     balance_cmd
         .args(["--home", tmpdir.path().to_str().unwrap(), "view", "balance"])
@@ -684,7 +692,7 @@ fn swap() {
     close_cmd.assert().success();
 
     // Wait for processing.
-    thread::sleep(*UNBONDING_DURATION);
+    thread::sleep(*UNBONDING_DELAY);
     let mut withdraw_cmd = Command::cargo_bin("pcli").unwrap();
     withdraw_cmd
         .args([
@@ -699,7 +707,7 @@ fn swap() {
         .timeout(std::time::Duration::from_secs(TIMEOUT_COMMAND_SECONDS));
     withdraw_cmd.assert().success();
 
-    thread::sleep(*UNBONDING_DURATION);
+    thread::sleep(*UNBONDING_DELAY);
     let mut balance_cmd = Command::cargo_bin("pcli").unwrap();
     balance_cmd
         .args(["--home", tmpdir.path().to_str().unwrap(), "view", "balance"])

deployments/scripts/run-local-devnet.sh

@@ -36,8 +36,11 @@ else
     use_tui="false"
 fi
 
+# Set unique API port for controlling running services.
+export PC_PORT_NUM="8888"
+
 # Run the core fullnode config, plus any additional params passed via `$@`.
-process-compose up --no-server \
+process-compose up \
     --ordered-shutdown \
     --tui="$use_tui" \
     --config "${repo_root}/deployments/compose/process-compose.yml" \

deployments/scripts/smoke-test.sh

@@ -34,19 +34,30 @@ fi
 
 >&2 echo "Building all test targets before running smoke tests..."
 # We want a warm cache before the tests run
-cargo build --release -p pcli -p pclientd -p pd
+cargo build --release --bins
+
+smoke_test_dir="${repo_root:?}/deployments/.smoke-test-state"
+rm -rf "$smoke_test_dir"
+mkdir -p "$smoke_test_dir"
 
 # Reuse existing dev-env script
-if ! "${repo_root}/deployments/scripts/run-local-devnet.sh" \
-        --config ./deployments/compose/process-compose-metrics.yml \
-        --config ./deployments/compose/process-compose-dev-tooling.yml \
-        --config ./deployments/compose/process-compose-postgres.yml \
-        --config ./deployments/compose/process-compose-smoke-test.yml \
-        ; then
-    >&2 echo "ERROR: smoke tests failed"
-    >&2 echo "Review logs in: deployments/logs/smoke-*.log"
-    find "${repo_root}/deployments/logs/smoke-"*".log" | sort >&2
-    exit 1
-else
-    echo "SUCCESS! Smoke test complete."
-fi
+"${repo_root}/deployments/scripts/run-local-devnet.sh" \
+    --config ./deployments/compose/process-compose-metrics.yml \
+    --config ./deployments/compose/process-compose-dev-tooling.yml \
+    --config ./deployments/compose/process-compose-postgres.yml \
+    --detached
+
+# Wait a bit for network to start.
+sleep 10
+
+# Ensure that process-compose environment gets cleaned up, even if tests error.
+trap 'process-compose down --port 8888' EXIT
+
+# Run the integration tests. Using `just` targets so that the exact
+# invocations are easily reusable on the CLI in dev loops.
+just integration-pclientd
+just integration-pcli
+# The pd tests come later, as they need work to have been performed for metrics to be emitted.
+just integration-pd
+# Finally, pindexer tests, to make assertions about emitted events.
+just integration-pindexer

flake.nix

@@ -181,6 +181,7 @@
               export LIBCLANG_PATH=${LIBCLANG_PATH}
               export RUST_SRC_PATH=${pkgs.rustPlatform.rustLibSrc} # Required for rust-analyzer
               export ROCKSDB_LIB_DIR=${ROCKSDB_LIB_DIR}
+              export RUST_LOG="info,network_integration=debug,pclientd=debug,pcli=info,pd=info,penumbra=info"
             '';
           };
         }

justfile

@@ -65,6 +65,24 @@ integration-pmonitor:
 
 # Run smoke test suite, via process-compose config.
 smoke:
-    ./deployments/scripts/check-nix-shell
-    ./deployments/scripts/warn-about-pd-state
-    ./deployments/scripts/smoke-test.sh
+  ./deployments/scripts/check-nix-shell
+  ./deployments/scripts/warn-about-pd-state
+  ./deployments/scripts/smoke-test.sh
+
+# Run integration tests for pclientd. Assumes specific dev env is already running.
+integration-pclientd:
+  cargo test --release --features sct-divergence-check --package pclientd -- \
+    --ignored --test-threads 1 --nocapture
+
+# Run integration tests for pcli. Assumes specific dev env is already running.
+integration-pcli:
+  cargo test --release --features sct-divergence-check,download-proving-keys --package pcli -- \
+    --ignored --test-threads 1 --nocapture
+
+# Run integration tests for pindexer. Assumes specific dev env is already running.
+integration-pindexer:
+  cargo nextest run --release -p pindexer --features network-integration
+
+# Run integration tests for pd. Assumes specific dev env is already running.
+integration-pd:
+  cargo test --release --package pd -- --ignored --test-threads 1 --nocapture