scripts: Add patching script for Agave devs (#5)

* scripts: Add patching script for Agave devs

#### Problem

Now that the solana-sdk is split out of the Agave repo, it will be
annoying for Agave developers to make simultaneous changes to the SDK
and Agave.

#### Summary of changes

Add a script for patching the solana-sdk in Agave, along with a CI check
to make sure that the SDK doesn't break Agave.

* Pacify shellcheck

* Install extra requirements for agave

* Simplify the check

Author: joncinque

.github/actions/setup/action.yml

@@ -28,6 +28,9 @@ inputs:
   llvm-tools-preview:
     description: Install llvm-tools-preview if `true`. Defaults to `false`.
     required: false
+  agave:
+    description: Install required packages for Agave if `true`. Defaults to `false`.
+    required: false
   purge:
     description: Purge unused directories if `true`. Defaults to `false`.
     required: false
@@ -105,6 +108,11 @@ runs:
         toolchain: ${{ env.RUST_NIGHTLY }}
         components: llvm-tools-preview
 
+    - name: Install Agave build dependencies
+      if: ${{ inputs.agave == 'true' }}
+      shell: bash
+      run: sudo apt update && sudo apt install -y libudev-dev protobuf-compiler
+
     - name: Cache Cargo Dependencies
       if: ${{ inputs.cargo-cache-key && !inputs.cargo-cache-fallback-key }}
       uses: actions/cache@v4

.github/workflows/main.yml

@@ -101,6 +101,25 @@ jobs:
       - name: Run checks
         run: ./scripts/check-nightly.sh
 
+  check-downstream-agave:
+    name: Cargo check Agave master
+    runs-on: ubuntu-latest
+    needs: [sanity]
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup
+        with:
+          agave: true
+          stable-toolchain: true
+          cargo-cache-key: cargo-downstream-agave-check
+          cargo-cache-fallback-key: cargo-downstream-agave
+
+      - name: Run checks
+        run: ./scripts/check-downstream-agave.sh
+
   clippy:
     name: Clippy
     needs: [sanity]

README.md

@@ -0,0 +1,17 @@
+# solana-sdk
+
+Rust SDK for the Solana blockchain, used by on-chain programs and the Agave
+validator.
+
+## For Agave Developers
+
+### Patching a local solana-sdk repository
+
+If your change to Agave also entails changes to the SDK, you will need to patch
+your Agave repo to use a local checkout of solana-sdk crates.
+
+To patch all of the crates in this repo for Agave, just run:
+
+```console
+./scripts/patch-crates-no-header.sh <AGAVE_PATH> <SOLANA_SDK_PATH>
+```

scripts/check-downstream-agave.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+here="$(dirname "$0")"
+src_root="$(readlink -f "${here}/..")"
+cd "${src_root}"
+
+rm -rf ./target/downstream/agave
+mkdir -p ./target/downstream
+cd ./target/downstream
+git clone --depth 1 https://github.com/anza-xyz/agave.git --single-branch --branch=master
+cd ./agave
+
+../../../scripts/patch-crates-no-header.sh . ../../..
+cargo check

scripts/patch-crates-functions.sh

@@ -0,0 +1,162 @@
+# source this file
+
+base="$(dirname "${BASH_SOURCE[0]}")"
+# pacify shellcheck: cannot follow dynamic path
+# shellcheck disable=SC1090,SC1091
+source "$base/read-cargo-variable.sh"
+
+crate_dirs=(
+  account
+  account-info
+  address-lookup-table-interface
+  atomic-u64
+  big-mod-exp
+  bincode
+  blake3-hasher
+  bn254
+  borsh
+  client-traits
+  clock
+  cluster-type
+  commitment-config
+  compute-budget-interface
+  cpi
+  decode-error
+  define-syscall
+  derivation-path
+  ed25519-program
+  epoch-info
+  epoch-rewards
+  epoch-rewards-hasher
+  epoch-schedule
+  example-mocks
+  feature-gate-interface
+  feature-set
+  fee-calculator
+  fee-structure
+  file-download
+  frozen-abi
+  frozen-abi-macro
+  genesis-config
+  hard-forks
+  hash
+  inflation
+  instruction
+  instructions-sysvar
+  keccak-hasher
+  keypair
+  last-restart-slot
+  loader-v2-interface
+  loader-v3-interface
+  loader-v4-interface
+  logger
+  message
+  msg
+  native-token
+  nonce
+  nonce-account
+  offchain-message
+  package-metadata
+  package-metadata-macro
+  packet
+  poh-config
+  precompile-error
+  precompiles
+  presigner
+  program
+  program-entrypoint
+  program-error
+  program-memory
+  program-option
+  program-pack
+  pubkey
+  quic-definitions
+  rent
+  rent-collector
+  rent-debits
+  reserved-account-keys
+  reward-info
+  sanitize
+  scripts
+  sdk
+  sdk-ids
+  sdk-macro
+  secp256k1-program
+  secp256k1-recover
+  secp256r1-program
+  seed-derivable
+  seed-phrase
+  serde
+  serde-varint
+  serialize-utils
+  sha256-hasher
+  short-vec
+  shred-version
+  signature
+  signer
+  slot-hashes
+  slot-history
+  stable-layout
+  system-transaction
+  sysvar
+  sysvar-id
+  target
+  time-utils
+  transaction
+  transaction-context
+  transaction-error
+  validator-exit
+  vote-interface
+)
+
+update_solana_sdk_dependencies() {
+  declare project_root="$1"
+  declare solana_sdk_dir="$2"
+  declare tomls=()
+  while IFS='' read -r line; do tomls+=("$line"); done < <(find "$project_root" -name Cargo.toml)
+
+  set -x
+  for crate_dir in "${crate_dirs[@]}"; do
+    full_path="$solana_sdk_dir/$crate_dir"
+    crate_version=$(readCargoVariable version "$full_path/Cargo.toml")
+    sed -E -i'' -e "s:(solana-${crate_dir} = \")([=<>]*)[0-9.]+([^\"]*)\".*:\1\2${crate_version}\3\":" "${tomls[@]}"
+    sed -E -i'' -e "s:(solana-${crate_dir} = \{ version = \")([=<>]*)[0-9.]+([^\"]*)(\".*):\1\2${crate_version}\3\4:" "${tomls[@]}"
+  done
+}
+
+patch_crates_io_solana_sdk() {
+  declare Cargo_toml="$1"
+  declare solana_sdk_dir="$2"
+  cat >> "$Cargo_toml" <<EOF
+[patch.crates-io]
+EOF
+  patch_crates_io_solana_sdk_no_header "$Cargo_toml" "$solana_sdk_dir"
+}
+
+patch_crates_io_solana_sdk_no_header() {
+  declare Cargo_toml="$1"
+  declare solana_sdk_dir="$2"
+
+  full_path_solana_sdk_dir="$(cd "$solana_sdk_dir" && pwd -P)"
+  patch_crates=()
+  for crate_dir in "${crate_dirs[@]}"; do
+    full_path="$full_path_solana_sdk_dir/$crate_dir"
+    if [[ -r "$full_path/Cargo.toml" ]]; then
+      patch_crates+=("solana-${crate_dir} = { path = \"$full_path\" }")
+    fi
+  done
+
+  echo "Patching in crates from $solana_sdk_dir"
+  echo
+  if grep -q "# The following entries are auto-generated by $0" "$Cargo_toml"; then
+    echo "$Cargo_toml is already patched"
+  else
+    if ! grep -q '\[patch.crates-io\]' "$Cargo_toml"; then
+      echo "[patch.crates-io]" >> "$Cargo_toml"
+    fi
+    cat >> "$Cargo_toml" <<PATCH
+# The following entries are auto-generated by $0
+$(printf "%s\n" "${patch_crates[@]}")
+PATCH
+  fi
+}

scripts/patch-crates-no-header.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+here="$(dirname "$0")"
+# pacify shellcheck: cannot follow dynamic path
+# shellcheck disable=SC1090,SC1091
+source "$here"/patch-crates-functions.sh
+
+usage() {
+  cat <<EOF >&2
+USAGE:
+    $0 <AGAVE_PATH> <SOLANA_SDK_PATH>
+
+ARGS:
+    <AGAVE_PATH>        Path to the root of an agave repo
+    <SOLANA_SDK_PATH>   Path to the root of a solana-sdk repo
+EOF
+}
+
+agave_path="$1"
+if [ -z "$agave_path" ]; then
+  usage
+  exit 1
+fi
+
+solana_sdk_path="$2"
+if [ -z "$solana_sdk_path" ]; then
+  usage
+  exit 1
+fi
+
+update_solana_sdk_dependencies "$agave_path" "$solana_sdk_path"
+patch_crates_io_solana_sdk_no_header "$agave_path"/Cargo.toml "$solana_sdk_path"