diff --git a/.buildkite/bench-memory.sh b/.buildkite/bench-memory.sh new file mode 100755 index 00000000000..aa45bb387cd --- /dev/null +++ b/.buildkite/bench-memory.sh @@ -0,0 +1,49 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p coreutils gnugrep gawk time haskellPackages.hp2pretty buildkite-agent + +set -euo pipefail +echo "------------------------ Setup ------------------------------------------" +TMPDIR="${TMPDIR:-/tmp}" +export TMPDIR="/$TMPDIR/bench/memory" +mkdir -p $TMPDIR +echo "TMPDIR: $TMPDIR" + +artifact_name=memory +echo "artifact_name: $artifact_name" + +export log="$artifact_name.log" +echo "log: $log" + +export error_log="$artifact_name.error.log" +echo "error_log: $error_log" + +echo "------------------------ Setup done -------------------------------------" + +echo "------------------------ Nix call ---------------------------------------" +nix shell \ + '.#ci.benchmarks.memory' \ + '.#cardano-node' \ + '.#cardano-wallet' \ + -c "scripts/bench-memory.sh" + +echo "------------------------ Nix call done ----------------------------------" + +echo "------------------------ Results ----------------------------------------" +mv cardano-wallet.hp $artifact_name.hp +hp2pretty $artifact_name.hp + +if [ -n "${BUILDKITE:-}" ]; then + echo "--- Upload" + buildkite-agent artifact upload $artifact_name.svg + buildkite-agent artifact upload $log + buildkite-agent artifact upload $error_log + + echo "+++ Heap profile" + printf '\033]1338;url='"artifact://$artifact_name.svg"';alt='"Heap profile"'\a\n' + +fi +echo "------------------------ Results done -----------------------------------" + +echo "------------------------ Cleanup ----------------------------------------" +rm -rf $TMPDIR +echo "------------------------ Cleanup done -----------------------------------" diff --git a/.buildkite/nightly.yml b/.buildkite/nightly.yml index 9938bc2b348..66cd728c91c 100644 --- a/.buildkite/nightly.yml +++ b/.buildkite/nightly.yml @@ -56,6 +56,14 @@ steps: queue: adrestia-bench if: 'build.env("step") == null || build.env("step") =~ /bench-latency/' + - label: 'Memory benchmark' + command: "./.buildkite/bench-memory.sh" + timeout_in_minutes: 30 + agents: + system: x86_64-linux + queue: adrestia-bench + if: 'build.env("step") == null || build.env("step") =~ /bench-memory/' + # TODO: ADP-549 Port migrations test to shelley # - label: 'Database Migrations Test' # commands: diff --git a/.gitignore b/.gitignore index bc903e4d25a..a0c6241c5b6 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,13 @@ Makefile ## Lean *.olean leanpkg.path + +## Artifacts +bench-wallet +memory-time.txt +memory.log +memory +memory.svg +bench-restore +restore-time.txt +restore.log diff --git a/flake.nix b/flake.nix index 69c263c1548..7373ab7dd45 100644 --- a/flake.nix +++ b/flake.nix @@ -442,7 +442,10 @@ constituents = lib.collect lib.isDerivation packages.tests; }; - ci.benchmarks = packages.benchmarks.cardano-wallet // { + + ci.benchmarks = + packages.benchmarks.cardano-wallet-benchmarks // + packages.benchmarks.cardano-wallet // { all = pkgs.releaseTools.aggregate { name = "cardano-wallet-benchmarks"; meta.description = "Build all benchmarks"; diff --git a/lib/launcher/src/Cardano/Launcher/Node.hs b/lib/launcher/src/Cardano/Launcher/Node.hs index 217b8aebcf7..8ad04383a6a 100644 --- a/lib/launcher/src/Cardano/Launcher/Node.hs +++ b/lib/launcher/src/Cardano/Launcher/Node.hs @@ -29,7 +29,7 @@ import Data.Bifunctor import Data.List ( isPrefixOf ) import Data.Maybe - ( maybeToList ) + ( fromMaybe, maybeToList ) import Data.Text.Class ( FromText (..), TextDecodingError (..), ToText (..) ) import System.Environment @@ -95,6 +95,7 @@ data CardanoNodeConfig = CardanoNodeConfig , nodeVrfKeyFile :: Maybe FilePath , nodePort :: Maybe NodePort , nodeLoggingHostname :: Maybe String + , nodeExecutable :: Maybe FilePath } deriving (Show, Eq) -- | Spawns a @cardano-node@ process. @@ -122,7 +123,7 @@ cardanoNodeProcess cfg socketPath = do myEnv <- getEnvironment let env' = ("CARDANO_NODE_LOGGING_HOSTNAME",) <$> nodeLoggingHostname cfg - pure $ (proc "cardano-node" args) + pure $ (proc (fromMaybe "cardano-node" $ nodeExecutable cfg) args) { env = Just $ maybeToList env' ++ myEnv , cwd = Just $ nodeDir cfg } diff --git a/lib/launcher/src/Cardano/Launcher/Wallet.hs b/lib/launcher/src/Cardano/Launcher/Wallet.hs index f2b138db87b..8f51cd086fa 100644 --- a/lib/launcher/src/Cardano/Launcher/Wallet.hs +++ b/lib/launcher/src/Cardano/Launcher/Wallet.hs @@ -11,7 +11,7 @@ module Cardano.Launcher.Wallet , CardanoWalletConfig (..) , NetworkConfig (..) - -- * Run + -- * Run , CardanoWalletConn , getWalletPort ) where @@ -24,6 +24,8 @@ import Cardano.Launcher.Node ( CardanoNodeConn, nodeSocketFile ) import Control.Tracer ( Tracer (..) ) +import Data.Maybe + ( fromMaybe ) import Data.Text.Class ( FromText (..), ToText (..) ) import Network.Socket @@ -34,8 +36,9 @@ import UnliftIO.Process {----------------------------------------------------------------------------- Launching a `cardano-wallet` process ------------------------------------------------------------------------------} + -- | Parameters for connecting to the running wallet process. -newtype CardanoWalletConn = CardanoWalletConn { getWalletPort :: PortNumber } +newtype CardanoWalletConn = CardanoWalletConn {getWalletPort :: PortNumber} deriving (Show, Eq) instance ToText CardanoWalletConn where @@ -46,21 +49,25 @@ instance FromText CardanoWalletConn where data NetworkConfig = Mainnet - | Testnet { nodeByronGenesis :: FilePath } + | Testnet {nodeByronGenesis :: FilePath} deriving (Show, Eq) -- | A subset of the @cardano-wallet@ CLI parameters, -- used for starting the process. data CardanoWalletConfig = CardanoWalletConfig - { walletPort :: PortNumber - -- ^ Port number for HTTP API. Good default: 8090. - , walletDatabaseDir :: FilePath - -- ^ Path to the wallet database file. - , walletNetwork :: NetworkConfig - -- ^ Network (mainnet or a testnet) that we connect to. - , extraArgs :: [String] - -- ^ Extra arguments to be passed to the process - } deriving (Show, Eq) + { walletPort :: PortNumber + -- ^ Port number for HTTP API. Good default: 8090. + , walletDatabaseDir :: FilePath + -- ^ Path to the wallet database file. + , walletNetwork :: NetworkConfig + -- ^ Network (mainnet or a testnet) that we connect to. + , extraArgs :: [String] + -- ^ Extra arguments to be passed to the process + , executable :: Maybe FilePath + -- ^ Path to the @cardano-wallet@ executable. + , workingDir :: Maybe FilePath + } + deriving (Show, Eq) -- | Spawns a @cardano-wallet@ process. -- @@ -74,18 +81,23 @@ withCardanoWallet -- ^ Callback function with a socket filename and genesis params -> IO (Either ProcessHasExited a) withCardanoWallet tr node cfg@CardanoWalletConfig{..} action = - withBackendCreateProcess tr (cardanoWallet cfg node) $ - \_ _ -> action $ CardanoWalletConn walletPort + withBackendCreateProcess tr (cardanoWallet cfg node) + $ \_ _ -> action $ CardanoWalletConn walletPort cardanoWallet :: CardanoWalletConfig -> CardanoNodeConn -> CreateProcess cardanoWallet CardanoWalletConfig{..} node = - proc "cardano-wallet" $ - [ "serve" - , "--node-socket", nodeSocketFile node - , "--database", walletDatabaseDir - , "--port", show walletPort - ] - <> case walletNetwork of - Mainnet -> ["--mainnet"] - Testnet path -> ["--testnet", path] - <> extraArgs + + let cp = proc (fromMaybe "cardano-wallet" executable) + $ [ "serve" + , "--node-socket" + , nodeSocketFile node + , "--database" + , walletDatabaseDir + , "--port" + , show walletPort + ] + <> case walletNetwork of + Mainnet -> ["--mainnet"] + Testnet path -> ["--testnet", path] + <> extraArgs + in cp { cwd = workingDir } diff --git a/lib/wallet-benchmarks/bench/memory-benchmark.hs b/lib/wallet-benchmarks/bench/memory-benchmark.hs index 6e451242e2b..26a0539dab8 100644 --- a/lib/wallet-benchmarks/bench/memory-benchmark.hs +++ b/lib/wallet-benchmarks/bench/memory-benchmark.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PackageImports #-} {-# LANGUAGE RecordWildCards #-} import Prelude @@ -36,8 +38,11 @@ import qualified Cardano.BM.Setup as Log import qualified Cardano.Launcher as C import qualified Cardano.Launcher.Node as C import qualified Cardano.Launcher.Wallet as C +import qualified "optparse-applicative" Options.Applicative as O import qualified System.Process as S +-- See ADP-1910 for Options.Applicative usage + {----------------------------------------------------------------------------- Configuration ------------------------------------------------------------------------------} @@ -47,9 +52,6 @@ testSnapshot = "membench-snapshot.tgz" testBlockHeight :: Int testBlockHeight = 7280 -cabalDataDir :: FilePath -cabalDataDir = "lib/wallet-benchmarks/data" - testMnemonic :: [String] testMnemonic = [ "slab","praise","suffer","rabbit","during","dream" @@ -61,37 +63,81 @@ testMnemonic = {----------------------------------------------------------------------------- Main ------------------------------------------------------------------------------} + +data Config = Config + { nodeExe :: FilePath + , walletExe :: FilePath + , snapshot :: FilePath + , workingDir :: FilePath + } + +configParser :: O.Parser Config +configParser = Config + <$> O.strOption + ( O.long "node" + <> O.metavar "PATH" + <> O.help "Path to the cardano-node executable" + ) + <*> O.strOption + ( O.long "wallet" + <> O.metavar "PATH" + <> O.help "Path to the cardano-wallet executable" + ) + <*> O.strOption + ( O.long "snapshot" + <> O.metavar "PATH" + <> O.help "Path to the snapshot archive" + ) + <*> O.strOption + ( O.long "work-dir" + <> O.metavar "PATH" + <> O.help "Path to the working directory" + ) + +configInfo :: O.ParserInfo Config +configInfo = O.info (configParser O.<**> O.helper) $ mconcat + [ O.fullDesc + , O.progDesc "Run the memory benchmark" + , O.header "memory-benchmark - a benchmark for cardano-wallet memory usage" + ] + main :: IO () main = withUtf8Encoding $ do - requireExecutable "cardano-node" "--version" - requireExecutable "cardano-wallet" "version" + Config{..} <- O.execParser configInfo + requireExecutable nodeExe "--version" + requireExecutable walletExe "version" requireExecutable "curl" "--version" requireExecutable "jq" "--version" installSignalHandlers (pure ()) + trText <- initLogging "memory-benchmark" Log.Debug - let tr = contramap toText trText + let tr0 = trText + tr = contramap toText tr0 withSystemTempDirectory "wallet" $ \tmp -> do - cfg <- copyNodeSnapshot tmp - void $ withCardanoNode tr cfg $ \node -> do - sleep 2 - withCardanoWallet tr cfg node $ \wallet -> void $ do - sleep 1 - createWallet wallet testMnemonic - sleep 1 - waitUntilSynchronized wallet - -waitUntilSynchronized :: C.CardanoWalletConn -> IO () -waitUntilSynchronized wallet = do - height <- getLatestBlockHeight wallet + cfg <- copyNodeSnapshot snapshot tmp + void $ withCardanoNode tr nodeExe cfg $ \node -> do + sleep 5 + withCardanoWallet tr workingDir walletExe cfg node + $ \wallet -> void $ do + sleep 1 + createWallet wallet testMnemonic + sleep 1 + waitUntilSynchronized tr0 wallet + +waitUntilSynchronized :: Tracer IO Text -> C.CardanoWalletConn -> IO () +waitUntilSynchronized tr wallet = do + + height <- getLatestBlockHeight tr wallet + when (height < testBlockHeight) $ do - sleep 1 - waitUntilSynchronized wallet + sleep 10 + waitUntilSynchronized tr wallet -copyNodeSnapshot :: FilePath -> IO BenchmarkConfig -copyNodeSnapshot tmp = do - copyFile (cabalDataDir testSnapshot) tmp +copyNodeSnapshot :: FilePath -> FilePath -> IO BenchmarkConfig +copyNodeSnapshot snapshot tmp = do + copyFile snapshot tmp let dir = tmp takeBaseName testSnapshot decompress (tmp testSnapshot) tmp pure $ BenchmarkConfig @@ -103,13 +149,13 @@ copyNodeSnapshot tmp = do {----------------------------------------------------------------------------- Cardano commands ------------------------------------------------------------------------------} -getLatestBlockHeight :: C.CardanoWalletConn -> IO Int -getLatestBlockHeight wallet = +getLatestBlockHeight :: Tracer IO Text -> C.CardanoWalletConn -> IO Int +getLatestBlockHeight tr wallet = do fmap (fromMaybe 0 . readMaybe) - . flip S.readCreateProcess "" - . S.shell - $ curlGetCommand wallet "/wallets" - <> " | jq '.[0].tip.height.quantity'" + . flip S.readCreateProcess "" + . S.shell + $ curlGetCommand wallet "/wallets" + <> " | jq '.[0].tip.height.quantity'" curlGetCommand :: C.CardanoWalletConn -> String -> String @@ -154,11 +200,13 @@ data BenchmarkConfig = BenchmarkConfig -- | Start a `cardano-wallet` process on the benchmark configuration. withCardanoWallet :: Tracer IO C.LauncherLog + -> FilePath + -> FilePath -> BenchmarkConfig -> C.CardanoNodeConn -> (C.CardanoWalletConn -> IO r) -> IO (Either C.ProcessHasExited r) -withCardanoWallet tr BenchmarkConfig{..} node = +withCardanoWallet tr workingDir walletExe BenchmarkConfig{..} node action = do C.withCardanoWallet tr node C.CardanoWalletConfig { C.walletPort = @@ -169,17 +217,22 @@ withCardanoWallet tr BenchmarkConfig{..} node = walletDatabaseDir , C.extraArgs = profilingOptions + , C.executable = + Just walletExe + , C.workingDir = Just workingDir } + action where profilingOptions = words "+RTS -N1 -qg -A1m -I0 -T -h -i0.01 -RTS" -- | Start a `cardano-node` process on the benchmark configuration. withCardanoNode :: Tracer IO C.LauncherLog + -> FilePath -> BenchmarkConfig -> (C.CardanoNodeConn -> IO r) -> IO (Either C.ProcessHasExited r) -withCardanoNode tr BenchmarkConfig{..} = +withCardanoNode tr nodeExe BenchmarkConfig{..} = C.withCardanoNode tr C.CardanoNodeConfig { C.nodeDir = nodeDatabaseDir @@ -193,6 +246,7 @@ withCardanoNode tr BenchmarkConfig{..} = , C.nodeVrfKeyFile = Nothing , C.nodePort = Just (C.NodePort 8061) , C.nodeLoggingHostname = Nothing + , C.nodeExecutable = Just nodeExe } {----------------------------------------------------------------------------- diff --git a/lib/wallet-benchmarks/cardano-wallet-benchmarks.cabal b/lib/wallet-benchmarks/cardano-wallet-benchmarks.cabal index ee70a32cfdb..d8693507d2d 100644 --- a/lib/wallet-benchmarks/cardano-wallet-benchmarks.cabal +++ b/lib/wallet-benchmarks/cardano-wallet-benchmarks.cabal @@ -12,6 +12,7 @@ copyright: 2023 Cardano Foundation license: Apache-2.0 category: Web build-type: Simple +data-files: data/membench-snapshot.tgz common language ghc-options: -threaded -Wall @@ -26,6 +27,8 @@ benchmark memory type: exitcode-stdio-1.0 hs-source-dirs: bench main-is: memory-benchmark.hs + other-modules: + Paths_cardano_wallet_benchmarks build-depends: , base , cardano-wallet-launcher @@ -41,3 +44,4 @@ benchmark memory , text , text-class , transformers + , optparse-applicative diff --git a/lib/wallet/api/http/Cardano/Wallet/Launch/Cluster.hs b/lib/wallet/api/http/Cardano/Wallet/Launch/Cluster.hs index d963e0cd700..5d2c02a14d9 100644 --- a/lib/wallet/api/http/Cardano/Wallet/Launch/Cluster.hs +++ b/lib/wallet/api/http/Cardano/Wallet/Launch/Cluster.hs @@ -567,6 +567,7 @@ configurePool tr baseDir era metadataServer recipe = do , nodeVrfKeyFile = Just vrfPrv , nodePort = Just (NodePort port) , nodeLoggingHostname = Just name + , nodeExecutable = Nothing } withCardanoNodeProcess tr name cfg $ \socket -> do @@ -1319,6 +1320,7 @@ withBFTNode tr baseDir params action = , nodeVrfKeyFile = Just vrfPrv , nodePort = Just (NodePort port) , nodeLoggingHostname = Just name + , nodeExecutable = Nothing } withCardanoNodeProcess tr name cfg $ \socket -> @@ -1366,6 +1368,7 @@ _withRelayNode tr baseDir params act = , nodeVrfKeyFile = Nothing , nodePort = Just (NodePort port) , nodeLoggingHostname = Just name + , nodeExecutable = Nothing } let act' socket = act $ RunningNode socket block0 (bp, vd) [] diff --git a/lib/wallet/bench/src/Cardano/Wallet/BenchShared.hs b/lib/wallet/bench/src/Cardano/Wallet/BenchShared.hs index 36986049be6..cfea80948d6 100644 --- a/lib/wallet/bench/src/Cardano/Wallet/BenchShared.hs +++ b/lib/wallet/bench/src/Cardano/Wallet/BenchShared.hs @@ -168,6 +168,7 @@ withNetworkConfiguration args action = do , nodeVrfKeyFile = Nothing , nodePort = Just (NodePort port) , nodeLoggingHostname = Nothing + , nodeExecutable = Nothing } argsNetworkDir :: RestoreBenchArgs -> FilePath diff --git a/lib/wallet/src/Cardano/Wallet/Primitive/Slotting.hs b/lib/wallet/src/Cardano/Wallet/Primitive/Slotting.hs index 0b65590dccd..b7044159ea7 100644 --- a/lib/wallet/src/Cardano/Wallet/Primitive/Slotting.hs +++ b/lib/wallet/src/Cardano/Wallet/Primitive/Slotting.hs @@ -433,7 +433,7 @@ instance ToText TimeInterpreterLog where , renderPastHorizonException e t0 ] MsgInterpreterPastHorizon (Just reason) t0 e -> mconcat - [ "Time interpreter queried past the horizon. " + [ "Time interpreter was queried past the horizon. " , "This should not happen because " , T.pack reason , renderPastHorizonException e t0 diff --git a/nix/haskell.nix b/nix/haskell.nix index 6a7952b3447..b6855c983f1 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -142,6 +142,7 @@ CHaP: haskell-nix: nixpkgs-recent: nodePkgs: haskell-nix.cabalProject' [ yq nixWrapped mdbook + (haskell-nix.tool "ghc8107" "hp2pretty" "latest") (haskell-nix.tool "ghc8107" "stylish-haskell" "0.11.0.3") (haskell-nix.tool "ghc8107" "hlint" "3.3") (haskell-nix.tool "ghc928" "fourmolu" "0.13.0.0") diff --git a/scripts/bench-memory.sh b/scripts/bench-memory.sh new file mode 100755 index 00000000000..b3fbf04a2fb --- /dev/null +++ b/scripts/bench-memory.sh @@ -0,0 +1,21 @@ +#! /usr/bin/env bash +echo "------------------------ Setup nix ---------------------------------------" +wallet_exe="$(which cardano-wallet)" +echo "wallet_exe: $wallet_exe" +node_exe="$(which cardano-node)" +echo "node_exe: $node_exe" +node_db="$(pwd)/lib/wallet-benchmarks/data/membench-snapshot.tgz" +echo "node_db: $node_db" +work_dir="$(pwd)" +echo "work_dir: $work_dir" +bench="memory \ + --snapshot=$node_db \ + --wallet=$wallet_exe \ + --node=$node_exe \ + --work-dir=$work_dir " +echo "bench command: $bench" +echo "------------------------ Setup nix done ---------------------------------" + +echo "------------------------ Run --------------------------------------------" +$bench 2>"${error_log:?}" 1>"${log:?}" +echo "------------------------ Run done ---------------------------------------"