Merge pull request #874 from IntersectMBO/smelc/fix-create-testnet-da…

…ta-default-total-supply

create-testnet-data: better behavior for create-testnet-data --total-supply and --delegated-supply

cardano-cli/cardano-cli.cabal

@@ -292,6 +292,7 @@ library cardano-cli-test-lib
     bytestring,
     cardano-api,
     cardano-cli,
+    containers,
     directory,
     exceptions,
     filepath,
@@ -302,6 +303,7 @@ library cardano-cli-test-lib
     process,
     text,
     transformers-base,
+    vector,
 
 test-suite cardano-cli-test
   import: project-config

cardano-cli/src/Cardano/CLI/EraBased/Options/Genesis.hs

@@ -328,10 +328,9 @@ pGenesisCreateTestNetData sbe envCli =
             , Opt.metavar "LOVELACE"
             , Opt.help $
                 mconcat
-                  [ "The maximum possible amount of Lovelace, which is evenly distributed across stake holders. Defaults to 1 million Ada (i.e. 10^12 Lovelace)."
+                  [ "The maximum possible amount of Lovelace, which is evenly distributed across stake holders. Overrides the value from the shelley genesis."
                   , " If --delegated-supply is specified, a part of this amount will be delegated."
                   ]
-            , Opt.value 1_000_000_000_000
             ]
   pSupplyDelegated :: Parser (Maybe Coin)
   pSupplyDelegated =
@@ -343,10 +342,9 @@ pGenesisCreateTestNetData sbe envCli =
             , Opt.metavar "LOVELACE"
             , Opt.help $
                 mconcat
-                  [ "The amount of the total supply which is evenly delegated. Defaults to 500 000 Ada (i.e. (10^12) / 2 Lovelace)."
+                  [ "The amount of the total supply which is evenly delegated. Defaulted to half of the total supply."
                   , " Cannot be more than the amount specified with --total-supply."
                   ]
-            , Opt.value 500_000_000_000
             ]
   pRelays :: Parser FilePath
   pRelays =

cardano-cli/src/Cardano/CLI/EraBased/Run/Genesis/CreateTestnetData.hs

@@ -738,7 +738,7 @@ updateOutputTemplate
     totalSupply = fromIntegral $ maybe maximumLovelaceSupply unLovelace mTotalSupply
 
     delegCoinRaw, nonDelegCoinRaw :: Integer
-    delegCoinRaw = maybe 0 unLovelace mDelegatedSupply
+    delegCoinRaw = maybe (totalSupply `div` 2) unLovelace mDelegatedSupply
     -- Since the user can specify total supply and delegated amount, the non-delegated amount is:
     nonDelegCoinRaw = totalSupply - delegCoinRaw
 

cardano-cli/test/cardano-cli-golden/Test/Golden/CreateTestnetData.hs

@@ -1,15 +1,16 @@
 {-# LANGUAGE NumericUnderscores #-}
 {-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TupleSections #-}
 
 module Test.Golden.CreateTestnetData where
 
+import           Cardano.Api
 import           Cardano.Api.Ledger (ConwayGenesis (..), StandardCrypto)
 import           Cardano.Api.Shelley (ShelleyGenesis (..))
 
 import qualified Cardano.Ledger.Shelley.API as L
 
 import           Control.Monad
-import           Control.Monad.IO.Class
 import           Data.List (intercalate, sort)
 import qualified Data.Sequence.Strict as Seq
 import           Data.Word (Word32)
@@ -18,6 +19,7 @@ import           System.Directory
 import           System.Directory.Extra (listDirectories)
 import           System.FilePath
 
+import           Test.Cardano.CLI.Aeson
 import           Test.Cardano.CLI.Util (FileSem, bracketSem, execCardanoCLI, newFileSem)
 
 import           Hedgehog (Property)
@@ -191,3 +193,43 @@ hprop_golden_create_testnet_data_deleg_non_deleg =
     -- https://github.com/IntersectMBO/cardano-cli/issues/631 witnesses.
 
     H.assertWith (fromIntegral onlyHolderCoin) (\ohc -> ohc > totalSupply `div` 2)
+
+-- Execute this test with:
+-- @cabal test cardano-cli-golden --test-options '-p "/golden create testnet data shelley genesis output/"'@
+--
+-- This test checks that we use the total supply value from the shelley template when --total-supply
+-- is not specified. It was broken in the past (see https://github.com/IntersectMBO/cardano-node/issues/5953).
+hprop_golden_create_testnet_data_shelley_genesis_output :: Property
+hprop_golden_create_testnet_data_shelley_genesis_output =
+  propertyOnce $ moduleWorkspace "tmp" $ \tempDir -> do
+    vanillaShelleyGenesis :: ShelleyGenesis StandardCrypto <-
+      H.readJsonFileOk "test/cardano-cli-golden/files/input/shelley/genesis/genesis.spec.json"
+    let tweakedValue = 3_123_456_000_000
+        tweakedShelleyGenesis = vanillaShelleyGenesis{L.sgMaxLovelaceSupply = tweakedValue}
+        tweakedShelleyGenesisFp = tempDir </> "tweaked-shelley-genesis.json"
+
+    void $ liftIO $ writeFileJSON tweakedShelleyGenesisFp tweakedShelleyGenesis
+
+    let outputDir = tempDir </> "out"
+    void $
+      execCardanoCLI
+        [ "conway"
+        , "genesis"
+        , "create-testnet-data"
+        , "--spec-shelley"
+        , tweakedShelleyGenesisFp
+        , "--out-dir"
+        , outputDir
+        ]
+
+    let outputGenesisFp = outputDir </> "shelley-genesis.json"
+        redactedOutputGenesisFp = outputDir </> "shelley-genesis-redacted.json"
+
+    redactJsonFieldsInFile
+      (fromList $ map (,"<redacted>") ["genDelegs", "systemStart"])
+      outputGenesisFp
+      redactedOutputGenesisFp
+
+    H.diffFileVsGoldenFile
+      redactedOutputGenesisFp
+      "test/cardano-cli-golden/files/golden/conway/custom-lovelace-supply-shelley-genesis.json"

cardano-cli/test/cardano-cli-golden/files/golden/conway/custom-lovelace-supply-shelley-genesis.json

@@ -0,0 +1,43 @@
+{
+    "activeSlotsCoeff": 0.99,
+    "epochLength": 21600,
+    "genDelegs": "<redacted>",
+    "initialFunds": {},
+    "maxKESEvolutions": 1080000,
+    "maxLovelaceSupply": 3123456000000,
+    "networkId": "Testnet",
+    "networkMagic": 403,
+    "protocolParams": {
+        "a0": 0,
+        "decentralisationParam": 0.99,
+        "eMax": 0,
+        "extraEntropy": {
+            "tag": "NeutralNonce"
+        },
+        "keyDeposit": 0,
+        "maxBlockBodySize": 2097152,
+        "maxBlockHeaderSize": 8192,
+        "maxTxSize": 2048,
+        "minFeeA": 0,
+        "minFeeB": 0,
+        "minPoolCost": 100,
+        "minUTxOValue": 1,
+        "nOpt": 100,
+        "poolDeposit": 0,
+        "protocolVersion": {
+            "major": 0,
+            "minor": 0
+        },
+        "rho": 0,
+        "tau": 0
+    },
+    "securityParam": 2160,
+    "slotLength": 20,
+    "slotsPerKESPeriod": 216000,
+    "staking": {
+        "pools": {},
+        "stake": {}
+    },
+    "systemStart": "<redacted>",
+    "updateQuorum": 12
+}

cardano-cli/test/cardano-cli-golden/files/golden/help/allegra_genesis_create-testnet-data.cli

@@ -45,15 +45,15 @@ Available options:
                            is 0).
   --utxo-keys INT          The number of UTxO keys to make (default is 0).
   --total-supply LOVELACE  The maximum possible amount of Lovelace, which is
-                           evenly distributed across stake holders. Defaults to
-                           1 million Ada (i.e. 10^12 Lovelace). If
+                           evenly distributed across stake holders. Overrides
+                           the value from the shelley genesis. If
                            --delegated-supply is specified, a part of this
                            amount will be delegated.
   --delegated-supply LOVELACE
                            The amount of the total supply which is evenly
-                           delegated. Defaults to 500 000 Ada (i.e. (10^12) / 2
-                           Lovelace). Cannot be more than the amount specified
-                           with --total-supply.
+                           delegated. Defaulted to half of the total supply.
+                           Cannot be more than the amount specified with
+                           --total-supply.
   --testnet-magic NATURAL  Specify a testnet magic id for the cluster. This
                            overrides both the network magic from the spec file
                            and CARDANO_NODE_NETWORK_ID environment variable.

cardano-cli/test/cardano-cli-golden/files/golden/help/alonzo_genesis_create-testnet-data.cli

@@ -45,15 +45,15 @@ Available options:
                            is 0).
   --utxo-keys INT          The number of UTxO keys to make (default is 0).
   --total-supply LOVELACE  The maximum possible amount of Lovelace, which is
-                           evenly distributed across stake holders. Defaults to
-                           1 million Ada (i.e. 10^12 Lovelace). If
+                           evenly distributed across stake holders. Overrides
+                           the value from the shelley genesis. If
                            --delegated-supply is specified, a part of this
                            amount will be delegated.
   --delegated-supply LOVELACE
                            The amount of the total supply which is evenly
-                           delegated. Defaults to 500 000 Ada (i.e. (10^12) / 2
-                           Lovelace). Cannot be more than the amount specified
-                           with --total-supply.
+                           delegated. Defaulted to half of the total supply.
+                           Cannot be more than the amount specified with
+                           --total-supply.
   --testnet-magic NATURAL  Specify a testnet magic id for the cluster. This
                            overrides both the network magic from the spec file
                            and CARDANO_NODE_NETWORK_ID environment variable.

cardano-cli/test/cardano-cli-golden/files/golden/help/babbage_genesis_create-testnet-data.cli

@@ -45,15 +45,15 @@ Available options:
                            is 0).
   --utxo-keys INT          The number of UTxO keys to make (default is 0).
   --total-supply LOVELACE  The maximum possible amount of Lovelace, which is
-                           evenly distributed across stake holders. Defaults to
-                           1 million Ada (i.e. 10^12 Lovelace). If
+                           evenly distributed across stake holders. Overrides
+                           the value from the shelley genesis. If
                            --delegated-supply is specified, a part of this
                            amount will be delegated.
   --delegated-supply LOVELACE
                            The amount of the total supply which is evenly
-                           delegated. Defaults to 500 000 Ada (i.e. (10^12) / 2
-                           Lovelace). Cannot be more than the amount specified
-                           with --total-supply.
+                           delegated. Defaulted to half of the total supply.
+                           Cannot be more than the amount specified with
+                           --total-supply.
   --testnet-magic NATURAL  Specify a testnet magic id for the cluster. This
                            overrides both the network magic from the spec file
                            and CARDANO_NODE_NETWORK_ID environment variable.

cardano-cli/test/cardano-cli-golden/files/golden/help/conway_genesis_create-testnet-data.cli

@@ -45,15 +45,15 @@ Available options:
                            is 0).
   --utxo-keys INT          The number of UTxO keys to make (default is 0).
   --total-supply LOVELACE  The maximum possible amount of Lovelace, which is
-                           evenly distributed across stake holders. Defaults to
-                           1 million Ada (i.e. 10^12 Lovelace). If
+                           evenly distributed across stake holders. Overrides
+                           the value from the shelley genesis. If
                            --delegated-supply is specified, a part of this
                            amount will be delegated.
   --delegated-supply LOVELACE
                            The amount of the total supply which is evenly
-                           delegated. Defaults to 500 000 Ada (i.e. (10^12) / 2
-                           Lovelace). Cannot be more than the amount specified
-                           with --total-supply.
+                           delegated. Defaulted to half of the total supply.
+                           Cannot be more than the amount specified with
+                           --total-supply.
   --testnet-magic NATURAL  Specify a testnet magic id for the cluster. This
                            overrides both the network magic from the spec file
                            and CARDANO_NODE_NETWORK_ID environment variable.

cardano-cli/test/cardano-cli-golden/files/golden/help/latest_genesis_create-testnet-data.cli

@@ -45,15 +45,15 @@ Available options:
                            is 0).
   --utxo-keys INT          The number of UTxO keys to make (default is 0).
   --total-supply LOVELACE  The maximum possible amount of Lovelace, which is
-                           evenly distributed across stake holders. Defaults to
-                           1 million Ada (i.e. 10^12 Lovelace). If
+                           evenly distributed across stake holders. Overrides
+                           the value from the shelley genesis. If
                            --delegated-supply is specified, a part of this
                            amount will be delegated.
   --delegated-supply LOVELACE
                            The amount of the total supply which is evenly
-                           delegated. Defaults to 500 000 Ada (i.e. (10^12) / 2
-                           Lovelace). Cannot be more than the amount specified
-                           with --total-supply.
+                           delegated. Defaulted to half of the total supply.
+                           Cannot be more than the amount specified with
+                           --total-supply.
   --testnet-magic NATURAL  Specify a testnet magic id for the cluster. This
                            overrides both the network magic from the spec file
                            and CARDANO_NODE_NETWORK_ID environment variable.

cardano-cli/test/cardano-cli-golden/files/golden/help/mary_genesis_create-testnet-data.cli

@@ -45,15 +45,15 @@ Available options:
                            is 0).
   --utxo-keys INT          The number of UTxO keys to make (default is 0).
   --total-supply LOVELACE  The maximum possible amount of Lovelace, which is
-                           evenly distributed across stake holders. Defaults to
-                           1 million Ada (i.e. 10^12 Lovelace). If
+                           evenly distributed across stake holders. Overrides
+                           the value from the shelley genesis. If
                            --delegated-supply is specified, a part of this
                            amount will be delegated.
   --delegated-supply LOVELACE
                            The amount of the total supply which is evenly
-                           delegated. Defaults to 500 000 Ada (i.e. (10^12) / 2
-                           Lovelace). Cannot be more than the amount specified
-                           with --total-supply.
+                           delegated. Defaulted to half of the total supply.
+                           Cannot be more than the amount specified with
+                           --total-supply.
   --testnet-magic NATURAL  Specify a testnet magic id for the cluster. This
                            overrides both the network magic from the spec file
                            and CARDANO_NODE_NETWORK_ID environment variable.

cardano-cli/test/cardano-cli-golden/files/golden/help/shelley_genesis_create-testnet-data.cli

@@ -45,15 +45,15 @@ Available options:
                            is 0).
   --utxo-keys INT          The number of UTxO keys to make (default is 0).
   --total-supply LOVELACE  The maximum possible amount of Lovelace, which is
-                           evenly distributed across stake holders. Defaults to
-                           1 million Ada (i.e. 10^12 Lovelace). If
+                           evenly distributed across stake holders. Overrides
+                           the value from the shelley genesis. If
                            --delegated-supply is specified, a part of this
                            amount will be delegated.
   --delegated-supply LOVELACE
                            The amount of the total supply which is evenly
-                           delegated. Defaults to 500 000 Ada (i.e. (10^12) / 2
-                           Lovelace). Cannot be more than the amount specified
-                           with --total-supply.
+                           delegated. Defaulted to half of the total supply.
+                           Cannot be more than the amount specified with
+                           --total-supply.
   --testnet-magic NATURAL  Specify a testnet magic id for the cluster. This
                            overrides both the network magic from the spec file
                            and CARDANO_NODE_NETWORK_ID environment variable.

cardano-cli/test/cardano-cli-test-lib/Test/Cardano/CLI/Aeson.hs

@@ -4,17 +4,22 @@ module Test.Cardano.CLI.Aeson
   ( assertEqualModuloDesc
   , assertHasKeys
   , assertHasMappings
+  , redactJsonFieldsInFile
   )
 where
 
 import           Control.Monad (forM_)
 import           Control.Monad.IO.Class
 import           Data.Aeson hiding (pairs)
+import           Data.Aeson.Encode.Pretty (encodePretty)
+import           Data.Aeson.Key
 import qualified Data.Aeson.Key as Aeson
 import qualified Data.Aeson.KeyMap as Aeson.KeyMap
 import qualified Data.ByteString.Lazy as LBS
+import qualified Data.Map as Map
 import           Data.Text (Text)
 import qualified Data.Text as T
+import qualified Data.Vector as Vector
 import           GHC.Stack (HasCallStack)
 import qualified GHC.Stack as GHC
 
@@ -176,3 +181,46 @@ removeDescription v =
   failWrongType got = do
     H.note_ $ "Expected object but got: " <> got
     H.failure
+
+-- | @redactJsonStringFieldInFile [(k0, v0), (k1, v1), ..] sourceFilePath targetFilePath@ reads the JSON at @sourceFilePath@, and then
+-- replaces the value associated to @k0@ by @v0@, replaces the value associated to @k1@ by @v1@, etc.
+-- Then the obtained JSON is written to @targetFilePath@. This replacement is done recursively
+-- so @k0@, @k1@, etc. can appear at any depth within the JSON.
+redactJsonFieldsInFile
+  :: ()
+  => MonadTest m
+  => MonadIO m
+  => HasCallStack
+  => Map.Map Text Text
+  -- ^ Map from key name, to the new (String) value to attach to this key
+  -> FilePath
+  -> FilePath
+  -> m ()
+redactJsonFieldsInFile changes sourceFilePath targetFilePath = GHC.withFrozenCallStack $ do
+  contents <- H.evalIO $ LBS.readFile sourceFilePath
+  case eitherDecode contents :: Either String Value of
+    Left err -> do
+      H.note_ $ "Failed to decode JSON: " <> err
+      H.success
+    Right json -> do
+      let redactedJson = redactJsonFields changes json
+      H.evalIO $ LBS.writeFile targetFilePath $ encodePretty redactedJson
+
+redactJsonFields :: () => Map.Map Text Text -> Value -> Value
+redactJsonFields changes v =
+  case v of
+    Object obj ->
+      let obj' =
+            Aeson.KeyMap.mapWithKey
+              ( \k v' ->
+                  case Map.lookup (toText k) changes of
+                    Just replacement -> String replacement
+                    Nothing -> recurse v'
+              )
+              obj
+       in Object obj'
+    Array vector ->
+      Array $ Vector.map recurse vector
+    _ -> v
+ where
+  recurse = redactJsonFields changes