diff --git a/automation/terraform/monitoring/o1-testnet-alerts.tf b/automation/terraform/monitoring/o1-testnet-alerts.tf index 8e9205daa95a..8bf9f118d145 100644 --- a/automation/terraform/monitoring/o1-testnet-alerts.tf +++ b/automation/terraform/monitoring/o1-testnet-alerts.tf @@ -22,7 +22,7 @@ module "o1testnet_alerts" { alert_timeframe = "1h" alert_duration = "10m" pagerduty_alert_filter = "devnet2|mainnet" - berkeley_testnet = "testnet=\"(berkeley|testworld-2-0)\"" + berkeley_testnet = "testnet=~\"(berkeley|testworld-2-0)\"" synced_status_filter = "syncStatus=\"SYNCED\"" } diff --git a/graphql_schema.json b/graphql_schema.json index 30802b858476..0bd86aa2767a 100644 --- a/graphql_schema.json +++ b/graphql_schema.json @@ -12280,6 +12280,54 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "snarkPoolDiffReceived", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "Int", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "snarkPoolDiffBroadcasted", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "Int", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pendingSnarkWork", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "Int", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "snarkPoolSize", + "description": null, + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "Int", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index 82e76c5dac7b..038551dbb152 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -542,7 +542,7 @@ let setup_daemon logger = ~transport: (Logger_file_system.dumb_logrotate ~directory:conf_dir ~log_filename:"mina-oversized-logs.log" - ~max_size:logrotate_max_size ~num_rotate:file_log_rotations ) ; + ~max_size:logrotate_max_size ~num_rotate:20 ) ; (* Consumer for `[%log internal]` logging used for internal tracing *) Itn_logger.set_message_postprocessor Internal_tracing.For_itn_logger.post_process_message ; diff --git a/src/app/itn_orchestrator/schema.graphql b/src/app/itn_orchestrator/schema.graphql index ab0ff73abcc7..351e868273bb 100644 --- a/src/app/itn_orchestrator/schema.graphql +++ b/src/app/itn_orchestrator/schema.graphql @@ -173,6 +173,12 @@ scalar UInt16 """Keys and other information for scheduling zkapp commands""" input ZkappCommandsDetails { + """ + Parameter of zkapp generation, each generated zkapp tx will have + (2*maxAccountUpdates+2) account updates (including balancing and fee payer) + """ + maxAccountUpdates: Int + """Generate max cost zkApp command""" maxCost: Boolean! @@ -193,18 +199,18 @@ input ZkappCommandsDetails { """ initBalance: CurrencyAmount! + """Maximum new zkapp balance""" + maxNewZkappBalance: CurrencyAmount! + + """Minimum new zkapp balance""" + minNewZkappBalance: CurrencyAmount! + """Maximum balance change""" maxBalanceChange: CurrencyAmount! """Minimum balance change""" minBalanceChange: CurrencyAmount! - """Minimum new zkapp balance""" - minNewZkappBalance: CurrencyAmount! - - """Maximum new zkapp balance""" - maxNewZkappBalance: CurrencyAmount! - """Disable the precondition in account updates""" noPrecondition: Boolean! diff --git a/src/app/itn_orchestrator/src/generate.go b/src/app/itn_orchestrator/src/generate.go new file mode 100644 index 000000000000..dac361bd1839 --- /dev/null +++ b/src/app/itn_orchestrator/src/generate.go @@ -0,0 +1,409 @@ +package itn_orchestrator + +import ( + "encoding/json" + "fmt" + "itn_json_types" + "math" + "math/rand" + "sort" + "strconv" + "strings" +) + +type GenParams struct { + MinTps, BaseTps, StressTps, SenderRatio, ZkappRatio, NewAccountRatio float64 + StopCleanRatio, MinStopRatio, MaxStopRatio float64 + RoundDurationMin, PauseMin, Rounds, StopsPerRound, Gap int + SendFromNonBpsOnly, StopOnlyBps, UseRestartScript, MaxCost bool + ExperimentName, PasswordEnv, FundKeyPrefix string + Privkeys []string + PaymentReceiver itn_json_types.MinaPublicKey + PrivkeysPerFundCmd int + GenerateFundKeys int + RotationKeys, RotationServers []string + RotationPermutation bool + RotationRatio float64 + MixMaxCostTpsRatio float64 + LargePauseEveryNRounds, LargePauseMin int + MinBalanceChange, MaxBalanceChange, DeploymentFee uint64 + PaymentAmount, MinZkappFee, MaxZkappFee, FundFee uint64 + MinPaymentFee, MaxPaymentFee uint64 +} + +type GeneratedCommand struct { + Action string `json:"action"` + Params any `json:"params"` + comment string +} + +func (cmd *GeneratedCommand) Comment() string { + return cmd.comment +} + +type GeneratedRound struct { + Commands []GeneratedCommand + PaymentFundCommand *FundParams + ZkappFundCommand *FundParams +} + +func withComment(comment string, cmd GeneratedCommand) GeneratedCommand { + cmd.comment = comment + return cmd +} + +func formatDur(min, sec int) string { + sec += min * 60 + min = sec / 60 + sec %= 60 + hour := min / 60 + min %= 60 + day := hour / 24 + hour %= 24 + parts := []string{} + if day > 0 { + parts = append(parts, strconv.Itoa(day), "days") + } + if hour > 0 { + parts = append(parts, strconv.Itoa(hour), "hours") + } + if min > 0 { + parts = append(parts, strconv.Itoa(min), "mins") + } + if sec > 0 { + parts = append(parts, strconv.Itoa(sec), "secs") + } + if len(parts) == 0 { + return "immediately" + } + return strings.Join(parts, " ") +} + +func rotate(p RotateParams) GeneratedCommand { + return GeneratedCommand{Action: RotateAction{}.Name(), Params: p} +} + +func loadKeys(p KeyloaderParams) GeneratedCommand { + return GeneratedCommand{Action: KeyloaderAction{}.Name(), Params: p} +} + +func discovery(p DiscoveryParams) GeneratedCommand { + return GeneratedCommand{Action: DiscoveryAction{}.Name(), Params: p} +} + +type SampleRefParams struct { + Group ComplexValue `json:"group"` + Ratios []float64 `json:"ratios"` +} + +func sample(groupRef int, groupName string, ratios []float64) GeneratedCommand { + group := LocalComplexValue(groupRef, groupName) + group.OnEmpty = emptyArrayRawMessage + return GeneratedCommand{Action: SampleAction{}.Name(), Params: SampleRefParams{ + Group: group, + Ratios: ratios, + }} +} + +type ZkappRefParams struct { + ZkappSubParams + FeePayers ComplexValue `json:"feePayers"` + Nodes ComplexValue `json:"nodes"` +} + +func zkapps(feePayersRef int, nodesRef int, nodesName string, params ZkappSubParams) GeneratedCommand { + cmd := GeneratedCommand{Action: ZkappCommandsAction{}.Name(), Params: ZkappRefParams{ + ZkappSubParams: params, + FeePayers: LocalComplexValue(feePayersRef, "key"), + Nodes: LocalComplexValue(nodesRef, nodesName), + }} + maxCostStr := "" + if params.MaxCost { + maxCostStr = "max-cost " + } + comment := fmt.Sprintf("Scheduling %d %szkapp transactions to be sent over period of %d minutes (%.2f txs/min)", + int(params.Tps*float64(params.DurationMin)*60), maxCostStr, params.DurationMin, params.Tps*60, + ) + return withComment(comment, cmd) +} + +type PaymentRefParams struct { + PaymentSubParams + FeePayers ComplexValue `json:"feePayers"` + Nodes ComplexValue `json:"nodes"` +} + +func payments(feePayersRef int, nodesRef int, nodesName string, params PaymentSubParams) GeneratedCommand { + cmd := GeneratedCommand{Action: PaymentsAction{}.Name(), Params: PaymentRefParams{ + PaymentSubParams: params, + FeePayers: LocalComplexValue(feePayersRef, "key"), + Nodes: LocalComplexValue(nodesRef, nodesName), + }} + comment := fmt.Sprintf("Scheduling %d payments to be sent over period of %d minutes (%.2f txs/min)", + int(params.Tps*float64(params.DurationMin)*60), params.DurationMin, params.Tps*60, + ) + return withComment(comment, cmd) +} + +func waitMin(min int) GeneratedCommand { + return GeneratedCommand{Action: WaitAction{}.Name(), Params: WaitParams{ + Minutes: min, + }} +} + +func GenWait(sec int) GeneratedCommand { + return GeneratedCommand{Action: WaitAction{}.Name(), Params: WaitParams{ + Seconds: sec, + }} +} + +type RestartRefParams struct { + Nodes ComplexValue `json:"nodes"` + Clean bool `json:"clean,omitempty"` +} + +/* + * Returns random number in normal distribution centering on 0. + * ~95% of numbers returned should fall between -2 and 2 + * ie within two standard deviations + */ +func gaussRandom() float64 { + u := 2*rand.Float64() - 1 + v := 2*rand.Float64() - 1 + r := u*u + v*v + // if outside interval [0,1] start over + if r == 0 || r >= 1 { + return gaussRandom() + } + + c := math.Sqrt(-2 * math.Log(r) / r) + return u * c +} + +func SampleTps(baseTps, stressTps float64) float64 { + tpsStddev := (stressTps - baseTps) / 2 + return tpsStddev*math.Abs(gaussRandom()) + baseTps +} + +func SampleStopRatio(minRatio, maxRatio float64) float64 { + stddev := (maxRatio - minRatio) / 3 + return stddev*math.Abs(gaussRandom()) + minRatio +} + +func genStopDaemon(useRestartScript bool, nodesRef int, nodesName string, clean bool) GeneratedCommand { + var name string + if useRestartScript { + name = RestartAction{}.Name() + } else { + name = StopDaemonAction{}.Name() + } + return GeneratedCommand{Action: name, Params: RestartRefParams{ + Nodes: LocalComplexValue(nodesRef, nodesName), + Clean: clean, + }} +} + +type JoinRefParams struct { + Group1 ComplexValue `json:"group1"` + Group2 ComplexValue `json:"group2"` +} + +func join(g1Ref int, g1Name string, g2Ref int, g2Name string) GeneratedCommand { + return GeneratedCommand{Action: JoinAction{}.Name(), Params: JoinRefParams{ + Group1: LocalComplexValue(g1Ref, g1Name), + Group2: LocalComplexValue(g2Ref, g2Name), + }} +} + +type ExceptRefParams struct { + Group ComplexValue `json:"group"` + Except ComplexValue `json:"except"` +} + +var emptyArrayRawMessage json.RawMessage + +func init() { + emptyArrayRawMessage, _ = json.Marshal([]string{}) +} + +func except(groupRef int, groupName string, exceptRef int, exceptName string) GeneratedCommand { + group := LocalComplexValue(groupRef, groupName) + group.OnEmpty = emptyArrayRawMessage + except := LocalComplexValue(exceptRef, exceptName) + except.OnEmpty = emptyArrayRawMessage + return GeneratedCommand{Action: ExceptAction{}.Name(), Params: ExceptRefParams{ + Group: group, + Except: except, + }} +} +func (p *GenParams) Generate(round int) GeneratedRound { + zkappsKeysDir := fmt.Sprintf("%s/%s/round-%d/zkapps", p.FundKeyPrefix, p.ExperimentName, round) + paymentsKeysDir := fmt.Sprintf("%s/%s/round-%d/payments", p.FundKeyPrefix, p.ExperimentName, round) + tps := SampleTps(p.BaseTps, p.StressTps) + maxCost := p.MaxCost + zkappRatio := p.ZkappRatio + if p.MixMaxCostTpsRatio > 1e-3 && (round&1) == 1 { + maxCost = true + zkappRatio = 1 + tps *= p.MixMaxCostTpsRatio + } + experimentName := fmt.Sprintf("%s-%d", p.ExperimentName, round) + onlyZkapps := math.Abs(1-zkappRatio) < 1e-3 + onlyPayments := zkappRatio < 1e-3 + zkappTps := tps * zkappRatio + zkappParams := ZkappSubParams{ + ExperimentName: experimentName, + Tps: zkappTps, + MinTps: p.MinTps, + DurationMin: p.RoundDurationMin, + Gap: p.Gap, + MinBalanceChange: p.MinBalanceChange, + MaxBalanceChange: p.MaxBalanceChange, + MinFee: p.MinZkappFee, + MaxFee: p.MaxZkappFee, + DeploymentFee: p.DeploymentFee, + MaxCost: maxCost, + NewAccountRatio: p.NewAccountRatio, + } + if maxCost { + // This can be set to arbitrary value as for max-cost it only + // matters that total zkapps deployed is above 5 + // We need to set it this way to override setting accountQueueSize + // by the orchestrator + zkappParams.ZkappsToDeploy = 8 + zkappParams.NewAccountRatio = 0 + } + paymentParams := PaymentSubParams{ + ExperimentName: experimentName, + Tps: tps - zkappTps, + MinTps: p.MinTps, + DurationMin: p.RoundDurationMin, + MinFee: p.MinPaymentFee, + MaxFee: p.MaxPaymentFee, + Amount: p.PaymentAmount, + Receiver: p.PaymentReceiver, + } + cmds := []GeneratedCommand{} + roundStartMin := round*(p.RoundDurationMin+p.PauseMin) + round/p.LargePauseEveryNRounds*p.LargePauseMin + if len(p.RotationKeys) > 0 { + var mapping []int + nKeys := len(p.RotationKeys) + if p.RotationPermutation { + mapping = rand.Perm(nKeys) + } else { + mapping = make([]int, nKeys) + for i := range mapping { + mapping[i] = rand.Intn(len(p.RotationKeys)) + } + } + cmds = append(cmds, rotate(RotateParams{ + Pubkeys: p.RotationKeys, + RestServers: p.RotationServers, + Mapping: mapping, + Ratio: p.RotationRatio, + PasswordEnv: p.PasswordEnv, + })) + } + cmds = append(cmds, withComment(fmt.Sprintf("Starting round %d, %s after start", round, formatDur(roundStartMin, 0)), discovery(DiscoveryParams{ + OffsetMin: 15, + NoBlockProducers: p.SendFromNonBpsOnly, + }))) + sendersOutName := "participant" + if 1-p.SenderRatio > 1e-6 { + sendersOutName = "group1" + cmds = append(cmds, sample(-1, "participant", []float64{p.SenderRatio})) + } + if onlyPayments { + cmds = append(cmds, loadKeys(KeyloaderParams{Dir: paymentsKeysDir})) + cmds = append(cmds, payments(-1, -2, sendersOutName, paymentParams)) + } else if onlyZkapps { + cmds = append(cmds, loadKeys(KeyloaderParams{Dir: zkappsKeysDir})) + cmds = append(cmds, zkapps(-1, -2, sendersOutName, zkappParams)) + } else { + cmds = append(cmds, loadKeys(KeyloaderParams{Dir: zkappsKeysDir})) + cmds = append(cmds, loadKeys(KeyloaderParams{Dir: paymentsKeysDir})) + cmds = append(cmds, zkapps(-2, -3, sendersOutName, zkappParams)) + cmds = append(cmds, payments(-2, -4, sendersOutName, paymentParams)) + cmds = append(cmds, join(-1, "participant", -2, "participant")) + } + sendersCmdId := len(cmds) + stopWaits := make([]int, p.StopsPerRound) + for i := 0; i < p.StopsPerRound; i++ { + stopWaits[i] = rand.Intn(60 * p.RoundDurationMin) + } + sort.Ints(stopWaits) + for i := p.StopsPerRound - 1; i > 0; i-- { + stopWaits[i] -= stopWaits[i-1] + } + stopRatio := SampleStopRatio(p.MinStopRatio, p.MaxStopRatio) + elapsed := 0 + for _, waitSec := range stopWaits { + cmds = append(cmds, withComment(fmt.Sprintf("Running round %d, %s after start, waiting for %s", round, formatDur(roundStartMin, elapsed), formatDur(0, waitSec)), GenWait(waitSec))) + cmds = append(cmds, discovery(DiscoveryParams{ + OffsetMin: 15, + OnlyBlockProducers: p.StopOnlyBps, + })) + exceptRefName := "group" + if onlyPayments || onlyZkapps { + exceptRefName = "participant" + } + cmds = append(cmds, except(-1, "participant", sendersCmdId-len(cmds)-1, exceptRefName)) + stopCleanRatio := p.StopCleanRatio * stopRatio + stopNoCleanRatio := (1 - p.StopCleanRatio) * stopRatio + nodesOrBps := "nodes" + if p.StopOnlyBps { + nodesOrBps = "block producers" + } + if stopCleanRatio > 1e-6 && stopNoCleanRatio > 1e-6 { + cmds = append(cmds, sample(-1, "group", []float64{stopCleanRatio, stopNoCleanRatio})) + comment1 := fmt.Sprintf("Stopping %.1f%% %s with cleaning", stopCleanRatio*100, nodesOrBps) + cmds = append(cmds, withComment(comment1, genStopDaemon(p.UseRestartScript, -1, "group1", true))) + comment2 := fmt.Sprintf("Stopping %.1f%% %s without cleaning", stopNoCleanRatio*100, nodesOrBps) + cmds = append(cmds, withComment(comment2, genStopDaemon(p.UseRestartScript, -2, "group2", false))) + } else if stopCleanRatio > 1e-6 { + comment := fmt.Sprintf("Stopping %.1f%% %s with cleaning", stopCleanRatio*100, nodesOrBps) + cmds = append(cmds, sample(-1, "group", []float64{stopCleanRatio})) + cmds = append(cmds, withComment(comment, genStopDaemon(p.UseRestartScript, -1, "group1", true))) + } else if stopNoCleanRatio > 1e-6 { + comment := fmt.Sprintf("Stopping %.1f%% %s without cleaning", stopNoCleanRatio*100, nodesOrBps) + cmds = append(cmds, sample(-1, "group", []float64{stopNoCleanRatio})) + cmds = append(cmds, withComment(comment, genStopDaemon(p.UseRestartScript, -1, "group1", false))) + } + elapsed += waitSec + } + if round < p.Rounds-1 { + comment1 := fmt.Sprintf("Waiting for remainder of round %d, %s after start", round, formatDur(roundStartMin, elapsed)) + cmds = append(cmds, withComment(comment1, GenWait(p.RoundDurationMin*60-elapsed))) + if p.PauseMin > 0 { + comment2 := fmt.Sprintf("Pause after round %d, %s after start", round, formatDur(roundStartMin+p.RoundDurationMin, 0)) + cmds = append(cmds, withComment(comment2, waitMin(p.PauseMin))) + } + if p.LargePauseMin > 0 && (round+1)%p.LargePauseEveryNRounds == 0 { + comment3 := fmt.Sprintf("Large pause after round %d, %s after start", round, formatDur(roundStartMin+p.RoundDurationMin+p.PauseMin, 0)) + cmds = append(cmds, withComment(comment3, waitMin(p.LargePauseMin))) + } + } + res := GeneratedRound{Commands: cmds} + if !onlyPayments { + _, _, _, initBalance := ZkappBalanceRequirements(zkappTps, zkappParams) + zkappKeysNum, zkappAmount := ZkappKeygenRequirements(initBalance, zkappParams) + res.ZkappFundCommand = &FundParams{ + PasswordEnv: p.PasswordEnv, + Prefix: zkappsKeysDir + "/key", + Amount: zkappAmount, + Fee: p.FundFee, + Num: zkappKeysNum, + } + } + if !onlyZkapps { + paymentKeysNum, paymentAmount := PaymentKeygenRequirements(p.Gap, paymentParams) + res.PaymentFundCommand = &FundParams{ + PasswordEnv: p.PasswordEnv, + Prefix: paymentsKeysDir + "/key", + Amount: paymentAmount, + Fee: p.FundFee, + Num: paymentKeysNum, + } + } + return res +} diff --git a/src/app/itn_orchestrator/src/generate_test.go b/src/app/itn_orchestrator/src/generate_test.go new file mode 100644 index 000000000000..ff820806f698 --- /dev/null +++ b/src/app/itn_orchestrator/src/generate_test.go @@ -0,0 +1,252 @@ +package itn_orchestrator + +import ( + "math/rand" + "testing" +) + +func someParams() GenParams { + return GenParams{ + MinTps: 0.01, + BaseTps: 0.6, + StressTps: 1.4, + SenderRatio: 0.8, + ZkappRatio: 0.7, + NewAccountRatio: 1.5, + StopCleanRatio: 0.5, + MinStopRatio: 0.1, + MaxStopRatio: 0.25, + RoundDurationMin: 50, + PauseMin: 10, + Rounds: 48, + StopsPerRound: 1, + Gap: 100, + ExperimentName: "test", + Privkeys: []string{"key0"}, + PaymentReceiver: "B62qpPita1s7Dbnr7MVb3UK8fdssZixL1a4536aeMYxbTJEtRGGyS8U", + PrivkeysPerFundCmd: 2, + GenerateFundKeys: 20, + MixMaxCostTpsRatio: 0.7, + LargePauseEveryNRounds: 8, + LargePauseMin: 240, + MinBalanceChange: 1e3, + MaxBalanceChange: 3e3, + DeploymentFee: 1e9, + PaymentAmount: 1e5, + MinZkappFee: 1e9, + MaxZkappFee: 3e9, + MinPaymentFee: 1e8, + MaxPaymentFee: 3e8, + FundFee: 1e9, + } +} + +type shuffledIxs []int + +func (ixs *shuffledIxs) Push(ix int) { + *ixs = append(*ixs, ix) +} + +func (ixs *shuffledIxs) Pop() int { + l := len(*ixs) + if l == 0 { + panic("unexpected pop") + } + i := rand.Intn(l) + res := (*ixs)[i] + (*ixs)[i] = (*ixs)[l-1] + *ixs = (*ixs)[:l-1] + return res +} + +type zkappGenState struct { + balances []int64 + availableDeployed shuffledIxs + availableNew shuffledIxs + queue []int + numNewAccounts int + conf ZkappCommandsDetails +} + +func newZkappGenState(pi ZkappCommandsDetails, numFeePayers int, balances []int64) zkappGenState { + availableDeployed := rand.Perm(pi.NumZkappsToDeploy) + for i := range availableDeployed { + availableDeployed[i] += numFeePayers + } + return zkappGenState{ + numNewAccounts: pi.NumNewAccounts, + balances: balances, + availableDeployed: availableDeployed, + conf: pi, + } +} + +func (state *zkappGenState) applyTx(numFeePayers, feePayerIx, accountUpdates, newAccounts, zkappAccounts int) { + state.numNewAccounts -= newAccounts + state.balances[feePayerIx] -= int64(state.conf.MaxFee) + availableDeployed := len(state.availableDeployed) + availableNew := len(state.availableNew) + tot := availableDeployed + availableNew + if tot <= 0 { + panic("not enough keys") + } + // TODO popped should be reused + popped := make([]int, 0, accountUpdates-newAccounts) + poppedCnt := make([]int, accountUpdates-newAccounts+1) + numNewPops := 0 + for i := 0; i < accountUpdates-newAccounts-zkappAccounts; i++ { + r := rand.Intn(tot) + if r < len(popped) { + poppedCnt[r]++ + numNewPops++ + } else if r < availableNew { + popped = append(popped, state.availableNew.Pop()) + numNewPops++ + } + } + numDeployedToPop := accountUpdates - newAccounts - numNewPops + 1 + newActuallyPopped := len(popped) + for i := 0; i < numDeployedToPop; i++ { + r := rand.Intn(availableDeployed) + newActuallyPopped + if r < len(popped) { + poppedCnt[r]++ + } else { + popped = append(popped, state.availableDeployed.Pop()) + } + } + balancing := popped[len(popped)-1] + poppedCnt[len(popped)-1]-- + state.queue = append(state.queue, popped...) + for i := 0; i < newAccounts; i++ { + ix := len(state.balances) + state.balances = append(state.balances, int64(state.conf.MinNewZkappBalance)-1e9) + state.queue = append(state.queue, ix) + } + if len(state.queue) > state.conf.AccountQueueSize { + prefix := len(state.queue) - state.conf.AccountQueueSize + for _, ix := range state.queue[:prefix] { + if ix < numFeePayers+state.conf.NumZkappsToDeploy { + state.availableDeployed.Push(ix) + } else { + state.availableNew.Push(ix) + } + } + state.queue = state.queue[prefix:] + } + for i, ix := range popped { + state.balances[ix] -= int64(state.conf.MaxBalanceChange*2) * int64(poppedCnt[i]+1) + } + state.balances[balancing] -= int64(accountUpdates*2-newAccounts)*int64(state.conf.MaxBalanceChange) + int64(newAccounts)*int64(state.conf.MaxNewZkappBalance) +} + +func testZkapp(t *testing.T, numFeePayers int, feePayerBalance int64, pi ZkappCommandsDetails) { + balances := make([]int64, numFeePayers+pi.NumZkappsToDeploy) + for i := 0; i < numFeePayers; i++ { + balances[i] = feePayerBalance + } + for j := 0; j < pi.NumZkappsToDeploy; j++ { + balances[j%numFeePayers] -= int64(pi.DeploymentFee + pi.InitBalance) + balances[numFeePayers+j] = int64(pi.InitBalance) - 1e9 + } + numTxs := int(float64(pi.DurationMin*60) * pi.Tps) + if pi.MaxCost { + for j := pi.NumZkappsToDeploy; j < numTxs; j++ { + balances[j%numFeePayers] -= int64(pi.MaxFee) + } + } else { + state := newZkappGenState(pi, numFeePayers, balances) + for j := pi.NumZkappsToDeploy; j < numTxs; j++ { + accUpdates := rand.Intn(pi.MaxAccountUpdates) + 1 + newAccs := 0 + if state.numNewAccounts > 0 { + newAccs = rand.Intn(accUpdates + 1) + } + zkappAccs := 0 + for k := 0; k < accUpdates-newAccs; k++ { + if rand.Intn(3) == 0 { + zkappAccs++ + } + } + // t.Logf("%d:%d (%d new)", accUpdates*2+2, zkappAccs, newAccs) + state.applyTx(numFeePayers, j%numFeePayers, accUpdates, newAccs, zkappAccs) + } + balances = state.balances + } + for ix, b := range balances { + type_ := "new" + if ix < numFeePayers { + type_ = "fee payer" + } else if ix < numFeePayers+pi.NumZkappsToDeploy { + type_ = "zkapp" + } + if b < 0 { + t.Errorf("balance underflow %d for %s account", b, type_) + } + } + if t.Failed() { + t.Logf("Fee payer balance: %d", feePayerBalance) + t.Logf("Init zkapp balance: %d", pi.InitBalance) + t.Logf("New account balance: %d", pi.MaxNewZkappBalance) + } +} + +func testPayment(t *testing.T, numFeePayers int, feePayerBalance int64, pi PaymentsDetails) { + balances := make([]int64, numFeePayers) + for i := 0; i < numFeePayers; i++ { + balances[i] = feePayerBalance + } + numTxs := int(float64(pi.DurationMin*60) * pi.Tps) + for i := 0; i < numTxs; i++ { + balances[i%numFeePayers] -= int64(pi.Amount + pi.MaxFee) + } + for _, b := range balances { + if b < 0 { + t.Errorf("balance underflow %d for fee payer", b) + } + } + if t.Failed() { + t.Logf("Fee payer balance: %d", feePayerBalance) + } +} + +func TestGenerate(t *testing.T) { + for i := 0; i < 100; i++ { + params := someParams() + for r := 0; r < params.Rounds; r++ { + round := params.Generate(r) + var zkappParams *ZkappSubParams + var paymentParams *PaymentSubParams + for _, c := range round.Commands { + if c.Action == (ZkappCommandsAction{}).Name() { + p := (c.Params.(ZkappRefParams).ZkappSubParams) + zkappParams = &p + } else if c.Action == (PaymentsAction{}).Name() { + p := (c.Params.(PaymentRefParams).PaymentSubParams) + paymentParams = &p + } + } + if zkappParams != nil { + fund := *round.ZkappFundCommand + participants := int(zkappParams.Tps / zkappParams.MinTps) + tpsPerNode := zkappParams.Tps / float64(participants) + pi := ZkappPaymentsInput(*zkappParams, 0, tpsPerNode) + numFeePayers := fund.Num / participants + feePayerBalance := int64(fund.Amount) / int64(fund.Num) + for j := 0; j < 1000; j++ { + testZkapp(t, numFeePayers, feePayerBalance, pi) + } + } + if paymentParams != nil { + fund := *round.PaymentFundCommand + participants := int(paymentParams.Tps / paymentParams.MinTps) + tpsPerNode := paymentParams.Tps / float64(participants) + numFeePayers := fund.Num / participants + feePayerBalance := int64(fund.Amount) / int64(fund.Num) + pi := paymentInput(*paymentParams, 0, tpsPerNode) + for j := 0; j < 1000; j++ { + testPayment(t, numFeePayers, feePayerBalance, pi) + } + } + } + } +} diff --git a/src/app/itn_orchestrator/src/generator/main.go b/src/app/itn_orchestrator/src/generator/main.go index 2504e9948437..7d67ad65a9bc 100644 --- a/src/app/itn_orchestrator/src/generator/main.go +++ b/src/app/itn_orchestrator/src/generator/main.go @@ -4,417 +4,14 @@ import ( "encoding/json" "flag" "fmt" - "itn_json_types" - "math" - "math/rand" "os" - "sort" - "strconv" "strings" lib "itn_orchestrator" ) -/* - * Returns random number in normal distribution centering on 0. - * ~95% of numbers returned should fall between -2 and 2 - * ie within two standard deviations - */ -func gaussRandom() float64 { - u := 2*rand.Float64() - 1 - v := 2*rand.Float64() - 1 - r := u*u + v*v - // if outside interval [0,1] start over - if r == 0 || r >= 1 { - return gaussRandom() - } - - c := math.Sqrt(-2 * math.Log(r) / r) - return u * c -} - -func sampleTps(baseTps, stressTps float64) float64 { - tpsStddev := (stressTps - baseTps) / 2 - return tpsStddev*math.Abs(gaussRandom()) + baseTps -} - -func sampleStopRatio(minRatio, maxRatio float64) float64 { - stddev := (maxRatio - minRatio) / 3 - return stddev*math.Abs(gaussRandom()) + minRatio -} - -type Params struct { - MinTps, BaseTps, StressTps, SenderRatio, ZkappRatio, NewAccountRatio float64 - StopCleanRatio, MinStopRatio, MaxStopRatio float64 - RoundDurationMin, PauseMin, Rounds, StopsPerRound, Gap int - SendFromNonBpsOnly, StopOnlyBps, UseRestartScript, MaxCost bool - ExperimentName, PasswordEnv, FundKeyPrefix string - Privkeys []string - PaymentReceiver itn_json_types.MinaPublicKey - PrivkeysPerFundCmd int - GenerateFundKeys int - RotationKeys, RotationServers []string - RotationPermutation bool - RotationRatio float64 - MixMaxCostTpsRatio float64 - LargePauseEveryNRounds, LargePauseMin int - MinBalanceChange, MaxBalanceChange, DeploymentFee uint64 - PaymentAmount, MinFee, MaxFee, FundFee uint64 -} - -type Command struct { - Action string `json:"action"` - Params any `json:"params"` - comment string -} - -type GeneratedRound struct { - Commands []Command - FundCommands []lib.FundParams -} - -func fund(p lib.FundParams) Command { - return Command{Action: lib.FundAction{}.Name(), Params: p} -} - -func rotate(p lib.RotateParams) Command { - return Command{Action: lib.RotateAction{}.Name(), Params: p} -} - -func loadKeys(p lib.KeyloaderParams) Command { - return Command{Action: lib.KeyloaderAction{}.Name(), Params: p} -} - -func discovery(p lib.DiscoveryParams) Command { - return Command{Action: lib.DiscoveryAction{}.Name(), Params: p} -} - -type SampleRefParams struct { - Group lib.ComplexValue `json:"group"` - Ratios []float64 `json:"ratios"` -} - -func sample(groupRef int, groupName string, ratios []float64) Command { - group := lib.LocalComplexValue(groupRef, groupName) - group.OnEmpty = emptyArrayRawMessage - return Command{Action: lib.SampleAction{}.Name(), Params: SampleRefParams{ - Group: group, - Ratios: ratios, - }} -} - -type ZkappRefParams struct { - lib.ZkappSubParams - FeePayers lib.ComplexValue `json:"feePayers"` - Nodes lib.ComplexValue `json:"nodes"` -} - -func zkapps(feePayersRef int, nodesRef int, nodesName string, params lib.ZkappSubParams) Command { - cmd := Command{Action: lib.ZkappCommandsAction{}.Name(), Params: ZkappRefParams{ - ZkappSubParams: params, - FeePayers: lib.LocalComplexValue(feePayersRef, "key"), - Nodes: lib.LocalComplexValue(nodesRef, nodesName), - }} - maxCostStr := "" - if params.MaxCost { - maxCostStr = "max-cost " - } - comment := fmt.Sprintf("Scheduling %d %szkapp transactions to be sent over period of %d minutes (%.2f txs/min)", - int(params.Tps*float64(params.DurationMin)*60), maxCostStr, params.DurationMin, params.Tps*60, - ) - return withComment(comment, cmd) -} - -type PaymentRefParams struct { - lib.PaymentSubParams - FeePayers lib.ComplexValue `json:"feePayers"` - Nodes lib.ComplexValue `json:"nodes"` -} - -func payments(feePayersRef int, nodesRef int, nodesName string, params lib.PaymentSubParams) Command { - cmd := Command{Action: lib.PaymentsAction{}.Name(), Params: PaymentRefParams{ - PaymentSubParams: params, - FeePayers: lib.LocalComplexValue(feePayersRef, "key"), - Nodes: lib.LocalComplexValue(nodesRef, nodesName), - }} - comment := fmt.Sprintf("Scheduling %d payments to be sent over period of %d minutes (%.2f txs/min)", - int(params.Tps*float64(params.DurationMin)*60), params.DurationMin, params.Tps*60, - ) - return withComment(comment, cmd) -} - -func waitMin(min int) Command { - return Command{Action: lib.WaitAction{}.Name(), Params: lib.WaitParams{ - Minutes: min, - }} -} - -func wait(sec int) Command { - return Command{Action: lib.WaitAction{}.Name(), Params: lib.WaitParams{ - Seconds: sec, - }} -} - -type RestartRefParams struct { - Nodes lib.ComplexValue `json:"nodes"` - Clean bool `json:"clean,omitempty"` -} - -func stopDaemon(useRestartScript bool, nodesRef int, nodesName string, clean bool) Command { - var name string - if useRestartScript { - name = lib.RestartAction{}.Name() - } else { - name = lib.StopDaemonAction{}.Name() - } - return Command{Action: name, Params: RestartRefParams{ - Nodes: lib.LocalComplexValue(nodesRef, nodesName), - Clean: clean, - }} -} - -type JoinRefParams struct { - Group1 lib.ComplexValue `json:"group1"` - Group2 lib.ComplexValue `json:"group2"` -} - -func join(g1Ref int, g1Name string, g2Ref int, g2Name string) Command { - return Command{Action: lib.JoinAction{}.Name(), Params: JoinRefParams{ - Group1: lib.LocalComplexValue(g1Ref, g1Name), - Group2: lib.LocalComplexValue(g2Ref, g2Name), - }} -} - -type ExceptRefParams struct { - Group lib.ComplexValue `json:"group"` - Except lib.ComplexValue `json:"except"` -} - -var emptyArrayRawMessage json.RawMessage - -func init() { - emptyArrayRawMessage, _ = json.Marshal([]string{}) -} - -func except(groupRef int, groupName string, exceptRef int, exceptName string) Command { - group := lib.LocalComplexValue(groupRef, groupName) - group.OnEmpty = emptyArrayRawMessage - except := lib.LocalComplexValue(exceptRef, exceptName) - except.OnEmpty = emptyArrayRawMessage - return Command{Action: lib.ExceptAction{}.Name(), Params: ExceptRefParams{ - Group: group, - Except: except, - }} -} - -func withComment(comment string, cmd Command) Command { - cmd.comment = comment - return cmd -} - -func formatDur(min, sec int) string { - sec += min * 60 - min = sec / 60 - sec %= 60 - hour := min / 60 - min %= 60 - day := hour / 24 - hour %= 24 - parts := []string{} - if day > 0 { - parts = append(parts, strconv.Itoa(day), "days") - } - if hour > 0 { - parts = append(parts, strconv.Itoa(hour), "hours") - } - if min > 0 { - parts = append(parts, strconv.Itoa(min), "mins") - } - if sec > 0 { - parts = append(parts, strconv.Itoa(sec), "secs") - } - if len(parts) == 0 { - return "immediately" - } - return strings.Join(parts, " ") -} - -func (p *Params) Generate(round int) GeneratedRound { - zkappsKeysDir := fmt.Sprintf("%s/round-%d/zkapps", p.FundKeyPrefix, round) - paymentsKeysDir := fmt.Sprintf("%s/round-%d/payments", p.FundKeyPrefix, round) - tps := sampleTps(p.BaseTps, p.StressTps) - maxCost := p.MaxCost - zkappRatio := p.ZkappRatio - if p.MixMaxCostTpsRatio > 1e-3 && (round&1) == 1 { - maxCost = true - zkappRatio = 1 - tps *= p.MixMaxCostTpsRatio - } - experimentName := fmt.Sprintf("%s-%d", p.ExperimentName, round) - onlyZkapps := math.Abs(1-zkappRatio) < 1e-3 - onlyPayments := zkappRatio < 1e-3 - zkappTps := tps * zkappRatio - zkappParams := lib.ZkappSubParams{ - ExperimentName: experimentName, - Tps: zkappTps, - MinTps: p.MinTps, - DurationMin: p.RoundDurationMin, - Gap: p.Gap, - MinBalanceChange: p.MinBalanceChange, - MaxBalanceChange: p.MaxBalanceChange, - MinFee: p.MinFee, - MaxFee: p.MaxFee, - DeploymentFee: p.DeploymentFee, - MaxCost: maxCost, - NewAccountRatio: p.NewAccountRatio, - } - if maxCost { - // This can be set to arbitrary value as for max-cost it only - // matters that total zkapps deployed is above 5 - // We need to set it this way to override setting accountQueueSize - // by the orchestrator - zkappParams.ZkappsToDeploy = 20 - zkappParams.NewAccountRatio = 0 - } - paymentParams := lib.PaymentSubParams{ - ExperimentName: experimentName, - Tps: tps - zkappTps, - MinTps: p.MinTps, - DurationMin: p.RoundDurationMin, - MinFee: p.MinFee, - MaxFee: p.MaxFee, - Amount: p.PaymentAmount, - Receiver: p.PaymentReceiver, - } - cmds := []Command{} - roundStartMin := round*(p.RoundDurationMin+p.PauseMin) + round/p.LargePauseEveryNRounds*p.LargePauseMin - if len(p.RotationKeys) > 0 { - var mapping []int - nKeys := len(p.RotationKeys) - if p.RotationPermutation { - mapping = rand.Perm(nKeys) - } else { - mapping = make([]int, nKeys) - for i := range mapping { - mapping[i] = rand.Intn(len(p.RotationKeys)) - } - } - cmds = append(cmds, rotate(lib.RotateParams{ - Pubkeys: p.RotationKeys, - RestServers: p.RotationServers, - Mapping: mapping, - Ratio: p.RotationRatio, - PasswordEnv: p.PasswordEnv, - })) - } - cmds = append(cmds, withComment(fmt.Sprintf("Starting round %d, %s after start", round, formatDur(roundStartMin, 0)), discovery(lib.DiscoveryParams{ - OffsetMin: 15, - NoBlockProducers: p.SendFromNonBpsOnly, - }))) - sendersOutName := "participant" - if 1-p.SenderRatio > 1e-6 { - sendersOutName = "group1" - cmds = append(cmds, sample(-1, "participant", []float64{p.SenderRatio})) - } - if onlyPayments { - cmds = append(cmds, loadKeys(lib.KeyloaderParams{Dir: paymentsKeysDir})) - cmds = append(cmds, payments(-1, -2, sendersOutName, paymentParams)) - } else if onlyZkapps { - cmds = append(cmds, loadKeys(lib.KeyloaderParams{Dir: zkappsKeysDir})) - cmds = append(cmds, zkapps(-1, -2, sendersOutName, zkappParams)) - } else { - cmds = append(cmds, loadKeys(lib.KeyloaderParams{Dir: zkappsKeysDir})) - cmds = append(cmds, loadKeys(lib.KeyloaderParams{Dir: paymentsKeysDir})) - cmds = append(cmds, zkapps(-2, -3, sendersOutName, zkappParams)) - cmds = append(cmds, payments(-2, -4, sendersOutName, paymentParams)) - cmds = append(cmds, join(-1, "participant", -2, "participant")) - } - sendersCmdId := len(cmds) - stopWaits := make([]int, p.StopsPerRound) - for i := 0; i < p.StopsPerRound; i++ { - stopWaits[i] = rand.Intn(60 * p.RoundDurationMin) - } - sort.Ints(stopWaits) - for i := p.StopsPerRound - 1; i > 0; i-- { - stopWaits[i] -= stopWaits[i-1] - } - stopRatio := sampleStopRatio(p.MinStopRatio, p.MaxStopRatio) - elapsed := 0 - for _, waitSec := range stopWaits { - cmds = append(cmds, withComment(fmt.Sprintf("Running round %d, %s after start, waiting for %s", round, formatDur(roundStartMin, elapsed), formatDur(0, waitSec)), wait(waitSec))) - cmds = append(cmds, discovery(lib.DiscoveryParams{ - OffsetMin: 15, - OnlyBlockProducers: p.StopOnlyBps, - })) - exceptRefName := "group" - if onlyPayments || onlyZkapps { - exceptRefName = "participant" - } - cmds = append(cmds, except(-1, "participant", sendersCmdId-len(cmds)-1, exceptRefName)) - stopCleanRatio := p.StopCleanRatio * stopRatio - stopNoCleanRatio := (1 - p.StopCleanRatio) * stopRatio - nodesOrBps := "nodes" - if p.StopOnlyBps { - nodesOrBps = "block producers" - } - if stopCleanRatio > 1e-6 && stopNoCleanRatio > 1e-6 { - cmds = append(cmds, sample(-1, "group", []float64{stopCleanRatio, stopNoCleanRatio})) - comment1 := fmt.Sprintf("Stopping %.1f%% %s with cleaning", stopCleanRatio*100, nodesOrBps) - cmds = append(cmds, withComment(comment1, stopDaemon(p.UseRestartScript, -1, "group1", true))) - comment2 := fmt.Sprintf("Stopping %.1f%% %s without cleaning", stopNoCleanRatio*100, nodesOrBps) - cmds = append(cmds, withComment(comment2, stopDaemon(p.UseRestartScript, -2, "group2", false))) - } else if stopCleanRatio > 1e-6 { - comment := fmt.Sprintf("Stopping %.1f%% %s with cleaning", stopCleanRatio*100, nodesOrBps) - cmds = append(cmds, sample(-1, "group", []float64{stopCleanRatio})) - cmds = append(cmds, withComment(comment, stopDaemon(p.UseRestartScript, -1, "group1", true))) - } else if stopNoCleanRatio > 1e-6 { - comment := fmt.Sprintf("Stopping %.1f%% %s without cleaning", stopNoCleanRatio*100, nodesOrBps) - cmds = append(cmds, sample(-1, "group", []float64{stopNoCleanRatio})) - cmds = append(cmds, withComment(comment, stopDaemon(p.UseRestartScript, -1, "group1", false))) - } - elapsed += waitSec - } - if round < p.Rounds-1 { - comment1 := fmt.Sprintf("Waiting for remainder of round %d, %s after start", round, formatDur(roundStartMin, elapsed)) - cmds = append(cmds, withComment(comment1, wait(p.RoundDurationMin*60-elapsed))) - if p.PauseMin > 0 { - comment2 := fmt.Sprintf("Pause after round %d, %s after start", round, formatDur(roundStartMin+p.RoundDurationMin, 0)) - cmds = append(cmds, withComment(comment2, waitMin(p.PauseMin))) - } - if p.LargePauseMin > 0 && (round+1)%p.LargePauseEveryNRounds == 0 { - comment3 := fmt.Sprintf("Large pause after round %d, %s after start", round, formatDur(roundStartMin+p.RoundDurationMin+p.PauseMin, 0)) - cmds = append(cmds, withComment(comment3, waitMin(p.LargePauseMin))) - } - } - fundCmds := []lib.FundParams{} - if !onlyPayments { - _, _, _, initBalance := lib.ZkappBalanceRequirements(zkappTps, zkappParams) - zkappKeysNum, zkappAmount := lib.ZkappKeygenRequirements(initBalance, zkappParams) - fundCmds = append(fundCmds, - lib.FundParams{ - PasswordEnv: p.PasswordEnv, - Prefix: zkappsKeysDir + "/key", - Amount: zkappAmount, - Fee: p.FundFee, - Num: zkappKeysNum, - }) - } - if !onlyZkapps { - paymentKeysNum, paymentAmount := lib.PaymentKeygenRequirements(p.Gap, paymentParams) - fundCmds = append(fundCmds, - lib.FundParams{ - PasswordEnv: p.PasswordEnv, - // Privkeys: privkeys, - Prefix: paymentsKeysDir + "/key", - Amount: paymentAmount, - Fee: p.FundFee, - Num: paymentKeysNum, - }) - } - return GeneratedRound{ - Commands: cmds, - FundCommands: fundCmds, - } +func fund(p lib.FundParams) lib.GeneratedCommand { + return lib.GeneratedCommand{Action: lib.FundAction{}.Name(), Params: p} } func checkRatio(ratio float64, msg string) { @@ -429,7 +26,7 @@ const mixMaxCostTpsRatioHelp = "when provided, specifies ratio of tps (proportio func main() { var rotateKeys, rotateServers string var mode string - var p Params + var p lib.GenParams flag.Float64Var(&p.BaseTps, "base-tps", 0.3, "Base tps rate for the whole network") flag.Float64Var(&p.StressTps, "stress-tps", 1, "stress tps rate for the whole network") flag.Float64Var(&p.MinTps, "min-tps", 0.01, "minimal tps per node") @@ -466,8 +63,10 @@ func main() { flag.Uint64Var(&p.MinBalanceChange, "min-balance-change", 0, "Min balance change for zkapp account update") flag.Uint64Var(&p.DeploymentFee, "deployment-fee", 1e9, "Zkapp deployment fee") flag.Uint64Var(&p.FundFee, "fund-fee", 1e9, "Funding tx fee") - flag.Uint64Var(&p.MinFee, "min-fee", 1e9, "Min tx fee") - flag.Uint64Var(&p.MaxFee, "max-fee", 2e9, "Max tx fee") + flag.Uint64Var(&p.MinPaymentFee, "min-payment-fee", 1e8, "Min payment fee") + flag.Uint64Var(&p.MaxPaymentFee, "max-payment-fee", 2e8, "Max payment fee") + flag.Uint64Var(&p.MinZkappFee, "min-zkapp-fee", 1e9, "Min zkapp tx fee") + flag.Uint64Var(&p.MaxZkappFee, "max-zkapp-fee", 2e9, "Max zkapp tx fee") flag.Uint64Var(&p.PaymentAmount, "payment-amount", 1e5, "Payment amount") flag.Parse() checkRatio(p.SenderRatio, "wrong sender ratio") @@ -519,13 +118,13 @@ func main() { switch mode { case "stop-ratio-distribution": for i := 0; i < 10000; i++ { - v := sampleStopRatio(p.MinStopRatio, p.MaxStopRatio) + v := lib.SampleStopRatio(p.MinStopRatio, p.MaxStopRatio) fmt.Println(v) } return case "tps-distribution": for i := 0; i < 10000; i++ { - v := sampleTps(p.BaseTps, p.StressTps) + v := lib.SampleTps(p.BaseTps, p.StressTps) fmt.Println(v) } return @@ -546,25 +145,31 @@ func main() { } writeComment("Generated with: " + strings.Join(os.Args, " ")) writeComment("Funding keys for the experiment") - writeCommand := func(cmd Command) { - if cmd.comment != "" { - writeComment(cmd.comment) + writeCommand := func(cmd lib.GeneratedCommand) { + comment := cmd.Comment() + if comment != "" { + writeComment(comment) } if err := encoder.Encode(cmd); err != nil { fmt.Fprintf(os.Stderr, "Error writing command: %v\n", err) os.Exit(3) } } - cmds := []Command{} + cmds := []lib.GeneratedCommand{} fundCmds := []lib.FundParams{} for r := 0; r < p.Rounds; r++ { round := p.Generate(r) cmds = append(cmds, round.Commands...) - fundCmds = append(fundCmds, round.FundCommands...) + if round.PaymentFundCommand != nil { + fundCmds = append(fundCmds, *round.PaymentFundCommand) + } + if round.ZkappFundCommand != nil { + fundCmds = append(fundCmds, *round.ZkappFundCommand) + } } privkeys := p.Privkeys if p.GenerateFundKeys > 0 { - fundKeysDir := fmt.Sprintf("%s/funding", p.FundKeyPrefix) + fundKeysDir := fmt.Sprintf("%s/funding/%s", p.FundKeyPrefix, p.ExperimentName) privkeys = make([]string, p.GenerateFundKeys) privkeyAmounts := make([]uint64, p.GenerateFundKeys) for i := range privkeys { @@ -594,7 +199,7 @@ func main() { Fee: p.FundFee, Num: p.GenerateFundKeys, })) - writeCommand(wait(1)) + writeCommand(lib.GenWait(1)) } privkeysExt := append(privkeys, privkeys...) for i, cmd := range fundCmds { diff --git a/src/app/itn_orchestrator/src/payments.go b/src/app/itn_orchestrator/src/payments.go index 7eba4722ad14..9267d50df983 100644 --- a/src/app/itn_orchestrator/src/payments.go +++ b/src/app/itn_orchestrator/src/payments.go @@ -32,15 +32,15 @@ type ScheduledPaymentsReceipt struct { func PaymentKeygenRequirements(gap int, params PaymentSubParams) (int, uint64) { maxParticipants := int(math.Ceil(params.Tps / params.MinTps)) txCost := params.MaxFee + params.Amount - tpsGap := uint64(math.Round(params.Tps * float64(gap))) + tpsGap := uint64(math.Ceil(params.Tps * float64(gap))) totalTxs := uint64(math.Ceil(float64(params.DurationMin) * 60 * params.Tps)) balance := 3 * txCost * totalTxs keys := maxParticipants + int(tpsGap)*2 return keys, balance } -func schedulePaymentsDo(config Config, params PaymentParams, nodeAddress NodeAddress, batchIx int, tps float64, feePayers []itn_json_types.MinaPrivateKey) (string, error) { - paymentInput := PaymentsDetails{ +func paymentInput(params PaymentSubParams, batchIx int, tps float64) PaymentsDetails { + return PaymentsDetails{ DurationMin: params.DurationMin, Tps: tps, MemoPrefix: fmt.Sprintf("%s-%d", params.ExperimentName, batchIx), @@ -48,8 +48,12 @@ func schedulePaymentsDo(config Config, params PaymentParams, nodeAddress NodeAdd MinFee: params.MinFee, Amount: params.Amount, Receiver: params.Receiver, - Senders: feePayers, } +} + +func schedulePaymentsDo(config Config, params PaymentSubParams, nodeAddress NodeAddress, batchIx int, tps float64, feePayers []itn_json_types.MinaPrivateKey) (string, error) { + paymentInput := paymentInput(params, batchIx, tps) + paymentInput.Senders = feePayers handle, err := SchedulePaymentsGql(config, nodeAddress, paymentInput) if err == nil { config.Log.Infof("scheduled payment batch %d with tps %f for %s: %s", batchIx, tps, nodeAddress, handle) @@ -67,7 +71,7 @@ func SchedulePayments(config Config, params PaymentParams, output func(Scheduled for nodeIx, nodeAddress := range nodes { feePayers := remFeePayers[:feePayersPerNode] var handle string - handle, err = schedulePaymentsDo(config, params, nodeAddress, len(successfulNodes), tps, feePayers) + handle, err = schedulePaymentsDo(config, params.PaymentSubParams, nodeAddress, len(successfulNodes), tps, feePayers) if err != nil { config.Log.Warnf("error scheduling payments for %s: %v", nodeAddress, err) n := len(nodes) - nodeIx - 1 @@ -88,7 +92,7 @@ func SchedulePayments(config Config, params PaymentParams, output func(Scheduled if err != nil { // last schedule payment request didn't work well for _, nodeAddress := range successfulNodes { - handle, err2 := schedulePaymentsDo(config, params, nodeAddress, len(successfulNodes), tps, remFeePayers) + handle, err2 := schedulePaymentsDo(config, params.PaymentSubParams, nodeAddress, len(successfulNodes), tps, remFeePayers) if err2 != nil { config.Log.Warnf("error scheduling second batch of payments for %s: %v", nodeAddress, err2) continue diff --git a/src/app/itn_orchestrator/src/zkapp.go b/src/app/itn_orchestrator/src/zkapp.go index 66510f55554c..d2e67ee2b4c0 100644 --- a/src/app/itn_orchestrator/src/zkapp.go +++ b/src/app/itn_orchestrator/src/zkapp.go @@ -37,15 +37,19 @@ type ScheduledZkappCommandsReceipt struct { Handle string `json:"handle"` } +func tpsGap(tps float64, gapSec int) int { + return int(math.Ceil(tps * float64(gapSec))) +} + func ZkappBalanceRequirements(tps float64, p ZkappSubParams) (int, uint64, uint64, uint64) { - totalZkappsToDeploy := p.Gap * 4 - // new accounts are set to ration multplied by (number of txs minus zkapp deploying txs) + totalZkappsToDeploy, _ := zkappParams(p, tps) + // new accounts are set to ratio multplied by (number of txs minus zkapp deploying txs) newAccounts := int(tps * float64(p.DurationMin*60-totalZkappsToDeploy) * p.NewAccountRatio) // not accounting for Birthday paradox, we multiply by three at a later stage for this reason maxExpectedUsagesPerNewAccount := float64(p.DurationMin*60) / float64(p.Gap) //We multiply by three because by matter of chance some zkapps may generate more new accounts newAccountsPerZkapp := int(float64(newAccounts) / float64(totalZkappsToDeploy) * 3) - balanceDeductedPerUsage := p.MaxBalanceChange * 5 + balanceDeductedPerUsage := p.MaxBalanceChange * 2 // We add 1e9 because of account creation fee minNewZkappBalance := uint64(maxExpectedUsagesPerNewAccount*float64(balanceDeductedPerUsage))*3 + 1e9 maxNewZkappBalance := minNewZkappBalance * 3 @@ -56,25 +60,28 @@ func ZkappBalanceRequirements(tps float64, p ZkappSubParams) (int, uint64, uint6 func ZkappKeygenRequirements(initZkappBalance uint64, params ZkappSubParams) (int, uint64) { maxParticipants := int(math.Ceil(params.Tps / params.MinTps)) - txCost := params.MaxBalanceChange + params.MaxFee - tpsGap := uint64(math.Round(params.Tps * float64(params.Gap))) + minTpsGap := tpsGap(params.MinTps, params.Gap) + // Minimal number of keys allocated per node + minKeysPerNode := minTpsGap * 2 + minTpsZkappsToDeploy, _ := zkappParams(params, params.MinTps) + zkappsToDeployPerKey := uint64(minTpsZkappsToDeploy/minKeysPerNode) + 1 + tpsGap := tpsGap(params.Tps, params.Gap) + keys := maxParticipants + tpsGap*2 + txCost := params.MaxBalanceChange*8 + params.MaxFee totalTxs := uint64(math.Ceil(float64(params.DurationMin) * 60 * params.Tps)) - totalZkappsToDeploy := params.Gap * 4 - balance := 3 * ((initZkappBalance+params.DeploymentFee)*uint64(totalZkappsToDeploy) + txCost*totalTxs) - keys := maxParticipants + int(tpsGap)*2 + balance := uint64(keys)*zkappsToDeployPerKey*(initZkappBalance+params.DeploymentFee)*2 + 3*txCost*totalTxs return keys, balance } -func scheduleZkappCommandsDo(config Config, params ZkappCommandParams, nodeAddress NodeAddress, batchIx int, tps float64, feePayers []itn_json_types.MinaPrivateKey) (string, error) { +func ZkappPaymentsInput(params ZkappSubParams, batchIx int, tps float64) ZkappCommandsDetails { zkappsToDeploy, accountQueueSize := zkappParams(params, tps) - newAccounts, minNewZkappBalance, maxNewZkappBalance, initBalance := ZkappBalanceRequirements(tps, params.ZkappSubParams) - paymentInput := ZkappCommandsDetails{ + newAccounts, minNewZkappBalance, maxNewZkappBalance, initBalance := ZkappBalanceRequirements(tps, params) + return ZkappCommandsDetails{ MemoPrefix: fmt.Sprintf("%s-%d", params.ExperimentName, batchIx), DurationMin: params.DurationMin, Tps: tps, NumZkappsToDeploy: zkappsToDeploy, NumNewAccounts: newAccounts, - FeePayers: feePayers, NoPrecondition: params.NoPrecondition, MinBalanceChange: params.MinBalanceChange, MaxBalanceChange: params.MaxBalanceChange, @@ -86,7 +93,13 @@ func scheduleZkappCommandsDo(config Config, params ZkappCommandParams, nodeAddre DeploymentFee: params.DeploymentFee, AccountQueueSize: accountQueueSize, MaxCost: params.MaxCost, + MaxAccountUpdates: 2, } +} + +func scheduleZkappCommandsDo(config Config, params ZkappCommandParams, nodeAddress NodeAddress, batchIx int, tps float64, feePayers []itn_json_types.MinaPrivateKey) (string, error) { + paymentInput := ZkappPaymentsInput(params.ZkappSubParams, batchIx, tps) + paymentInput.FeePayers = feePayers handle, err := ScheduleZkappCommands(config, nodeAddress, paymentInput) if err == nil { config.Log.Infof("scheduled zkapp batch %d with tps %f for %s: %s", batchIx, tps, nodeAddress, handle) @@ -94,10 +107,10 @@ func scheduleZkappCommandsDo(config Config, params ZkappCommandParams, nodeAddre return handle, err } -func zkappParams(params ZkappCommandParams, tps float64) (zkappsToDeploy int, accountQueueSize int) { +func zkappParams(params ZkappSubParams, tps float64) (zkappsToDeploy int, accountQueueSize int) { zkappsToDeploy, accountQueueSize = params.ZkappsToDeploy, params.AccountQueueSize if zkappsToDeploy == 0 { - tpsGap := int(math.Round(tps * float64(params.Gap))) + tpsGap := tpsGap(tps, params.Gap) zkappsToDeploy, accountQueueSize = tpsGap*4, tpsGap*3 } return diff --git a/src/lib/daemon_rpcs/types.ml b/src/lib/daemon_rpcs/types.ml index 9de70baa6837..be7d76b82b29 100644 --- a/src/lib/daemon_rpcs/types.ml +++ b/src/lib/daemon_rpcs/types.ml @@ -191,6 +191,10 @@ module Status = struct ; transaction_pool_diff_broadcasted : int ; transactions_added_to_pool : int ; transaction_pool_size : int + ; snark_pool_diff_received : int + ; snark_pool_diff_broadcasted : int + ; pending_snark_work : int + ; snark_pool_size : int } [@@deriving to_yojson, bin_io_unversioned, fields] end @@ -423,9 +427,19 @@ module Status = struct let transaction_pool_size = fmt_field "transaction_pool_size" string_of_int in + let snark_pool_diff_received = + fmt_field "snark_pool_diff_received" string_of_int + in + let snark_pool_diff_broadcasted = + fmt_field "snark_pool_diff_broadcasted" string_of_int + in + let pending_snark_work = fmt_field "pending_snark_work" string_of_int in + let snark_pool_size = fmt_field "snark_pool_size" string_of_int in Metrics.Fields.to_list ~block_production_delay ~transaction_pool_diff_received ~transaction_pool_diff_broadcasted ~transactions_added_to_pool ~transaction_pool_size + ~snark_pool_diff_received ~snark_pool_diff_broadcasted + ~pending_snark_work ~snark_pool_size |> List.concat |> List.map ~f:(fun (s, v) -> ("\t" ^ s, v)) |> digest_entries ~title:"" diff --git a/src/lib/mina_commands/mina_commands.ml b/src/lib/mina_commands/mina_commands.ml index ce83a6c6b79f..ba09bc700a89 100644 --- a/src/lib/mina_commands/mina_commands.ml +++ b/src/lib/mina_commands/mina_commands.ml @@ -466,6 +466,13 @@ let get_status ~flag t = ; transactions_added_to_pool = Float.to_int @@ Counter.value Transaction_pool.transactions_added_to_pool + ; snark_pool_diff_received = + Float.to_int @@ Gauge.value Network.snark_pool_diff_received + ; snark_pool_diff_broadcasted = + Float.to_int @@ Gauge.value Network.snark_pool_diff_broadcasted + ; snark_pool_size = Float.to_int @@ Gauge.value Snark_work.snark_pool_size + ; pending_snark_work = + Float.to_int @@ Gauge.value Snark_work.pending_snark_work } in { Daemon_rpcs.Types.Status.num_accounts diff --git a/src/lib/mina_generators/zkapp_command_generators.ml b/src/lib/mina_generators/zkapp_command_generators.ml index e1bd88062acb..65bd1b6407f0 100644 --- a/src/lib/mina_generators/zkapp_command_generators.ml +++ b/src/lib/mina_generators/zkapp_command_generators.ml @@ -1665,21 +1665,23 @@ let mk_account_update ~pk ~vk : Account_update.Simple.t = ; authorization = Control.(dummy_of_tag Proof) } -let mk_fee_payer ~pk ~nonce : Account_update.Fee_payer.t = - { body = - { public_key = pk - ; fee = Currency.Fee.of_mina_string_exn "1.0" - ; valid_until = None - ; nonce - } +let mk_fee_payer ~fee ~pk ~nonce : Account_update.Fee_payer.t = + { body = { public_key = pk; fee; valid_until = None; nonce } ; authorization = Signature.dummy } -let gen_max_cost_zkapp_command_from +let gen_max_cost_zkapp_command_from ?memo ?fee_range ~(fee_payer_keypair : Signature_lib.Keypair.t) ~(account_state_tbl : (Account.t * role) Account_id.Table.t) ~vk - ~(genesis_constants : Genesis_constants.t) = + ~(genesis_constants : Genesis_constants.t) () = let open Quickcheck.Generator.Let_syntax in + let%bind memo = + match memo with + | Some memo -> + return @@ Signed_command_memo.create_from_string_exn memo + | None -> + Signed_command_memo.gen + in let zkapp_accounts = Account_id.Table.data account_state_tbl |> List.filter_map ~f:(fun ((a, role) : Account.t * role) -> @@ -1702,13 +1704,12 @@ let gen_max_cost_zkapp_command_from |> Quickcheck.Generator.list_with_length genesis_constants.max_event_elements in - let%map actions = + let%bind actions = Snark_params.Tick.Field.gen |> Quickcheck.Generator.map ~f:(fun x -> [| x |]) |> Quickcheck.Generator.list_with_length genesis_constants.max_action_elements in - let account_updates = { head with body = { head.body with events; actions } } :: tail in @@ -1719,16 +1720,20 @@ let gen_max_cost_zkapp_command_from let fee_payer_account, _ = Account_id.Table.find_exn account_state_tbl fee_payer_id in + let%map fee = + Option.value_map fee_range + ~default:(return @@ Currency.Fee.of_mina_string_exn "1.0") + ~f:Currency.Fee.(fun (lo, hi) -> gen_incl lo hi) + in let fee_payer = - mk_fee_payer ~pk:fee_payer_pk ~nonce:fee_payer_account.nonce + mk_fee_payer ~fee ~pk:fee_payer_pk ~nonce:fee_payer_account.nonce in Account_id.Table.change account_state_tbl fee_payer_id ~f:(function | None -> None | Some (a, role) -> Some ({ a with nonce = Account.Nonce.succ a.nonce }, role) ) ; - Zkapp_command.of_simple - { fee_payer; account_updates; memo = Signed_command_memo.empty } + Zkapp_command.of_simple { fee_payer; account_updates; memo } let%test_module _ = ( module struct diff --git a/src/lib/mina_generators/zkapp_command_generators.mli b/src/lib/mina_generators/zkapp_command_generators.mli index 87ee4e1b1625..2c89247f28b7 100644 --- a/src/lib/mina_generators/zkapp_command_generators.mli +++ b/src/lib/mina_generators/zkapp_command_generators.mli @@ -97,8 +97,11 @@ val gen_list_of_zkapp_command_from : (** Generate a zkapp command with max cost *) val gen_max_cost_zkapp_command_from : - fee_payer_keypair:Signature_lib.Keypair.t + ?memo:string + -> ?fee_range:Currency.Fee.t * Currency.Fee.t + -> fee_payer_keypair:Signature_lib.Keypair.t -> account_state_tbl:(Account.t * role) Account_id.Table.t -> vk:(Side_loaded_verification_key.t, State_hash.t) With_hash.Stable.V1.t -> genesis_constants:Genesis_constants.t + -> unit -> Zkapp_command.t Quickcheck.Generator.t diff --git a/src/lib/mina_graphql/dune b/src/lib/mina_graphql/dune index 461d958f647e..da27b951d5af 100644 --- a/src/lib/mina_graphql/dune +++ b/src/lib/mina_graphql/dune @@ -60,6 +60,7 @@ graphql_lib block_time currency + merkle_ledger mina_lib mina_commands mina_state diff --git a/src/lib/mina_graphql/itn_zkapps.ml b/src/lib/mina_graphql/itn_zkapps.ml index 8a50d1a109f2..330a799a313c 100644 --- a/src/lib/mina_graphql/itn_zkapps.ml +++ b/src/lib/mina_graphql/itn_zkapps.ml @@ -125,9 +125,10 @@ let rec wait_until_zkapps_deployed ?(deployed = false) ~scheduler_tbl ~mina "Some deployed zkApp accounts weren't found in the best tip ledger, \ trying again" ; let%bind () = + (* Checking three times per block window to avoid unnecessary waiting after the block is created *) Async.after (Time.Span.of_ms - (Float.of_int constraint_constants.block_window_duration_ms) ) + (Float.of_int constraint_constants.block_window_duration_ms /. 3.0) ) in let ledger = Utils.get_ledger_and_breadcrumb mina @@ -223,13 +224,19 @@ let send_zkapps ~fee_payer_array ~constraint_constants ~tm_end ~scheduler_tbl @@ Quickcheck.Generator.generate ( if zkapp_command_details.max_cost then Mina_generators.Zkapp_command_generators - .gen_max_cost_zkapp_command_from ~fee_payer_keypair:fee_payer - ~account_state_tbl ~vk + .gen_max_cost_zkapp_command_from ~memo + ~fee_range: + ( zkapp_command_details.min_fee + , zkapp_command_details.max_fee ) + ~fee_payer_keypair:fee_payer ~account_state_tbl ~vk ~genesis_constants: (Mina_lib.config mina).precomputed_values.genesis_constants + () else Mina_generators.Zkapp_command_generators.gen_zkapp_command_from ~memo + ?max_account_updates: + zkapp_command_details.max_account_updates ~no_account_precondition: zkapp_command_details.no_precondition ~fee_range: diff --git a/src/lib/mina_graphql/mina_graphql.ml b/src/lib/mina_graphql/mina_graphql.ml index 306170fd218d..078f4fa1c83e 100644 --- a/src/lib/mina_graphql/mina_graphql.ml +++ b/src/lib/mina_graphql/mina_graphql.ml @@ -1290,36 +1290,35 @@ module Mutations = struct Public_key.compress public_key ) |> Public_key.Compressed.Hash_set.of_list in - upon - (Itn_zkapps.wait_until_zkapps_deployed ~scheduler_tbl ~mina - ~ledger ~deployment_fee:zkapp_command_details.deployment_fee - ~max_cost:zkapp_command_details.max_cost - ~init_balance:zkapp_command_details.init_balance - ~fee_payer_array ~constraint_constants zkapp_account_keypairs - ~logger ~uuid ~stop_signal ~stop_time:tm_end - ~memo_prefix:zkapp_command_details.memo_prefix ~wait_span ) - (fun result -> - match result with - | None -> - () - | Some ledger -> - let account_state_tbl = - let get_account ids role = - List.map ids ~f:(fun id -> - (id, (Utils.account_of_id id ledger, role)) ) - in - Account_id.Table.of_alist_exn - ( get_account fee_payer_ids `Fee_payer - @ get_account zkapp_account_ids `Ordinary_participant ) + let deploy_zkapps_do () = + Itn_zkapps.wait_until_zkapps_deployed ~scheduler_tbl ~mina ~ledger + ~deployment_fee:zkapp_command_details.deployment_fee + ~max_cost:zkapp_command_details.max_cost + ~init_balance:zkapp_command_details.init_balance + ~fee_payer_array ~constraint_constants zkapp_account_keypairs + ~logger ~uuid ~stop_signal ~stop_time:tm_end + ~memo_prefix:zkapp_command_details.memo_prefix ~wait_span + in + upon (deploy_zkapps_do ()) (function + | None -> + () + | Some ledger -> + let account_state_tbl = + let get_account ids role = + List.map ids ~f:(fun id -> + (id, (Utils.account_of_id id ledger, role)) ) in - let tm_next = Time.add (Time.now ()) wait_span in - don't_wait_for - @@ Itn_zkapps.send_zkapps ~fee_payer_array - ~constraint_constants ~scheduler_tbl ~uuid ~keymap - ~unused_pks ~stop_signal ~mina ~zkapp_command_details - ~wait_span ~logger ~tm_end ~account_state_tbl tm_next - (List.length zkapp_account_keypairs) ) ; - + Account_id.Table.of_alist_exn + ( get_account fee_payer_ids `Fee_payer + @ get_account zkapp_account_ids `Ordinary_participant ) + in + let tm_next = Time.add (Time.now ()) wait_span in + don't_wait_for + @@ Itn_zkapps.send_zkapps ~fee_payer_array + ~constraint_constants ~scheduler_tbl ~uuid ~keymap + ~unused_pks ~stop_signal ~mina ~zkapp_command_details + ~wait_span ~logger ~tm_end ~account_state_tbl tm_next + (List.length zkapp_account_keypairs) ) ; Ok (Uuid.to_string uuid) ) let stop_scheduled_transactions = @@ -2294,15 +2293,6 @@ module Queries = struct |> Runtime_config.to_yojson |> Yojson.Safe.to_basic ) let fork_config = - let rec map_results ~f = function - | [] -> - Result.return [] - | r :: rs -> - let open Result.Let_syntax in - let%bind r' = f r in - let%map rs = map_results ~f rs in - r' :: rs - in field "fork_config" ~doc: "The runtime configuration for a blockchain fork intended to be a \ @@ -2315,59 +2305,69 @@ module Queries = struct `Assoc [ ("error", `String "Daemon is bootstrapping") ] | `Active best_tip -> ( let block = Transition_frontier.Breadcrumb.(block best_tip) in + let blockchain_length = Mina_block.blockchain_length block in let global_slot = - Mina_block.blockchain_length block |> Unsigned.UInt32.to_int + Mina_block.consensus_state block + |> Consensus.Data.Consensus_state.curr_global_slot in - let accounts_or_error = + let staged_ledger = Transition_frontier.Breadcrumb.staged_ledger best_tip |> Staged_ledger.ledger - |> Ledger.foldi ~init:[] ~f:(fun _ accum act -> act :: accum) - |> map_results - ~f:Runtime_config.Json_layout.Accounts.Single.of_account in let protocol_state = Transition_frontier.Breadcrumb.protocol_state best_tip in - match accounts_or_error with + let consensus = + Mina_state.Protocol_state.consensus_state protocol_state + in + let staking_epoch = + Consensus.Proof_of_stake.Data.Consensus_state.staking_epoch_data + consensus + in + let next_epoch = + Consensus.Proof_of_stake.Data.Consensus_state.next_epoch_data + consensus + in + let staking_epoch_seed = + Mina_base.Epoch_seed.to_base58_check + staking_epoch.Mina_base.Epoch_data.Poly.seed + in + let next_epoch_seed = + Mina_base.Epoch_seed.to_base58_check + next_epoch.Mina_base.Epoch_data.Poly.seed + in + let runtime_config = Mina_lib.runtime_config mina in + match + let open Result.Let_syntax in + let%bind staking_ledger = + match Mina_lib.staking_ledger mina with + | None -> + Error "Staking ledger is not initialized." + | Some (Genesis_epoch_ledger l) -> + Ok (Ledger.Any_ledger.cast (module Ledger) l) + | Some (Ledger_db l) -> + Ok (Ledger.Any_ledger.cast (module Ledger.Db) l) + in + let%bind next_epoch_ledger = + match Mina_lib.next_epoch_ledger mina with + | None -> + Error "Next epoch ledger is not initialized." + | Some `Notfinalized -> + Ok None + | Some (`Finalized (Genesis_epoch_ledger l)) -> + Ok (Some (Ledger.Any_ledger.cast (module Ledger) l)) + | Some (`Finalized (Ledger_db l)) -> + Ok (Some (Ledger.Any_ledger.cast (module Ledger.Db) l)) + in + Runtime_config.make_fork_config ~staged_ledger ~global_slot + ~staking_ledger ~staking_epoch_seed ~next_epoch_ledger + ~next_epoch_seed ~blockchain_length + ~protocol_state_hash:protocol_state.previous_state_hash + runtime_config + with | Error e -> `Assoc [ ("error", `String e) ] - | Ok accounts -> - let runtime_config = Mina_lib.runtime_config mina in - let ledger = Option.value_exn runtime_config.ledger in - let previous_length = - let open Option.Let_syntax in - let%bind proof = runtime_config.proof in - let%map fork = proof.fork in - fork.previous_length + global_slot - in - let fork = - Runtime_config.Fork_config. - { previous_state_hash = - State_hash.to_base58_check - protocol_state.previous_state_hash - ; previous_length = - Option.value ~default:global_slot previous_length - ; genesis_slot = global_slot - } - in - let update = - Runtime_config.make - (* add_genesis_winner must be set to false, because this - config effectively creates a continuation of the current - blockchain state and therefore the genesis ledger already - contains the winner of the previous block. No need to - artificially add it. In fact, it wouldn't work at all, - because the new node would try to create this account at - startup, even though it already exists, leading to an error.*) - ~ledger: - { ledger with - base = Accounts accounts - ; add_genesis_winner = Some false - } - ~proof:(Runtime_config.Proof_keys.make ~fork ()) - () - in - let new_config = Runtime_config.combine runtime_config update in + | Ok new_config -> Runtime_config.to_yojson new_config |> Yojson.Safe.to_basic ) ) let thread_graph = diff --git a/src/lib/mina_graphql/types.ml b/src/lib/mina_graphql/types.ml index 63f8e006bcbe..8cb582219ff4 100644 --- a/src/lib/mina_graphql/types.ml +++ b/src/lib/mina_graphql/types.ml @@ -299,7 +299,10 @@ module DaemonStatus = struct ~block_production_delay:nn_int_list ~transaction_pool_diff_received:nn_int ~transaction_pool_diff_broadcasted:nn_int - ~transactions_added_to_pool:nn_int ~transaction_pool_size:nn_int ) + ~transactions_added_to_pool:nn_int ~transaction_pool_size:nn_int + ~snark_pool_diff_received:nn_int + ~snark_pool_diff_broadcasted:nn_int ~pending_snark_work:nn_int + ~snark_pool_size:nn_int ) let t : (_, Daemon_rpcs.Types.Status.t option) typ = obj "DaemonStatus" ~fields:(fun _ -> @@ -2941,6 +2944,7 @@ module Input = struct ; max_cost : bool ; balance_change_range : Mina_generators.Zkapp_command_generators.balance_change_range_t + ; max_account_updates : int option } let arg_typ : ((input, string) result option, input option) arg_typ = @@ -2951,7 +2955,7 @@ module Input = struct min_balance_change max_balance_change min_new_zkapp_balance max_new_zkapp_balance init_balance min_fee max_fee deployment_fee account_queue_size - max_cost -> + max_cost max_account_updates -> Result.return { fee_payers ; num_zkapps_to_deploy @@ -2966,6 +2970,7 @@ module Input = struct ; deployment_fee ; account_queue_size ; max_cost + ; max_account_updates ; balance_change_range = { min_balance_change ; max_balance_change @@ -2981,7 +2986,7 @@ module Input = struct t.balance_change_range.min_new_zkapp_balance t.balance_change_range.max_new_zkapp_balance t.init_balance t.min_fee t.max_fee t.deployment_fee t.account_queue_size - t.max_cost ) + t.max_cost t.max_account_updates ) ~fields: Arg. [ arg "feePayers" @@ -3028,6 +3033,12 @@ module Input = struct ~typ:(non_null int) ; arg "maxCost" ~doc:"Generate max cost zkApp command" ~typ:(non_null bool) + ; arg "maxAccountUpdates" + ~doc: + "Parameter of zkapp generation, each generated zkapp tx \ + will have (2*maxAccountUpdates+2) account updates \ + (including balancing and fee payer)" + ~typ:int ] end diff --git a/src/lib/mina_ledger/sync_ledger.ml b/src/lib/mina_ledger/sync_ledger.ml index a0c1b125dda0..397cdbf34341 100644 --- a/src/lib/mina_ledger/sync_ledger.ml +++ b/src/lib/mina_ledger/sync_ledger.ml @@ -26,7 +26,7 @@ module Mask = Syncable_ledger.Make (struct module Hash = Hash module Root_hash = Root_hash - let account_subtree_height = 3 + let account_subtree_height = 6 end) module Any_ledger = Syncable_ledger.Make (struct @@ -36,7 +36,7 @@ module Any_ledger = Syncable_ledger.Make (struct module Hash = Hash module Root_hash = Root_hash - let account_subtree_height = 3 + let account_subtree_height = 6 end) module Db = Syncable_ledger.Make (struct @@ -46,7 +46,7 @@ module Db = Syncable_ledger.Make (struct module Hash = Hash module Root_hash = Root_hash - let account_subtree_height = 3 + let account_subtree_height = 6 end) module Answer = struct diff --git a/src/lib/network_pool/transaction_pool.ml b/src/lib/network_pool/transaction_pool.ml index 866b1d998ad5..d8bad926418e 100644 --- a/src/lib/network_pool/transaction_pool.ml +++ b/src/lib/network_pool/transaction_pool.ml @@ -1047,7 +1047,7 @@ struct in Deferred.return @@ Result.ok_if_true - (not (already_mem && not is_sender_local)) + ((not already_mem) || is_sender_local) ~error:Recently_seen in let%bind () = diff --git a/src/lib/runtime_config/dune b/src/lib/runtime_config/dune index 04660763290a..f630fd4d669f 100644 --- a/src/lib/runtime_config/dune +++ b/src/lib/runtime_config/dune @@ -16,16 +16,19 @@ data_hash_lib kimchi_backend.pasta kimchi_backend.pasta.basic + merkle_ledger mina_base mina_base.import + mina_ledger mina_numbers - signature_lib snark_params unsigned_extended pickles pickles.backend pickles_types with_hash + signature_lib + staged_ledger ) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_custom_printf ppx_sexp_conv ppx_let ppx_deriving_yojson diff --git a/src/lib/runtime_config/runtime_config.ml b/src/lib/runtime_config/runtime_config.ml index ef42c92554e0..ac57ea62527e 100644 --- a/src/lib/runtime_config/runtime_config.ml +++ b/src/lib/runtime_config/runtime_config.ml @@ -1,9 +1,21 @@ open Core_kernel module Fork_config = struct + (* previous_global_slot here is a global slot since genesis. previous + length is a number of blocks produced since genesis, which might be + smaller than previous_global_slot or equal if a block was produced + in every slot possible, which may or may not be the case. *) type t = { previous_state_hash : string; previous_length : int; genesis_slot : int } [@@deriving yojson, bin_io_unversioned] + + let gen = + let open Quickcheck.Generator.Let_syntax in + let%bind genesis_slot = Int.gen_incl 0 1_000_000 in + let%bind previous_length = Int.gen_incl 0 genesis_slot in + let%map state_hash = Mina_base.State_hash.gen in + let previous_state_hash = Mina_base.State_hash.to_base58_check state_hash in + { previous_state_hash; previous_length; genesis_slot } end let yojson_strip_fields ~keep_fields = function @@ -45,6 +57,15 @@ let of_yojson_generic ~fields of_yojson json = dump_on_error json @@ of_yojson @@ yojson_strip_fields ~keep_fields:fields json +let rec map_results ~f = function + | [] -> + Result.return [] + | r :: rs -> + let open Result.Let_syntax in + let%bind r' = f r in + let%map rs = map_results ~f rs in + r' :: rs + module Json_layout = struct module Accounts = struct module Single = struct @@ -229,89 +250,6 @@ module Json_layout = struct ; permissions = None ; token_symbol = None } - - let of_account (a : Mina_base.Account.t) : (t, string) Result.t = - let open Result.Let_syntax in - let open Signature_lib in - return - { pk = Public_key.Compressed.to_base58_check a.public_key - ; sk = None - ; balance = a.balance - ; delegate = - Option.map a.delegate ~f:(fun pk -> - Public_key.Compressed.to_base58_check pk ) - ; timing = - ( match a.timing with - | Untimed -> - None - | Timed t -> - let open Timed in - Some - { initial_minimum_balance = t.initial_minimum_balance - ; cliff_time = t.cliff_time - ; cliff_amount = t.cliff_amount - ; vesting_period = t.vesting_period - ; vesting_increment = t.vesting_increment - } ) - ; token = Some (Mina_base.Token_id.to_string a.token_id) - ; token_symbol = Some a.token_symbol - ; zkapp = - Option.map a.zkapp ~f:(fun zkapp -> - let open Zkapp_account in - { app_state = Mina_base.Zkapp_state.V.to_list zkapp.app_state - ; verification_key = - Option.map zkapp.verification_key ~f:With_hash.data - ; zkapp_version = zkapp.zkapp_version - ; action_state = - Pickles_types.Vector.Vector_5.to_list zkapp.action_state - ; last_action_slot = - Unsigned.UInt32.to_int - @@ Mina_numbers.Global_slot_since_genesis.to_uint32 - zkapp.last_action_slot - ; proved_state = zkapp.proved_state - ; zkapp_uri = zkapp.zkapp_uri - } ) - ; nonce = a.nonce - ; receipt_chain_hash = - Some - (Mina_base.Receipt.Chain_hash.to_base58_check - a.receipt_chain_hash ) - ; voting_for = - Some (Mina_base.State_hash.to_base58_check a.voting_for) - ; permissions = - Some - Permissions. - { edit_state = - Auth_required.of_account_perm a.permissions.edit_state - ; send = Auth_required.of_account_perm a.permissions.send - ; receive = - Auth_required.of_account_perm a.permissions.receive - ; set_delegate = - Auth_required.of_account_perm a.permissions.set_delegate - ; set_permissions = - Auth_required.of_account_perm - a.permissions.set_permissions - ; set_verification_key = - Auth_required.of_account_perm - a.permissions.set_verification_key - ; set_token_symbol = - Auth_required.of_account_perm - a.permissions.set_token_symbol - ; access = Auth_required.of_account_perm a.permissions.access - ; edit_action_state = - Auth_required.of_account_perm - a.permissions.edit_action_state - ; set_zkapp_uri = - Auth_required.of_account_perm a.permissions.set_zkapp_uri - ; increment_nonce = - Auth_required.of_account_perm - a.permissions.increment_nonce - ; set_timing = - Auth_required.of_account_perm a.permissions.set_timing - ; set_voting_for = - Auth_required.of_account_perm a.permissions.set_voting_for - } - } end type t = Single.t list [@@deriving yojson] @@ -516,9 +454,6 @@ module Accounts = struct } [@@deriving bin_io_unversioned, sexp] - let of_account : Mina_base.Account.t -> (t, string) Result.t = - Json_layout.Accounts.Single.of_account - let to_json_layout : t -> Json_layout.Accounts.Single.t = Fn.id let of_json_layout : Json_layout.Accounts.Single.t -> (t, string) Result.t = @@ -530,6 +465,94 @@ module Accounts = struct Result.bind ~f:of_json_layout (Json_layout.Accounts.Single.of_yojson json) let default = Json_layout.Accounts.Single.default + + let of_account (a : Mina_base.Account.t) : (t, string) Result.t = + let open Result.Let_syntax in + let open Signature_lib in + return + { pk = Public_key.Compressed.to_base58_check a.public_key + ; sk = None + ; balance = a.balance + ; delegate = + Option.map a.delegate ~f:(fun pk -> + Public_key.Compressed.to_base58_check pk ) + ; timing = + ( match a.timing with + | Untimed -> + None + | Timed t -> + let open Timed in + Some + { initial_minimum_balance = t.initial_minimum_balance + ; cliff_time = t.cliff_time + ; cliff_amount = t.cliff_amount + ; vesting_period = t.vesting_period + ; vesting_increment = t.vesting_increment + } ) + ; token = Some (Mina_base.Token_id.to_string a.token_id) + ; token_symbol = Some a.token_symbol + ; zkapp = + Option.map a.zkapp ~f:(fun zkapp -> + let open Zkapp_account in + { app_state = Mina_base.Zkapp_state.V.to_list zkapp.app_state + ; verification_key = + Option.map zkapp.verification_key ~f:With_hash.data + ; zkapp_version = zkapp.zkapp_version + ; action_state = + Pickles_types.Vector.Vector_5.to_list zkapp.action_state + ; last_action_slot = + Unsigned.UInt32.to_int + @@ Mina_numbers.Global_slot_since_genesis.to_uint32 + zkapp.last_action_slot + ; proved_state = zkapp.proved_state + ; zkapp_uri = zkapp.zkapp_uri + } ) + ; nonce = a.nonce + ; receipt_chain_hash = + Some + (Mina_base.Receipt.Chain_hash.to_base58_check + a.receipt_chain_hash ) + ; voting_for = + Some (Mina_base.State_hash.to_base58_check a.voting_for) + ; permissions = + Some + Permissions. + { edit_state = + Auth_required.of_account_perm a.permissions.edit_state + ; send = Auth_required.of_account_perm a.permissions.send + ; receive = + Auth_required.of_account_perm a.permissions.receive + ; set_delegate = + Auth_required.of_account_perm a.permissions.set_delegate + ; set_permissions = + Auth_required.of_account_perm + a.permissions.set_permissions + ; set_verification_key = + Auth_required.of_account_perm + a.permissions.set_verification_key + ; set_token_symbol = + Auth_required.of_account_perm + a.permissions.set_token_symbol + ; access = Auth_required.of_account_perm a.permissions.access + ; edit_action_state = + Auth_required.of_account_perm + a.permissions.edit_action_state + ; set_zkapp_uri = + Auth_required.of_account_perm a.permissions.set_zkapp_uri + ; increment_nonce = + Auth_required.of_account_perm + a.permissions.increment_nonce + ; set_timing = + Auth_required.of_account_perm a.permissions.set_timing + ; set_voting_for = + Auth_required.of_account_perm a.permissions.set_voting_for + } + } + + let gen = + Quickcheck.Generator.map Mina_base.Account.gen ~f:(fun a -> + (* This will never fail with a proper account generator. *) + of_account a |> Result.ok_or_failwith ) end type single = Single.t = @@ -644,6 +667,28 @@ module Ledger = struct let of_yojson json = Result.bind ~f:of_json_layout (Json_layout.Ledger.of_yojson json) + + let gen = + let open Quickcheck in + let open Generator.Let_syntax in + let%bind accounts = Generator.list Accounts.Single.gen in + let num_accounts = List.length accounts in + let balances = + List.mapi accounts ~f:(fun number a -> (number, a.balance)) + in + let%bind hash = + Mina_base.Ledger_hash.(Generator.map ~f:to_base58_check gen) + |> Option.quickcheck_generator + in + let%bind name = String.gen_nonempty in + let%map add_genesis_winner = Bool.quickcheck_generator in + { base = Accounts accounts + ; num_accounts = Some num_accounts + ; balances + ; hash + ; name = Some name + ; add_genesis_winner = Some add_genesis_winner + } end module Proof_keys = struct @@ -685,6 +730,8 @@ module Proof_keys = struct Error "Runtime_config.Proof_keys.Level.of_json_layout: Expected the \ field 'level' to contain a string" + + let gen = Quickcheck.Generator.of_list [ Full; Check; None ] end module Transaction_capacity = struct @@ -718,6 +765,16 @@ module Proof_keys = struct Result.bind ~f:of_json_layout (Json_layout.Proof_keys.Transaction_capacity.of_yojson json) + let gen = + let open Quickcheck in + let log_2_gen = + Generator.map ~f:(fun i -> Log_2 i) @@ Int.gen_incl 0 10 + in + let txns_per_second_x10_gen = + Generator.map ~f:(fun i -> Txns_per_second_x10 i) @@ Int.gen_incl 0 1000 + in + Generator.union [ log_2_gen; txns_per_second_x10_gen ] + let small : t = Log_2 2 let medium : t = Log_2 3 @@ -832,12 +889,43 @@ module Proof_keys = struct opt_fallthrough ~default:t1.account_creation_fee t2.account_creation_fee ; fork = opt_fallthrough ~default:t1.fork t2.fork } + + let gen = + let open Quickcheck.Generator.Let_syntax in + let%bind level = Level.gen in + let%bind sub_windows_per_window = Int.gen_incl 0 1000 in + let%bind ledger_depth = Int.gen_incl 0 64 in + let%bind work_delay = Int.gen_incl 0 1000 in + let%bind block_window_duration_ms = Int.gen_incl 1_000 360_000 in + let%bind transaction_capacity = Transaction_capacity.gen in + let%bind coinbase_amount = + Currency.Amount.(gen_incl zero (of_mina_int_exn 1_000_000_000)) + in + let%bind supercharged_coinbase_factor = Int.gen_incl 0 100 in + let%bind account_creation_fee = + Currency.Fee.(gen_incl one (of_mina_int_exn 10_000_000_000)) + in + let%map fork = + let open Quickcheck.Generator in + union [ map ~f:Option.some Fork_config.gen; return None ] + in + { level = Some level + ; sub_windows_per_window = Some sub_windows_per_window + ; ledger_depth = Some ledger_depth + ; work_delay = Some work_delay + ; block_window_duration_ms = Some block_window_duration_ms + ; transaction_capacity = Some transaction_capacity + ; coinbase_amount = Some coinbase_amount + ; supercharged_coinbase_factor = Some supercharged_coinbase_factor + ; account_creation_fee = Some account_creation_fee + ; fork + } end module Genesis = struct type t = Json_layout.Genesis.t = - { k : int option - ; delta : int option + { k : int option (* the depth of finality constant (in slots) *) + ; delta : int option (* max permissible delay of packets (in slots) *) ; slots_per_epoch : int option ; slots_per_sub_window : int option ; genesis_state_timestamp : string option @@ -865,6 +953,23 @@ module Genesis = struct opt_fallthrough ~default:t1.genesis_state_timestamp t2.genesis_state_timestamp } + + let gen = + let open Quickcheck.Generator.Let_syntax in + let%bind k = Int.gen_incl 0 1000 in + let%bind delta = Int.gen_incl 0 1000 in + let%bind slots_per_epoch = Int.gen_incl 1 1_000_000 in + let%bind slots_per_sub_window = Int.gen_incl 1 1_000 in + let%map genesis_state_timestamp = + Time.(gen_incl epoch (of_string "2050-01-01 00:00:00Z")) + |> Quickcheck.Generator.map ~f:Time.to_string + in + { k = Some k + ; delta = Some delta + ; slots_per_epoch = Some slots_per_epoch + ; slots_per_sub_window = Some slots_per_sub_window + ; genesis_state_timestamp = Some genesis_state_timestamp + } end module Daemon = struct @@ -911,6 +1016,28 @@ module Daemon = struct ; max_action_elements = opt_fallthrough ~default:t1.max_action_elements t2.max_action_elements } + + (* Peer list URL should usually be None. This option is better provided with + a command line argument. Putting it in the config makes the network explicitly + rely on a certain number of nodes, reducing decentralisation. *) + let gen = + let open Quickcheck.Generator.Let_syntax in + let%bind zkapp_proof_update_cost = Float.gen_incl 0. 1000. in + let%bind zkapp_signed_single_update_cost = Float.gen_incl 0. 1000. in + let%bind zkapp_signed_pair_update_cost = Float.gen_incl 0. 1000. in + let%bind zkapp_transaction_cost_limit = Float.gen_incl 0. 1000. in + let%bind max_event_elements = Int.gen_incl 0 1000 in + let%bind max_action_elements = Int.gen_incl 0 1000 in + let%map txpool_max_size = Int.gen_incl 0 1000 in + { txpool_max_size = Some txpool_max_size + ; peer_list_url = None + ; zkapp_proof_update_cost = Some zkapp_proof_update_cost + ; zkapp_signed_single_update_cost = Some zkapp_signed_single_update_cost + ; zkapp_signed_pair_update_cost = Some zkapp_signed_pair_update_cost + ; zkapp_transaction_cost_limit = Some zkapp_transaction_cost_limit + ; max_event_elements = Some max_event_elements + ; max_action_elements = Some max_action_elements + } end module Epoch_data = struct @@ -1036,6 +1163,101 @@ let combine t1 t2 = ; epoch_data = opt_fallthrough ~default:t1.epoch_data t2.epoch_data } +let gen = + let open Quickcheck.Generator.Let_syntax in + let%map daemon = Daemon.gen + and genesis = Genesis.gen + and proof = Proof_keys.gen + and ledger = Ledger.gen in + { daemon = Some daemon + ; genesis = Some genesis + ; proof = Some proof + ; ledger = Some ledger + ; epoch_data = + None (* Leave it empty until we know better what should go there. *) + } + +let ledger_accounts (ledger : Mina_ledger.Ledger.Any_ledger.witness) = + Mina_ledger.Ledger.Any_ledger.M.foldi ~init:[] + ~f:(fun _ accum act -> act :: accum) + ledger + |> map_results ~f:Accounts.Single.of_account + +let ledger_of_accounts accounts = + Ledger. + { base = Accounts accounts + ; num_accounts = Some (List.length accounts) + ; balances = List.mapi accounts ~f:(fun i a -> (i, a.balance)) + ; hash = None + ; name = None + ; add_genesis_winner = Some false + } + +let make_fork_config ~staged_ledger ~global_slot ~blockchain_length + ~protocol_state_hash ~staking_ledger ~staking_epoch_seed ~next_epoch_ledger + ~next_epoch_seed (runtime_config : t) = + let open Result.Let_syntax in + let global_slot = Mina_numbers.Global_slot_since_hard_fork.to_int global_slot in + let blockchain_length = Unsigned.UInt32.to_int blockchain_length in + let%bind accounts = + ledger_accounts + @@ Mina_ledger.Ledger.Any_ledger.cast (module Mina_ledger.Ledger) staged_ledger + in + let ledger = Option.value_exn runtime_config.ledger in + let previous_length = + let open Option.Let_syntax in + let%bind proof = runtime_config.proof in + let%map fork = proof.fork in + fork.previous_length + blockchain_length + in + let fork = + Fork_config. + { previous_state_hash = + Mina_base.State_hash.to_base58_check protocol_state_hash + ; previous_length = + Option.value ~default:blockchain_length previous_length + ; genesis_slot = global_slot + } + in + let%bind staking_ledger_accounts = ledger_accounts staking_ledger in + let%map next_epoch_ledger_accounts = + match next_epoch_ledger with + | None -> + return None + | Some l -> + ledger_accounts l >>| Option.return + in + let epoch_data = + let open Epoch_data in + let open Data in + { staking = + { ledger = ledger_of_accounts staking_ledger_accounts + ; seed = staking_epoch_seed + } + ; next = + Option.map next_epoch_ledger_accounts ~f:(fun accounts -> + { ledger = ledger_of_accounts accounts; seed = next_epoch_seed } ) + } + in + let update = + make + (* add_genesis_winner must be set to false, because this + config effectively creates a continuation of the current + blockchain state and therefore the genesis ledger already + contains the winner of the previous block. No need to + artificially add it. In fact, it wouldn't work at all, + because the new node would try to create this account at + startup, even though it already exists, leading to an error.*) + ~epoch_data + ~ledger: + { ledger with + base = Accounts accounts + ; add_genesis_winner = Some false + } + ~proof:(Proof_keys.make ~fork ()) () + in + combine runtime_config update + module Test_configs = struct let bootstrap = lazy diff --git a/src/lib/transaction/transaction.ml b/src/lib/transaction/transaction.ml index c3136bf7551c..5fa84bc61811 100644 --- a/src/lib/transaction/transaction.ml +++ b/src/lib/transaction/transaction.ml @@ -152,6 +152,14 @@ let check_well_formedness ~genesis_constants (t : t) = Ok () let yojson_summary_of_command = + let is_proof upd = + match Account_update.authorization upd with Proof _ -> true | _ -> false + in + let zkapp_type cmd = + let updates = Zkapp_command.account_updates_list cmd in + Printf.sprintf "zkapp:%d:%d" (List.length updates) + (List.count updates ~f:is_proof) + in let mk_record type_ memo signature = `List [ `String type_ @@ -161,7 +169,7 @@ let yojson_summary_of_command = in function | User_command.Zkapp_command cmd -> - mk_record "zkapp" (Zkapp_command.memo cmd) + mk_record (zkapp_type cmd) (Zkapp_command.memo cmd) ( Zkapp_command.fee_payer_account_update cmd |> Account_update.Fee_payer.authorization ) | Signed_command cmd ->