From 5348b23027de279202c0958a1a9fc8e7b46a1ea6 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Oct 2023 10:41:02 +0200 Subject: [PATCH 1/5] Removal of snarky js test in mina as one of step of spitting repos --- buildkite/src/Command/TestExecutive.dhall | 36 ---------- .../Jobs/Test/TestnetIntegrationTests.dhall | 9 +-- src/app/test_executive/snarkyjs.ml | 70 ------------------- src/app/test_executive/test_executive.ml | 1 - 4 files changed, 1 insertion(+), 115 deletions(-) delete mode 100644 src/app/test_executive/snarkyjs.ml diff --git a/buildkite/src/Command/TestExecutive.dhall b/buildkite/src/Command/TestExecutive.dhall index c8541dccd2d..524ea2c628f 100644 --- a/buildkite/src/Command/TestExecutive.dhall +++ b/buildkite/src/Command/TestExecutive.dhall @@ -28,41 +28,5 @@ in key = "integration-test-${testName}", target = Size.Integration, depends_on = dependsOn - }, - - buildJs = \(duneProfile : Text) -> - Command.build - Command.Config::{ - commands = - -- Build js test archive - RunInToolchain.runInToolchainBuster ([] : List Text) "./buildkite/scripts/build-js-tests.sh" - - # - - [ - -- Cache js test archive - Cmd.run "artifact-cache-helper.sh snarkyjs_test.tar.gz --upload" - ], - label = "Build JS integration tests", - key = "build-js-tests", - target = Size.XLarge - }, - - executeWithJs = \(testName : Text) -> \(dependsOn : List Command.TaggedKey.Type) -> - Command.build - Command.Config::{ - commands = - [ - Cmd.run "artifact-cache-helper.sh snarkyjs_test.tar.gz && tar -xzf snarkyjs_test.tar.gz", - - -- Execute test based on BUILD image - Cmd.run "MINA_DEB_CODENAME=bullseye ; source ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/run-test-executive.sh ${testName}" - ], - artifact_paths = [SelectFiles.exactly "." "${testName}.test.log"], - label = "${testName} integration test", - key = "integration-test-${testName}", - target = Size.Integration, - depends_on = dependsOn, - soft_fail = Some (B/SoftFail.Boolean True) } } diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 2e918ff3432..90fbea13b3f 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -11,11 +11,6 @@ let dependsOn = [ { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" }, { name = "MinaArtifactBullseye", key = "archive-bullseye-docker-image" } ] -let dependsOnJs = [ - { name = "TestnetIntegrationTests", key = "build-js-tests" }, - { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" }, - { name = "MinaArtifactBullseye", key = "archive-bullseye-docker-image" } -] in Pipeline.build Pipeline.Config::{ spec = @@ -34,7 +29,6 @@ in Pipeline.build Pipeline.Config::{ mode = PipelineMode.Type.Stable }, steps = [ - TestExecutive.buildJs "integration_tests", TestExecutive.execute "peers-reliability" dependsOn, TestExecutive.execute "chain-reliability" dependsOn, TestExecutive.execute "payment" dependsOn, @@ -45,7 +39,6 @@ in Pipeline.build Pipeline.Config::{ TestExecutive.execute "zkapps" dependsOn, TestExecutive.execute "zkapps-timing" dependsOn, TestExecutive.execute "zkapps-nonce" dependsOn, - TestExecutive.execute "verification-key" dependsOn, - TestExecutive.executeWithJs "snarkyjs" dependsOnJs + TestExecutive.execute "verification-key" dependsOn ] } diff --git a/src/app/test_executive/snarkyjs.ml b/src/app/test_executive/snarkyjs.ml deleted file mode 100644 index 70730d4c16d..00000000000 --- a/src/app/test_executive/snarkyjs.ml +++ /dev/null @@ -1,70 +0,0 @@ -open Core -open Async -open Integration_test_lib - -module Make (Inputs : Intf.Test.Inputs_intf) = struct - open Inputs - open Engine - open Dsl - - open Test_common.Make (Inputs) - - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t - - let config = - let open Test_config in - { default with - requires_graphql = true - ; genesis_ledger = - [ { account_name = "node-key"; balance = "8000000"; timing = Untimed } - ; { account_name = "extra-key"; balance = "10"; timing = Untimed } - ] - ; block_producers = [ { node_name = "node"; account_name = "node-key" } ] - } - - let check_and_print_stout_stderr ~logger process = - let open Deferred.Let_syntax in - let%map output = Async_unix.Process.collect_output_and_wait process in - let stdout = String.strip output.stdout in - [%log info] "Stdout: $stdout" ~metadata:[ ("stdout", `String stdout) ] ; - if not (String.is_empty output.stderr) then ( - [%log error] - "An error occured during the execution of the test script: \"%s\"" - output.stderr ; - Malleable_error.hard_error_string output.stderr ) - else Malleable_error.ok_unit - - let run network t = - let open Malleable_error.Let_syntax in - let logger = Logger.create () in - let node = - Core.String.Map.find_exn (Network.block_producers network) "node" - in - let%bind fee_payer_key = priv_key_of_node node in - let graphql_uri = Network.Node.get_ingress_uri node |> Uri.to_string in - - let%bind () = - [%log info] "Waiting for nodes to be initialized" ; - let%bind () = wait_for t (Wait_condition.node_to_initialize node) in - [%log info] "Running test script" ; - let%bind.Deferred result = - let%bind.Deferred process = - Async_unix.Process.create_exn - ~prog:"./src/lib/snarkyjs/tests/integration/node" - ~args: - [ "src/lib/snarkyjs/tests/integration/simple-zkapp.js" - ; Signature_lib.Private_key.to_base58_check fee_payer_key - ; graphql_uri - ] - () - in - check_and_print_stout_stderr ~logger process - in - section "Waiting for script to run and complete" result - in - return () -end diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 49e43c123f2..583317827b5 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -64,7 +64,6 @@ let tests : test list = , (module Verification_key_update.Make : Intf.Test.Functor_intf) ) ; ( "block-prod-prio" , (module Block_production_priority.Make : Intf.Test.Functor_intf) ) - ; ("snarkyjs", (module Snarkyjs.Make : Intf.Test.Functor_intf)) ; ("block-reward", (module Block_reward_test.Make : Intf.Test.Functor_intf)) ; ("hard-fork", (module Hard_fork.Make : Intf.Test.Functor_intf)) ] From 80a6fd3f729aac8a3056e896269c8717edc9d506 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 27 Nov 2023 14:54:22 +0100 Subject: [PATCH 2/5] fix version linter failure --- buildkite/scripts/version-linter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh index 960f4c09d2d..47c438f8a5e 100755 --- a/buildkite/scripts/version-linter.sh +++ b/buildkite/scripts/version-linter.sh @@ -17,7 +17,7 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -pip3 install sexpdata +pip3 install sexpdata==1.0.0 base_branch=origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH} pr_branch=origin/${BUILDKITE_BRANCH} From 3e56cf8dea0515348a754c5c51d65e568be33b3c Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 27 Nov 2023 15:11:55 +0100 Subject: [PATCH 3/5] Fix dirty when for version linter --- buildkite/src/Constants/DebianVersions.dhall | 7 ++++++- buildkite/src/Jobs/Test/VersionLint.dhall | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index 0ef0ee92cf7..d55ecd594c9 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -67,7 +67,12 @@ let minimalDirtyWhen = [ S.exactly "scripts/rebuild-deb" "sh", S.exactly "scripts/release-docker" "sh", S.exactly "buildkite/scripts/build-artifact" "sh", - S.exactly "buildkite/scripts/run-snark-transaction-profiler" "sh" + -- Snark profiler dirtyWhen + S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall", + S.exactly "buildkite/scripts/run-snark-transaction-profiler" "sh", + S.exactly "scripts/snark_transaction_profiler" "py" + S.exactly "buildkite/scripts/version-linter" "sh" + S.exactly "scripts/version-linter" "py" ] -- The default debian version (Bullseye) is used in all downstream CI jobs diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index a0aa132649a..c54ff48588c 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -49,7 +49,7 @@ Pipeline.build let lintDirtyWhen = [ S.strictlyStart (S.contains "src"), S.exactly "buildkite/src/Jobs/Test/VersionLint" "dhall", - S.exactly "buildkite/scripts/version_linter" "sh" + S.exactly "buildkite/scripts/version-linter" "sh" ] in From af2ead1eb8d3c90d825302d69cd616112d2560ff Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 27 Nov 2023 15:14:27 +0100 Subject: [PATCH 4/5] fix debian version --- buildkite/src/Constants/DebianVersions.dhall | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index d55ecd594c9..cf7b1a0be2a 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -70,8 +70,8 @@ let minimalDirtyWhen = [ -- Snark profiler dirtyWhen S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall", S.exactly "buildkite/scripts/run-snark-transaction-profiler" "sh", - S.exactly "scripts/snark_transaction_profiler" "py" - S.exactly "buildkite/scripts/version-linter" "sh" + S.exactly "scripts/snark_transaction_profiler" "py", + S.exactly "buildkite/scripts/version-linter" "sh", S.exactly "scripts/version-linter" "py" ] From 213742dfd8edf5ec0905dd933486fe5ac535a13b Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 27 Nov 2023 15:21:50 +0100 Subject: [PATCH 5/5] fetch before checkouting branch for nix --- buildkite/scripts/test-nix.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 buildkite/scripts/test-nix.sh diff --git a/buildkite/scripts/test-nix.sh b/buildkite/scripts/test-nix.sh new file mode 100755 index 00000000000..e9f9e488710 --- /dev/null +++ b/buildkite/scripts/test-nix.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -euo pipefail + +set -eou pipefail +set +x + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 branch" + exit 1 +fi + + +mkdir -p "${XDG_CONFIG_HOME-${HOME}/.config}/nix" +echo 'experimental-features = nix-command flakes' > "${XDG_CONFIG_HOME-${HOME}/.config}/nix/nix.conf" + +git config --global --add safe.directory /workdir + +git fetch +# Nix has issue when performing operations on detached head +# On Ci machine it spit out issues like: +# fatal: reference is not a tree: .... +# error: +# … while fetching the input 'git+file:///workdir' +# +# error: program 'git' failed with exit code 128 +# That is why we checkout branch explicitly +git checkout $1 + +./nix/pin.sh + +nix build mina --accept-flake-config \ No newline at end of file