Merge pull request #16002 from MinaProtocol/dkijania/merge-compatible…

…-04-09-2024

merge compatible to develop

.gitignore

@@ -104,3 +104,7 @@ Pipfile.lock
 # coverage report files
 *.coverage
 _coverage/
+
+# directory removed in this branch, but still existing in berkeley,
+# may appear as an untracked file occasionally
+/src/external

.gitmodules

@@ -1,19 +1,9 @@
 [submodule "src/lib/snarky"]
 	path = src/lib/snarky
 	url = https://github.com/o1-labs/snarky
-[submodule "src/external/ppx_optcomp"]
-	path = src/external/ppx_optcomp
-	url = https://github.com/MinaProtocol/ppx_optcomp
 [submodule "src/lib/crypto/proof-systems"]
 	path = src/lib/crypto/proof-systems
 	url = https://github.com/o1-labs/proof-systems.git
-[submodule "src/external/prometheus"]
-	path = src/external/prometheus
-	url = https://github.com/MinaProtocol/prometheus.git
-    branch = master
-[submodule "src/external/c-reference-signer"]
-	path = src/external/c-reference-signer
-	url = https://github.com/MinaProtocol/c-reference-signer.git
 [submodule "src/lib/crypto/kimchi_bindings/stubs/kimchi-stubs-vendors"]
 	path = src/lib/crypto/kimchi_bindings/stubs/kimchi-stubs-vendors
 	url = https://github.com/MinaProtocol/kimchi-stubs-vendors.git

CODEOWNERS

@@ -11,8 +11,6 @@
 /README.md @MinaProtocol/product-eng-reviewers
 /README-dev.md @MinaProtocol/protocol-eng-reviewers
 
-/src/external/ @bkase @psteckler @mrmr1993
-
 /src/app/archive @MinaProtocol/protocol-eng-reviewers
 /src/app/cli/src/mina.ml @MinaProtocol/protocol-eng-reviewers
 /src/app/cli/src/init @MinaProtocol/protocol-eng-reviewers

README-dev.md

@@ -70,7 +70,8 @@ You can build Mina using Docker. Using Docker works in any dev environment. See
         ```
 
 1. Install [rustup](https://rustup.rs/).
-2. Create your switch with deps `opam switch import --switch mina opam.export`
+2. Add o1-opam-repository with `opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git`
+3. Create your switch with deps `opam switch import --switch mina opam.export`
 
     M1- and M2- operating systems experience issues because Homebrew does not link include files automatically.
 
@@ -98,13 +99,13 @@ You can build Mina using Docker. Using Docker works in any dev environment. See
    - Note:If you get conf-openssl install errors, try running `export PKG_CONFIG_PATH=$(brew --prefix openssl@1.1)/lib/pkgconfig` and try `opam switch import opam.export` again.
    - If prompted, run `opam user-setup install` to enable opam-user-setup support for Merlin.
 
-3. Pin dependencies that override opam versions:
+4. Pin dependencies that override opam versions:
 
     ```sh
     scripts/pin-external-packages.sh
     ```
 
-7. Install the correct version of golang:
+5. Install the correct version of golang:
 
    - `goenv init`
    - To make sure the right `goenv` is used, update your shell env script with:
@@ -118,17 +119,17 @@ You can build Mina using Docker. Using Docker works in any dev environment. See
    - `goenv global 1.18.10`
    - Check that the `go version` returns the right version, otherwise you see the message `compile:version "go1.18.10" does not match go tool version "go1.20.2"`. If so, run `brew remove go` or get the matching version.
 
-9.  Invoke `make build`.
+6.  Invoke `make build`.
 
     If you get errors about `libp2p` and `capnp`, try with `brew install capnp`.
 
-9.  For better IDE support, install the OCaml-LSP language server for OCaml:
+7.  For better IDE support, install the OCaml-LSP language server for OCaml:
 
     ```sh
     opam install ocaml-lsp-server
     ```
 
-10. Set up your IDE. See [Customizing your dev environment for autocomplete/merlin](https://github.com/MinaProtocol/mina/blob/develop/README-dev.md#customizing-your-dev-environment-for-autocompletemerlin).
+8. Set up your IDE. See [Customizing your dev environment for autocomplete/merlin](https://github.com/MinaProtocol/mina/blob/develop/README-dev.md#customizing-your-dev-environment-for-autocompletemerlin).
 
 ### Developer Setup (Linux)
 
@@ -139,6 +140,7 @@ Mina has a variety of opam and system dependencies.
 To get all of the required opam dependencies, run:
 
 ```sh
+opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git
 opam switch import opam.export
 ```
 

buildkite/scripts/setup-database-for-archive-node.sh

@@ -8,5 +8,8 @@ db=$3
 
 
 sudo service postgresql start
-sudo -u postgres psql -c "CREATE USER ${user} WITH SUPERUSER PASSWORD '${password}';"
+sudo -u postgres psql -c "CREATE USER ${user} WITH LOGIN SUPERUSER PASSWORD '${password}';"
+sudo pg_isready
+service postgresql status
 sudo -u postgres createdb -O $user $db
+PGPASSWORD=$password psql -h localhost -p 5434 -U $user -d $db -a -f src/app/archive/create_schema.sql

buildkite/src/Constants/ContainerImages.dhall

@@ -4,11 +4,11 @@
 -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI
 { toolchainBase = "codaprotocol/ci-toolchain-base:v3"
 , minaToolchainBullseye =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:a1f60d69f3657060d6e7289dc770fd7c36fc5a067853019c2f3f6247cb4b6673"
 , minaToolchainBookworm =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:a1f60d69f3657060d6e7289dc770fd7c36fc5a067853019c2f3f6247cb4b6673"
 , minaToolchain =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:a1f60d69f3657060d6e7289dc770fd7c36fc5a067853019c2f3f6247cb4b6673"
 , elixirToolchain = "elixir:1.10-alpine"
 , nodeToolchain = "node:14.13.1-stretch-slim"
 , ubuntu2004 = "ubuntu:20.04"

buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall

@@ -0,0 +1,38 @@
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineTag = ../../Pipeline/Tag.dhall
+
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let DockerImage = ../../Command/DockerImage.dhall
+
+in  Pipeline.build
+      Pipeline.Config::{
+      , spec = JobSpec::{
+        , dirtyWhen =
+          [ S.strictlyStart (S.contains "dockerfiles/stages/1-")
+          , S.strictlyStart (S.contains "dockerfiles/stages/2-")
+          , S.strictlyStart (S.contains "dockerfiles/stages/3-")
+          , S.strictlyStart
+              (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact")
+          , S.strictly (S.contains "opam.export")
+          , S.strictlyEnd (S.contains "rust-toolchain.toml")
+          ]
+        , path = "Release"
+        , name = "MinaToolchainArtifactFocal"
+        , tags = [ PipelineTag.Type.Toolchain ]
+        }
+      , steps =
+        [ let toolchainSpec =
+                DockerImage.ReleaseSpec::{
+                , service = "mina-toolchain"
+                , deb_codename = "focal"
+                , no_cache = True
+                , step_key = "toolchain-focal-docker-image"
+                }
+
+          in  DockerImage.generateStep toolchainSpec
+        ]
+      }

buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall

@@ -50,14 +50,14 @@ in  Pipeline.build
                   [ "POSTGRES_PASSWORD=${password}"
                   , "POSTGRES_USER=${user}"
                   , "POSTGRES_DB=${db}"
+                  , "MINA_TEST_POSTGRES=postgres://${user}:${password}@localhost:5434/${db}"
                   , "GO=/usr/lib/go/bin/go"
                   , "DUNE_INSTRUMENT_WITH=bisect_ppx"
                   , "COVERALLS_TOKEN"
                   ]
                   ( Prelude.Text.concatSep
                       " && "
                       [ "bash buildkite/scripts/setup-database-for-archive-node.sh ${user} ${password} ${db}"
-                      , "PGPASSWORD=${password} psql -h localhost -p 5432 -U ${user} -d ${db} -a -f src/app/archive/create_schema.sql"
                       , WithCargo.withCargo
                           "eval \\\$(opam config env) && dune runtest src/app/archive && buildkite/scripts/upload-partial-coverage-data.sh ${command_key} dev"
                       ]

buildkite/src/Jobs/Test/VersionLint.dhall

@@ -62,5 +62,5 @@ in  Pipeline.build
                 , PipelineTag.Type.Stable
                 ]
               }
-      , steps = [ buildTestCmd "develop" Size.Small dependsOn ]
+      , steps = [ buildTestCmd "compatible" Size.Small dependsOn ]
       }

dockerfiles/stages/2-opam-deps

@@ -25,6 +25,8 @@ ENV OPAMREUSEBUILDDIR false
 # Limit logs for opam install errors to 20 lines per error
 ENV OPAMERRLOGLEN 20
 
+ENV GIT_LFS_SKIP_SMUDGE 1
+
 # --- Shallow clone the Mina repo, only focused on the given MINA_BRANCH
 # git will clone into an empty dir, but this also helps us set the workdir in advance
 RUN git clone \

flake.nix

@@ -21,6 +21,9 @@
   inputs.opam-nix.inputs.nixpkgs.follows = "nixpkgs";
   inputs.opam-nix.inputs.opam-repository.follows = "opam-repository";
 
+  inputs.o1-opam-repository.url = "github:o1-labs/opam-repository";
+  inputs.o1-opam-repository.flake = false;
+
   inputs.opam-repository.url = "github:ocaml/opam-repository";
   inputs.opam-repository.flake = false;