diff --git a/nix/workbench/backend/nomad.nix b/nix/workbench/backend/nomad.nix index 744dd9a58a7..a0cfcaea040 100644 --- a/nix/workbench/backend/nomad.nix +++ b/nix/workbench/backend/nomad.nix @@ -205,6 +205,13 @@ let oneTracerPerNode = true; withSsh = true; }; + # AWS S3 bucket that will be used to deploy the genesis files. + s3 = rec { + bucket = "cardano-perf-deploy"; + region = "eu-central-1"; + host = "s3.${region}.amazonaws.com"; + uri = "https://${bucket}.${host}"; + }; }; } ; diff --git a/nix/workbench/backend/nomad/cloud.nix b/nix/workbench/backend/nomad/cloud.nix index b0301bb3924..7b949dea1c8 100644 --- a/nix/workbench/backend/nomad/cloud.nix +++ b/nix/workbench/backend/nomad/cloud.nix @@ -9,7 +9,7 @@ let # The exec task driver can run in a cloud environment using SRE's Nomad # servers with the "nix_installable" patch and Amazon S3 to distribute the - # genesis files. All credentials are obtained using Vault. + # genesis files (Buckets needs write permissions for the deployer machine). name = "nomadcloud"; # Unlike the supervisor backend `useCabalRun` is always false here. @@ -24,11 +24,8 @@ let then builtins.abort "Nomad backends only available for x86_64-linux" else (import ./patch.nix {}) ) - # Amazon S3 HTTP to upload/download the genesis tar file. pkgs.awscli - # Use to obtain the AWS credentials. - pkgs.vault-bin ] ; diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh index 20863635f2b..35a98bc1899 100644 --- a/nix/workbench/backend/nomad/cloud.sh +++ b/nix/workbench/backend/nomad/cloud.sh @@ -252,18 +252,8 @@ setenv-defaults-nomadcloud() { ######### # AWS_* # ######### - # Check all the AWS S3 envars needed for the HTTP PUT request - # Using same names as the AWS CLI - # https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html - if test -z "${AWS_ACCESS_KEY_ID:-}" || test -z "${AWS_SECRET_ACCESS_KEY:-}" - then - msg $(blue "INFO: Amazon S3 \"AWS_ACCESS_KEY_ID\" or \"AWS_SECRET_ACCESS_KEY\" envar is not set") - msg $(yellow "WARNING: Fetching \"AWS_ACCESS_KEY_ID\" and \"AWS_SECRET_ACCESS_KEY\" from SRE provided Vault for \"Performance and Tracing\"") - local aws_credentials - aws_credentials="$(wb_nomad vault world aws-s3-credentials)" - export AWS_ACCESS_KEY_ID=$(echo "${aws_credentials}" | jq -r .data.access_key) - export AWS_SECRET_ACCESS_KEY=$(echo "${aws_credentials}" | jq -r .data.secret_key) - fi + local s3_bucket_name="$(jq -r .nomadJob.cloud.s3.bucket "${backend_dir}"/container-specs.json)" + msg $(blue "INFO: Using Amazon S3 \"${s3_bucket_name}\" as bucket") } # Sub-backend specific allocs and calls `backend_nomad`'s `allocate-run`. @@ -858,11 +848,9 @@ deploy-genesis-nomadcloud() { --directory="${dir}"/genesis --files-from=- # Upload genesis tar file - local s3_region="eu-central-1" - local s3_host="s3.${s3_region}.amazonaws.com"; - local s3_bucket_name="iog-cardano-perf"; - local s3_access_key="${AWS_ACCESS_KEY_ID}"; - local s3_access_key_secret="${AWS_SECRET_ACCESS_KEY}" + local s3_bucket_name="$(jq -r .nomadJob.cloud.s3.bucket "${dir}"/container-specs.json)" + local s3_region="$(jq -r .nomadJob.cloud.s3.region "${dir}"/container-specs.json)" + local s3_uri="$(jq -r .nomadJob.cloud.s3.uri "${dir}"/container-specs.json)" local s3_storage_class="STANDARD" local return_code=0 msg "$(blue Uploading) $(yellow "\"${genesis_file_name}\"") to $(yellow "\"s3://${s3_bucket_name}/\"") ..." @@ -887,8 +875,7 @@ deploy-genesis-nomadcloud() { fi # Generic download from every node. - local uri="https://${s3_bucket_name}.${s3_host}/${genesis_file_name}" - if ! backend_nomad deploy-genesis-wget "${dir}" "${uri}" + if ! backend_nomad deploy-genesis-wget "${dir}" "${s3_uri}"/"${genesis_file_name}" then # File kept for debugging! msg "$(red "FATAL: deploy-genesis-wget \"${dir}\" \"${uri}\"")" diff --git a/nix/workbench/nomad.sh b/nix/workbench/nomad.sh index 2734cf9e405..3aadb68af9a 100644 --- a/nix/workbench/nomad.sh +++ b/nix/workbench/nomad.sh @@ -1,26 +1,8 @@ usage_nomad() { usage "nomad" "nomad helpers" < Settings -> Developer Settings -> - Tokens (Classic) -> Generate New Token (Classic) - and create a new token with only the "read:org" permission. - $(helpcmd vault \(ci\|world\) nomad-token) - Gets the corresponding Nomad token from the chosen Vault - (WARNING: shows secrets!!!). - $(helpcmd vault ci pg-user) - Gets SRE's Postgres server username from Vault - (WARNING: shows secrets!!!). - $(helpcmd vault ci pg-pass) - Gets SRE's Postgres server password from Vault - (WARNING: shows secrets!!!). - $(helpcmd vault world aws-s3-credentials) - Gets Cardano World's AWS S3 crdentials from Vault in JSON - (WARNING: shows secrets!!!). - $(helpcmd clients ready) Creates a JSON array with "id", "name", "datacenter" and "ip" of all SRE's Nomad client nodes available @@ -110,18 +92,13 @@ wb_nomad() { ### dir-path ) ################################################################# ################################################################################ dir-path ) - local usage="USAGE: wb nomad ${op} vault|server|client|webfs" + local usage="USAGE: wb nomad ${op} server|client|webfs" # Calling `wb nomad dir-path XXX` inside a Nix derivation will fail: # "mkdir: cannot create directory '/homeless-shelter': Permission denied" local nomad_cache_dir="$(envjqr 'cacheDir')"/nomad mkdir -p "${nomad_cache_dir}" local subop=${1:?$usage}; shift case "${subop}" in - vault ) - local vault_dir="${nomad_cache_dir}"/vault - mkdir -p "${vault_dir}" - echo "${vault_dir}" - ;; server ) local nomad_servers_dir="${nomad_cache_dir}"/server mkdir -p "${nomad_servers_dir}" @@ -148,184 +125,6 @@ wb_nomad() { esac ;; ################################################################################ -### vault ) #################################################################### -################################################################################ - vault ) - local usage="USAGE: wb nomad ${op} world|ci" - local vault_dir="$(wb_nomad dir-path vault)" - local entity=${1:?$usage}; shift - case "${entity}" in -####### vault -> ci )########################################################### - ci ) - mkdir -p "${vault_dir}"/ci - local login_file="${vault_dir}"/ci/login.json - local vault_address="https://vault.ci.iog.io" - local action=${1:?$usage}; shift - case "${action}" in - login ) - msg "First create and copy your GitHub token by doing: " - msg "Your profile -> Settings -> Developer Settings -> Tokens (Classic) -> Generate New Token (Classic)" - msg "and create a new token with only the \"read:org\" permission." - read -p "Hit enter to continue ..." - vault login \ - -address="${vault_address}" \ - -method=github -path=github-employees \ - -no-store -format=json \ - > "${login_file}" - ;; - enabled ) - if test -f "${login_file}" - then - # Fetch token info from vault - local client_token - client_token=$(jq -r '.auth.client_token' "${login_file}") - local token_lookup_response - if token_lookup_response=$(VAULT_TOKEN="${client_token}" vault token lookup -address="${vault_address}" -namespace=perf -format=json) - then - local expire_time - expire_time=$(echo "${token_lookup_response}" | jq -r .data.expire_time) - # Compare expire date with the actual date minus one day. - # This avoids a token expiring while a profile is running. - if test "$(date -u -d "${expire_time}" "+%s")" -ge "$(($(date -u "+%s") - 86400))" - then - true - else - rm "${login_file}" - false - fi - else - fatal "Are you logged in to Vault? Call 'wb nomad vault ${entity} login' with your IOHK GitHub token (classic)" - fi - else - false - fi - ;; - nomad-token ) - if ! wb_nomad vault "${entity}" enabled - then - wb_nomad vault "${entity}" login - fi - local client_token - client_token=$(jq -r '.auth.client_token' "${login_file}") - local nomad_token_json - if nomad_token_json=$(VAULT_TOKEN="${client_token}" vault read -address="${vault_address}" -non-interactive -format=json nomad/creds/perf) - then - echo "${nomad_token_json}" | jq -r .data.secret_id - else - fatal "Unable to fetch Nomad token from Vault" - fi - ;; - pg-user ) - if ! wb_nomad vault "${entity}" enabled - then - wb_nomad vault "${entity}" login - fi - local client_token - client_token=$(jq -r '.auth.client_token' "${login_file}") - VAULT_TOKEN="${client_token}" vault kv get \ - --address="${vault_address}" \ - -non-interactive \ - -format=json \ - kv/postgrest/perf \ - | jq -r .data.data.postgrestDbUser - ;; - pg-pass ) - if ! wb_nomad vault "${entity}" enabled - then - wb_nomad vault "${entity}" login - fi - local client_token - client_token=$(jq -r '.auth.client_token' "${login_file}") - VAULT_TOKEN="${client_token}" vault kv get \ - --address="${vault_address}" \ - -non-interactive \ - -format=json \ - kv/postgrest/perf \ - | jq -r .data.data.postgrestDbPass - ;; -####### vault -> ci -> * )###################################################### - * ) - usage_nomad - ;; - esac - ;; -####### vault -> world )######################################################## - world ) - mkdir -p "${vault_dir}"/world - local login_file="${vault_dir}"/world/login.json - local vault_address="https://vault.world.dev.cardano.org" - local action=${1:?$usage}; shift - case "${action}" in - login ) - msg "First create and copy your GitHub token by doing: " - msg "Your profile -> Settings -> Developer Settings -> Tokens (Classic) -> Generate New Token (Classic)" - msg "and create a new token with only the \"read:org\" permission." - read -p "Hit enter to continue ..." - vault login \ - -address="${vault_address}" \ - -method=github -path=github-employees \ - -no-store -format=json \ - > "${login_file}" - ;; - enabled ) - if test -f "${login_file}" - then - local client_token - client_token=$(jq -r '.auth.client_token' "${login_file}") - local token_lookup_response - if token_lookup_response=$(VAULT_TOKEN="${client_token}" vault token lookup -address="${vault_address}" -namespace=perf -format=json) - then - # TODO: I need to check the expiration time? - # echo "${token_lookup_response}" | jq -r .data.expire_time - # 2023-02-19T13:07:26.125306646Z - true - else - fatal "Are you logged in to Vault? Call 'wb nomad vault ${entity} login' with your IOHK GitHub token (classic)" - fi - else - false - fi - ;; - nomad-token ) - if ! wb_nomad vault "${entity}" enabled - then - wb_nomad vault "${entity}" login - fi - local client_token - client_token=$(jq -r '.auth.client_token' "${login_file}") - local nomad_token_json - if nomad_token_json=$(VAULT_TOKEN="${client_token}" vault read -address="${vault_address}" -non-interactive -format=json nomad/creds/perf) - then - echo "${nomad_token_json}" | jq -r .data.secret_id - else - fatal "Unable to fetch Nomad token from Vault" - fi - ;; - aws-s3-credentials ) - if ! wb_nomad vault "${entity}" enabled - then - wb_nomad vault "${entity}" login - fi - local client_token - client_token=$(jq -r '.auth.client_token' "${login_file}") - VAULT_TOKEN="${client_token}" vault read \ - --address="${vault_address}" \ - -format=json \ - aws/creds/perf - ;; -####### vault -> world -> * )################################################### - * ) - usage_nomad - ;; - esac - ;; -####### vault -> * )############################################################ - * ) - usage_nomad - ;; - esac - ;; -################################################################################ ### ssh ) ###################################################################### ################################################################################ ssh ) @@ -1381,7 +1180,7 @@ EOF fi done # Nuke the Nomad Agents' .cache dir - # Keep top level Nomad cache dir because it includes Vault's dirs. + # Keep top level Nomad cache dir because it includes webfs and ssh dirs. rm -rf "${nomad_clients_dir}" >/dev/null 2>&1 rm -rf "${nomad_servers_dir}" >/dev/null 2>&1 # Bye HTTP server