From f642cca869cb13bcfaf39c5b4ab72906d4289783 Mon Sep 17 00:00:00 2001 From: Thiago Esteves Date: Thu, 12 Sep 2024 09:52:54 -0300 Subject: [PATCH 1/3] Adding support to AWS and GCP based on the configuration passed --- config/runtime.exs | 23 +++++++ .../modules/standard-account/cloud-config.tpl | 4 +- .../modules/standard-account/cloud-config.tpl | 4 +- lib/config_provider/secrets/adapter.ex | 7 ++ lib/config_provider/secrets/aws.ex | 55 ++++++++++++++++ lib/config_provider/secrets/gcp.ex | 65 +++++++++++++++++++ .../manager.ex} | 52 ++++----------- mix.exs | 3 +- mix.lock | 2 + 9 files changed, 174 insertions(+), 41 deletions(-) create mode 100644 lib/config_provider/secrets/adapter.ex create mode 100644 lib/config_provider/secrets/aws.ex create mode 100644 lib/config_provider/secrets/gcp.ex rename lib/config_provider/{aws_secrets_manager.ex => secrets/manager.ex} (60%) diff --git a/config/runtime.exs b/config/runtime.exs index b6550ba..6f44abb 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -45,6 +45,29 @@ if config_env() == :prod do port: port ] + secrets_adapter = System.fetch_env!("CALORI_SECRETS_ADAPTER") + secrets_path = System.fetch_env!("CALORI_SECRETS_PATH") + + case secrets_adapter do + "gcp" -> + config :deployex, Calori.ConfigProvider.Secrets.Manager, + adapter: Calori.ConfigProvider.Secrets.Gcp, + path: secrets_path + + "aws" -> + config :deployex, Calori.ConfigProvider.Secrets.Manager, + adapter: Calori.ConfigProvider.Secrets.Aws, + path: secrets_path + + adapter -> + raise "Secret #{adapter} not supported" + end + + if secrets_adapter == "gcp" do + config :goth, + file_credentials: System.fetch_env!("GOOGLE_APPLICATION_CREDENTIALS") |> File.read!() + end + # ## SSL Support # # To get SSL working, you will need to add the `https` key diff --git a/devops/aws/terraform/modules/standard-account/cloud-config.tpl b/devops/aws/terraform/modules/standard-account/cloud-config.tpl index d2e35fb..fbfd057 100644 --- a/devops/aws/terraform/modules/standard-account/cloud-config.tpl +++ b/devops/aws/terraform/modules/standard-account/cloud-config.tpl @@ -50,7 +50,9 @@ write_files: "CALORI_PHX_HOST": "${hostname}", "CALORI_PHX_SERVER": true, "CALORI_CLOUD_ENVIRONMENT": "${account_name}", - "CALORI_OTP_TLS_CERT_PATH": "/usr/local/share/ca-certificates" + "CALORI_OTP_TLS_CERT_PATH": "/usr/local/share/ca-certificates", + "CALORI_SECRETS_ADAPTER": "aws", + "CALORI_SECRETS_PATH": "calori-${account_name}-secrets", } } - path: /home/ubuntu/config.json diff --git a/devops/gcp/terraform/modules/standard-account/cloud-config.tpl b/devops/gcp/terraform/modules/standard-account/cloud-config.tpl index 6ebfbf3..341ef6d 100644 --- a/devops/gcp/terraform/modules/standard-account/cloud-config.tpl +++ b/devops/gcp/terraform/modules/standard-account/cloud-config.tpl @@ -58,7 +58,9 @@ write_files: "CALORI_PHX_HOST": "${hostname}", "CALORI_PHX_SERVER": true, "CALORI_CLOUD_ENVIRONMENT": "${account_name}", - "CALORI_OTP_TLS_CERT_PATH": "/usr/local/share/ca-certificates" + "CALORI_OTP_TLS_CERT_PATH": "/usr/local/share/ca-certificates", + "CALORI_SECRETS_ADAPTER": "gcp", + "CALORI_SECRETS_PATH": "calori-${account_name}-secrets", } } - path: /etc/nginx/sites-available/default diff --git a/lib/config_provider/secrets/adapter.ex b/lib/config_provider/secrets/adapter.ex new file mode 100644 index 0000000..f5fca15 --- /dev/null +++ b/lib/config_provider/secrets/adapter.ex @@ -0,0 +1,7 @@ +defmodule Calori.ConfigProvider.Secrets.Adapter do + @moduledoc """ + Behaviour that defines the secret retrieval + """ + + @callback secrets([Keyword.t()], String.t(), [Keyword.t()]) :: map() +end diff --git a/lib/config_provider/secrets/aws.ex b/lib/config_provider/secrets/aws.ex new file mode 100644 index 0000000..791b54c --- /dev/null +++ b/lib/config_provider/secrets/aws.ex @@ -0,0 +1,55 @@ +defmodule Calori.ConfigProvider.Secrets.Aws do + @moduledoc """ + Adapter implementation for retrieving secrets from AWS secret manager + """ + @behaviour Calori.ConfigProvider.Secrets.Adapter + + require Logger + + alias ExAws.Operation.JSON + + @doc """ + secrets/2. + + Args: + - secret_path_id: Path to the secret content, e. g. calori-prod-secrets + - opts is just the return value of init/1. + """ + @impl true + def secrets(_config, path, opts) do + region = System.fetch_env!("AWS_REGION") + request_opts = Keyword.merge(opts, region: region) + + fetch_aws_secret_id(path, request_opts) + end + + ### ========================================================================== + ### Private functions + ### ========================================================================== + + defp build_request(secret_name) do + JSON.new( + :secretsmanager, + %{ + data: %{"SecretId" => secret_name}, + headers: [ + {"x-amz-target", "secretsmanager.GetSecretValue"}, + {"content-type", "application/x-amz-json-1.1"} + ] + } + ) + end + + defp fetch_aws_secret_id(secret_path_id, opts) do + secret_path_id + |> build_request() + |> ExAws.request(opts) + |> case do + {:ok, %{"SecretString" => json_secret}} -> + Jason.decode!(json_secret) + + reason -> + raise "Fail to retrieve secrests with reason #{inspect(reason)}" + end + end +end diff --git a/lib/config_provider/secrets/gcp.ex b/lib/config_provider/secrets/gcp.ex new file mode 100644 index 0000000..15f10a6 --- /dev/null +++ b/lib/config_provider/secrets/gcp.ex @@ -0,0 +1,65 @@ +defmodule Calori.ConfigProvider.Secrets.Gcp do + @moduledoc """ + Adapter implementation for retrieving secrets from Gcp + """ + @behaviour Calori.ConfigProvider.Secrets.Adapter + + require Logger + + # alias ExAws.Operation.JSON + + @doc """ + secrets/2. + + Args: + - secret_path_id: Path to the secret content, e. g. calori-prod-secrets + - opts is just the return value of init/1. + """ + @impl true + def secrets(config, secret_path, _opts) do + goth_name = Calori.SecretManager.Goth + + {:ok, _} = Application.ensure_all_started(:goth) + + file_credentials = Keyword.get(config, :goth) |> Keyword.get(:file_credentials) + + source = {:service_account, Jason.decode!(file_credentials)} + + children = [ + {Finch, name: FinchSecretManagerClient}, + {Goth, name: goth_name, source: source} + ] + + {:ok, sup} = Supervisor.start_link(children, strategy: :one_for_one) + + {:ok, project_id} = Goth.Config.get(:project_id) + token = Goth.fetch!(goth_name, 5_000) + + headers = [ + {"content-type", "application/json"}, + {"authorization", "Bearer #{token.token}"}, + {"accept", "application/json"} + ] + + path = + "https://secretmanager.googleapis.com/v1/projects/#{project_id}/secrets/#{secret_path}/versions/latest:access" + + data = + :get + |> Finch.build(path, headers, []) + |> Finch.request(FinchSecretManagerClient) + |> case do + {:ok, %Finch.Response{body: body}} -> + Jason.decode!(body)["payload"]["data"] + |> Base.decode64!() + |> Jason.decode!() + + reason -> + raise "Fail to retrieve secrests with reason #{inspect(reason)}" + end + + Supervisor.stop(sup, :normal) + + data + end +end diff --git a/lib/config_provider/aws_secrets_manager.ex b/lib/config_provider/secrets/manager.ex similarity index 60% rename from lib/config_provider/aws_secrets_manager.ex rename to lib/config_provider/secrets/manager.ex index 9502281..84adf43 100644 --- a/lib/config_provider/aws_secrets_manager.ex +++ b/lib/config_provider/secrets/manager.ex @@ -1,4 +1,4 @@ -defmodule Calori.AwsSecretsManagerProvider do +defmodule Calori.ConfigProvider.Secrets.Manager do @moduledoc """ https://hexdocs.pm/elixir/1.14.0-rc.1/Config.Provider.html @@ -12,8 +12,6 @@ defmodule Calori.AwsSecretsManagerProvider do require Logger - alias ExAws.Operation.JSON - @impl Config.Provider def init(_path), do: [] @@ -28,9 +26,19 @@ defmodule Calori.AwsSecretsManagerProvider do """ @impl Config.Provider def load(config, opts) do - Logger.info("Running AWS config provider") + Logger.info("Running Config Provider for Secrets") env = Keyword.get(config, :calori) |> Keyword.get(:env) + secrets_adapter = + Keyword.get(config, :calori) + |> Keyword.get(Calori.ConfigProvider.Secrets.Manager) + |> Keyword.get(:adapter) + + secrets_path = + Keyword.get(config, :calori) + |> Keyword.get(Calori.ConfigProvider.Secrets.Manager) + |> Keyword.get(:path) + if env == "local" do Logger.info(" - No secrets retrieved, local environment") config @@ -38,12 +46,9 @@ defmodule Calori.AwsSecretsManagerProvider do {:ok, _} = Application.ensure_all_started(:hackney) {:ok, _} = Application.ensure_all_started(:ex_aws) - Logger.info(" - Retrieve secrets") - - region = System.fetch_env!("AWS_REGION") - request_opts = Keyword.merge(opts, region: region) + Logger.info(" - Trying to retrieve secrets: #{secrets_adapter} - #{secrets_path}") - secrets = fetch_aws_secret_id("calori-#{env}-secrets", request_opts) + secrets = secrets_adapter.secrets(config, secrets_path, opts) secret_key_base = keyword(:secret_key_base, secrets["CALORI_SECRET_KEY_BASE"]) erlang_cookie = secrets["CALORI_ERLANG_COOKIE"] |> String.to_atom() @@ -67,33 +72,4 @@ defmodule Calori.AwsSecretsManagerProvider do defp keyword(key_name, value) do Keyword.new([{key_name, value}]) end - - defp fetch_aws_secret_id(secret_id, opts) do - secret_id - |> build_request() - |> ExAws.request(opts) - |> parse_secrets() - end - - defp build_request(secret_name) do - JSON.new( - :secretsmanager, - %{ - data: %{"SecretId" => secret_name}, - headers: [ - {"x-amz-target", "secretsmanager.GetSecretValue"}, - {"content-type", "application/x-amz-json-1.1"} - ] - } - ) - end - - defp parse_secrets({:ok, %{"SecretString" => json_secret}}) do - Jason.decode!(json_secret) - end - - defp parse_secrets({:error, {exception, reason}}) do - Logger.error("#{inspect(exception)}: #{inspect(reason)}") - %{} - end end diff --git a/mix.exs b/mix.exs index bfcb015..be0d7eb 100644 --- a/mix.exs +++ b/mix.exs @@ -75,7 +75,8 @@ defmodule Calori.MixProject do {:mix_audit, "~> 2.1", only: [:dev, :test], runtime: false}, {:ex_aws, "~> 2.1"}, {:ex_aws_s3, "~> 2.0"}, - {:hackney, "~> 1.20"} + {:hackney, "~> 1.20"}, + {:goth, "~> 1.3.0"} ] end diff --git a/mix.lock b/mix.lock index 5f05a2f..412be7a 100644 --- a/mix.lock +++ b/mix.lock @@ -15,12 +15,14 @@ "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"}, "floki": {:hex, :floki, "0.36.2", "a7da0193538c93f937714a6704369711998a51a6164a222d710ebd54020aa7a3", [:mix], [], "hexpm", "a8766c0bc92f074e5cb36c4f9961982eda84c5d2b8e979ca67f5c268ec8ed580"}, "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"}, + "goth": {:hex, :goth, "1.3.1", "f3e08a7f23ea8992ab92d2e1d5c72ea1a8fbd2fe3a46ad1b08d0620f71374fdc", [:mix], [{:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "90326c2c0a7acda7fb75fc4a4f0cba84945d8fcb22694d36c9967cec8949937c"}, "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized"]}, "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "jellyfish": {:hex, :jellyfish, "0.1.2", "64118761f5b1cefe0385c6a8535523f0948dc5ae2d061bee0973f3ad35f1d5d3", [:mix], [], "hexpm", "46aca26f42b02dbbf7bba6c5407c46ce30f9021f494c0b0f2d67ae19f586c484"}, + "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, From 45d50bbda7f74c4248d97fe1abbb93c9a7c689e3 Mon Sep 17 00:00:00 2001 From: Thiago Esteves Date: Thu, 12 Sep 2024 09:55:36 -0300 Subject: [PATCH 2/3] Adding new config provider to the mix file --- README.md | 12 ------------ mix.exs | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/README.md b/README.md index 3db6a71..b21422c 100644 --- a/README.md +++ b/README.md @@ -283,24 +283,12 @@ The logs for deployex can be found at `/var/log/deployex/deployex-stdout.log`. ```bash root@ip-10-0-1-56:/home/ubuntu$ tail -f /var/log/deployex/deployex-stdout.log -19:59:20.035 [info] module=Deployex.AwsSecretsManagerProvider function=load/2 pid=<0.9.0> - Retrieve secrets -19:59:20.487 [info] module=Deployex.Deployment function=init/1 pid=<0.1739.0> Initialising deployment server -19:59:20.493 [info] module=Bandit function=start_link/1 pid=<0.1755.0> Running DeployexWeb.Endpoint with Bandit 1.5.3 at :::5001 (http) -19:59:20.505 [info] module=Phoenix.Endpoint.Supervisor function=log_access_url/2 pid=<0.1735.0> Access DeployexWeb.Endpoint at https://deployex.calori.com.br -19:59:20.506 [info] module=Deployex.Monitor function=init/1 pid=<0.2065.0> Initialising monitor server for instance: 1 -19:59:20.508 [info] instance=1 module=Deployex.Monitor function=run_service/2 pid=<0.2065.0> Ensure running requested for instance: 1 version: 0.1.0-627e062 -19:59:20.509 [info] instance=1 module=Deployex.Monitor function=run_service/2 pid=<0.2065.0> # Starting /var/lib/deployex/service/calori/1/current/bin/calori... -19:59:20.509 [info] instance=1 module=Deployex.Monitor function=run_service/2 pid=<0.2065.0> # Running instance: 1, monitoring pid = #PID<0.2066.0>, OS process id = 828. -19:59:20.510 [info] module=Deployex.Monitor function=init/1 pid=<0.2067.0> Initialising monitor server for instance: 2 ``` The logs for calori can be found at `/var/log/calori/calori-{instance}-stdout.log` or `/var/log/calori/calori-{instance}-stderr.log`. ```bash root@ip-10-0-1-56:/home/ubuntu$ tail -f /var/log/calori/calori-1-stdout.log -13:53:25.623 module=Calori.AwsSecretsManagerProvider function=load/2 pid=<0.9.0> [info] - Retrieve secrets -13:53:25.929 module=Bandit function=start_link/1 pid=<0.1722.0> [info] Running CaloriWeb.Endpoint with Bandit 1.5.0 at :::4000 (http) -13:53:25.934 module=Phoenix.Endpoint.Supervisor function=log_access_url/2 pid=<0.1703.0> [info] Access CaloriWeb.Endpoint at https://calori.com.br ``` ##### 4. Updating CALORI_PHX_HOST diff --git a/mix.exs b/mix.exs index be0d7eb..6345a68 100644 --- a/mix.exs +++ b/mix.exs @@ -15,7 +15,7 @@ defmodule Calori.MixProject do calori: [ steps: [:assemble, &Jellyfish.Releases.Copy.relfile/1, :tar], config_providers: [ - {Calori.AwsSecretsManagerProvider, nil} + {Calori.ConfigProvider.Secrets.Manager, nil} ] ] ], From a827a210cad5442b06a7893778949ec162e5523c Mon Sep 17 00:00:00 2001 From: Thiago Esteves Date: Thu, 12 Sep 2024 09:57:04 -0300 Subject: [PATCH 3/3] Fixing runtime.exs --- config/runtime.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 6f44abb..1eda16e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -50,12 +50,12 @@ if config_env() == :prod do case secrets_adapter do "gcp" -> - config :deployex, Calori.ConfigProvider.Secrets.Manager, + config :calori, Calori.ConfigProvider.Secrets.Manager, adapter: Calori.ConfigProvider.Secrets.Gcp, path: secrets_path "aws" -> - config :deployex, Calori.ConfigProvider.Secrets.Manager, + config :calori, Calori.ConfigProvider.Secrets.Manager, adapter: Calori.ConfigProvider.Secrets.Aws, path: secrets_path