Adding support to AWS and GCP based on the configuration passed

README.md

@@ -283,24 +283,12 @@ The logs for deployex can be found at `/var/log/deployex/deployex-stdout.log`.
 
 ```bash
 root@ip-10-0-1-56:/home/ubuntu$ tail -f /var/log/deployex/deployex-stdout.log
-19:59:20.035 [info] module=Deployex.AwsSecretsManagerProvider function=load/2 pid=<0.9.0>    - Retrieve secrets
-19:59:20.487 [info] module=Deployex.Deployment function=init/1 pid=<0.1739.0>  Initialising deployment server
-19:59:20.493 [info] module=Bandit function=start_link/1 pid=<0.1755.0>  Running DeployexWeb.Endpoint with Bandit 1.5.3 at :::5001 (http)
-19:59:20.505 [info] module=Phoenix.Endpoint.Supervisor function=log_access_url/2 pid=<0.1735.0>  Access DeployexWeb.Endpoint at https://deployex.calori.com.br
-19:59:20.506 [info] module=Deployex.Monitor function=init/1 pid=<0.2065.0>  Initialising monitor server for instance: 1
-19:59:20.508 [info] instance=1 module=Deployex.Monitor function=run_service/2 pid=<0.2065.0>  Ensure running requested for instance: 1 version: 0.1.0-627e062
-19:59:20.509 [info] instance=1 module=Deployex.Monitor function=run_service/2 pid=<0.2065.0>   # Starting /var/lib/deployex/service/calori/1/current/bin/calori...
-19:59:20.509 [info] instance=1 module=Deployex.Monitor function=run_service/2 pid=<0.2065.0>   # Running instance: 1, monitoring pid = #PID<0.2066.0>, OS process id = 828.
-19:59:20.510 [info] module=Deployex.Monitor function=init/1 pid=<0.2067.0>  Initialising monitor server for instance: 2
 ```
 
 The logs for calori can be found at `/var/log/calori/calori-{instance}-stdout.log` or `/var/log/calori/calori-{instance}-stderr.log`.
 
 ```bash
 root@ip-10-0-1-56:/home/ubuntu$ tail -f /var/log/calori/calori-1-stdout.log
-13:53:25.623 module=Calori.AwsSecretsManagerProvider function=load/2 pid=<0.9.0> [info]   - Retrieve secrets
-13:53:25.929 module=Bandit function=start_link/1 pid=<0.1722.0> [info] Running CaloriWeb.Endpoint with Bandit 1.5.0 at :::4000 (http)
-13:53:25.934 module=Phoenix.Endpoint.Supervisor function=log_access_url/2 pid=<0.1703.0> [info] Access CaloriWeb.Endpoint at https://calori.com.br
 ```
 
 ##### 4. Updating CALORI_PHX_HOST

config/runtime.exs

@@ -45,6 +45,29 @@ if config_env() == :prod do
       port: port
     ]
 
+  secrets_adapter = System.fetch_env!("CALORI_SECRETS_ADAPTER")
+  secrets_path = System.fetch_env!("CALORI_SECRETS_PATH")
+
+  case secrets_adapter do
+    "gcp" ->
+      config :calori, Calori.ConfigProvider.Secrets.Manager,
+        adapter: Calori.ConfigProvider.Secrets.Gcp,
+        path: secrets_path
+
+    "aws" ->
+      config :calori, Calori.ConfigProvider.Secrets.Manager,
+        adapter: Calori.ConfigProvider.Secrets.Aws,
+        path: secrets_path
+
+    adapter ->
+      raise "Secret #{adapter} not supported"
+  end
+
+  if secrets_adapter == "gcp" do
+    config :goth,
+      file_credentials: System.fetch_env!("GOOGLE_APPLICATION_CREDENTIALS") |> File.read!()
+  end
+
   # ## SSL Support
   #
   # To get SSL working, you will need to add the `https` key

devops/aws/terraform/modules/standard-account/cloud-config.tpl

@@ -50,7 +50,9 @@ write_files:
             "CALORI_PHX_HOST": "${hostname}",
             "CALORI_PHX_SERVER": true,
             "CALORI_CLOUD_ENVIRONMENT": "${account_name}",
-            "CALORI_OTP_TLS_CERT_PATH": "/usr/local/share/ca-certificates"
+            "CALORI_OTP_TLS_CERT_PATH": "/usr/local/share/ca-certificates",
+            "CALORI_SECRETS_ADAPTER": "aws",
+            "CALORI_SECRETS_PATH": "calori-${account_name}-secrets",
         }
       }
   - path: /home/ubuntu/config.json

devops/gcp/terraform/modules/standard-account/cloud-config.tpl

@@ -58,7 +58,9 @@ write_files:
             "CALORI_PHX_HOST": "${hostname}",
             "CALORI_PHX_SERVER": true,
             "CALORI_CLOUD_ENVIRONMENT": "${account_name}",
-            "CALORI_OTP_TLS_CERT_PATH": "/usr/local/share/ca-certificates"
+            "CALORI_OTP_TLS_CERT_PATH": "/usr/local/share/ca-certificates",
+            "CALORI_SECRETS_ADAPTER": "gcp",
+            "CALORI_SECRETS_PATH": "calori-${account_name}-secrets",
         }
       }
   - path: /etc/nginx/sites-available/default

lib/config_provider/secrets/adapter.ex

@@ -0,0 +1,7 @@
+defmodule Calori.ConfigProvider.Secrets.Adapter do
+  @moduledoc """
+  Behaviour that defines the secret retrieval
+  """
+
+  @callback secrets([Keyword.t()], String.t(), [Keyword.t()]) :: map()
+end

lib/config_provider/secrets/aws.ex

@@ -0,0 +1,55 @@
+defmodule Calori.ConfigProvider.Secrets.Aws do
+  @moduledoc """
+  Adapter implementation for retrieving secrets from AWS secret manager
+  """
+  @behaviour Calori.ConfigProvider.Secrets.Adapter
+
+  require Logger
+
+  alias ExAws.Operation.JSON
+
+  @doc """
+  secrets/2.
+
+  Args:
+    - secret_path_id: Path to the secret content, e. g. calori-prod-secrets
+    - opts is just the return value of init/1.
+  """
+  @impl true
+  def secrets(_config, path, opts) do
+    region = System.fetch_env!("AWS_REGION")
+    request_opts = Keyword.merge(opts, region: region)
+
+    fetch_aws_secret_id(path, request_opts)
+  end
+
+  ### ==========================================================================
+  ### Private functions
+  ### ==========================================================================
+
+  defp build_request(secret_name) do
+    JSON.new(
+      :secretsmanager,
+      %{
+        data: %{"SecretId" => secret_name},
+        headers: [
+          {"x-amz-target", "secretsmanager.GetSecretValue"},
+          {"content-type", "application/x-amz-json-1.1"}
+        ]
+      }
+    )
+  end
+
+  defp fetch_aws_secret_id(secret_path_id, opts) do
+    secret_path_id
+    |> build_request()
+    |> ExAws.request(opts)
+    |> case do
+      {:ok, %{"SecretString" => json_secret}} ->
+        Jason.decode!(json_secret)
+
+      reason ->
+        raise "Fail to retrieve secrests with reason #{inspect(reason)}"
+    end
+  end
+end

lib/config_provider/secrets/gcp.ex

@@ -0,0 +1,65 @@
+defmodule Calori.ConfigProvider.Secrets.Gcp do
+  @moduledoc """
+  Adapter implementation for retrieving secrets from Gcp
+  """
+  @behaviour Calori.ConfigProvider.Secrets.Adapter
+
+  require Logger
+
+  # alias ExAws.Operation.JSON
+
+  @doc """
+  secrets/2.
+
+  Args:
+    - secret_path_id: Path to the secret content, e. g. calori-prod-secrets
+    - opts is just the return value of init/1.
+  """
+  @impl true
+  def secrets(config, secret_path, _opts) do
+    goth_name = Calori.SecretManager.Goth
+
+    {:ok, _} = Application.ensure_all_started(:goth)
+
+    file_credentials = Keyword.get(config, :goth) |> Keyword.get(:file_credentials)
+
+    source = {:service_account, Jason.decode!(file_credentials)}
+
+    children = [
+      {Finch, name: FinchSecretManagerClient},
+      {Goth, name: goth_name, source: source}
+    ]
+
+    {:ok, sup} = Supervisor.start_link(children, strategy: :one_for_one)
+
+    {:ok, project_id} = Goth.Config.get(:project_id)
+    token = Goth.fetch!(goth_name, 5_000)
+
+    headers = [
+      {"content-type", "application/json"},
+      {"authorization", "Bearer #{token.token}"},
+      {"accept", "application/json"}
+    ]
+
+    path =
+      "https://secretmanager.googleapis.com/v1/projects/#{project_id}/secrets/#{secret_path}/versions/latest:access"
+
+    data =
+      :get
+      |> Finch.build(path, headers, [])
+      |> Finch.request(FinchSecretManagerClient)
+      |> case do
+        {:ok, %Finch.Response{body: body}} ->
+          Jason.decode!(body)["payload"]["data"]
+          |> Base.decode64!()
+          |> Jason.decode!()
+
+        reason ->
+          raise "Fail to retrieve secrests with reason #{inspect(reason)}"
+      end
+
+    Supervisor.stop(sup, :normal)
+
+    data
+  end
+end

lib/config_provider/aws_secrets_manager.ex → lib/config_provider/secrets/manager.ex

@@ -1,4 +1,4 @@
-defmodule Calori.AwsSecretsManagerProvider do
+defmodule Calori.ConfigProvider.Secrets.Manager do
   @moduledoc """
   https://hexdocs.pm/elixir/1.14.0-rc.1/Config.Provider.html
 
@@ -12,8 +12,6 @@ defmodule Calori.AwsSecretsManagerProvider do
 
   require Logger
 
-  alias ExAws.Operation.JSON
-
   @impl Config.Provider
   def init(_path), do: []
 
@@ -28,22 +26,29 @@ defmodule Calori.AwsSecretsManagerProvider do
   """
   @impl Config.Provider
   def load(config, opts) do
-    Logger.info("Running AWS config provider")
+    Logger.info("Running Config Provider for Secrets")
     env = Keyword.get(config, :calori) |> Keyword.get(:env)
 
+    secrets_adapter =
+      Keyword.get(config, :calori)
+      |> Keyword.get(Calori.ConfigProvider.Secrets.Manager)
+      |> Keyword.get(:adapter)
+
+    secrets_path =
+      Keyword.get(config, :calori)
+      |> Keyword.get(Calori.ConfigProvider.Secrets.Manager)
+      |> Keyword.get(:path)
+
     if env == "local" do
       Logger.info("  - No secrets retrieved, local environment")
       config
     else
       {:ok, _} = Application.ensure_all_started(:hackney)
       {:ok, _} = Application.ensure_all_started(:ex_aws)
 
-      Logger.info("  - Retrieve secrets")
-
-      region = System.fetch_env!("AWS_REGION")
-      request_opts = Keyword.merge(opts, region: region)
+      Logger.info("  - Trying to retrieve secrets: #{secrets_adapter} - #{secrets_path}")
 
-      secrets = fetch_aws_secret_id("calori-#{env}-secrets", request_opts)
+      secrets = secrets_adapter.secrets(config, secrets_path, opts)
 
       secret_key_base = keyword(:secret_key_base, secrets["CALORI_SECRET_KEY_BASE"])
       erlang_cookie = secrets["CALORI_ERLANG_COOKIE"] |> String.to_atom()
@@ -67,33 +72,4 @@ defmodule Calori.AwsSecretsManagerProvider do
   defp keyword(key_name, value) do
     Keyword.new([{key_name, value}])
   end
-
-  defp fetch_aws_secret_id(secret_id, opts) do
-    secret_id
-    |> build_request()
-    |> ExAws.request(opts)
-    |> parse_secrets()
-  end
-
-  defp build_request(secret_name) do
-    JSON.new(
-      :secretsmanager,
-      %{
-        data: %{"SecretId" => secret_name},
-        headers: [
-          {"x-amz-target", "secretsmanager.GetSecretValue"},
-          {"content-type", "application/x-amz-json-1.1"}
-        ]
-      }
-    )
-  end
-
-  defp parse_secrets({:ok, %{"SecretString" => json_secret}}) do
-    Jason.decode!(json_secret)
-  end
-
-  defp parse_secrets({:error, {exception, reason}}) do
-    Logger.error("#{inspect(exception)}: #{inspect(reason)}")
-    %{}
-  end
 end

mix.exs

@@ -15,7 +15,7 @@ defmodule Calori.MixProject do
         calori: [
           steps: [:assemble, &Jellyfish.Releases.Copy.relfile/1, :tar],
           config_providers: [
-            {Calori.AwsSecretsManagerProvider, nil}
+            {Calori.ConfigProvider.Secrets.Manager, nil}
           ]
         ]
       ],
@@ -75,7 +75,8 @@ defmodule Calori.MixProject do
       {:mix_audit, "~> 2.1", only: [:dev, :test], runtime: false},
       {:ex_aws, "~> 2.1"},
       {:ex_aws_s3, "~> 2.0"},
-      {:hackney, "~> 1.20"}
+      {:hackney, "~> 1.20"},
+      {:goth, "~> 1.3.0"}
     ]
   end
 

mix.lock

@@ -15,12 +15,14 @@
   "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"},
   "floki": {:hex, :floki, "0.36.2", "a7da0193538c93f937714a6704369711998a51a6164a222d710ebd54020aa7a3", [:mix], [], "hexpm", "a8766c0bc92f074e5cb36c4f9961982eda84c5d2b8e979ca67f5c268ec8ed580"},
   "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"},
+  "goth": {:hex, :goth, "1.3.1", "f3e08a7f23ea8992ab92d2e1d5c72ea1a8fbd2fe3a46ad1b08d0620f71374fdc", [:mix], [{:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "90326c2c0a7acda7fb75fc4a4f0cba84945d8fcb22694d36c9967cec8949937c"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized"]},
   "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
   "jellyfish": {:hex, :jellyfish, "0.1.2", "64118761f5b1cefe0385c6a8535523f0948dc5ae2d061bee0973f3ad35f1d5d3", [:mix], [], "hexpm", "46aca26f42b02dbbf7bba6c5407c46ce30f9021f494c0b0f2d67ae19f586c484"},
+  "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
   "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"},