diff --git a/config/README.md b/config/README.md index 06b61e3e9..cf5aa23d2 100644 --- a/config/README.md +++ b/config/README.md @@ -24,9 +24,69 @@ It's up to the user to provide a way to read the config from file and unmarshal Also you might find `BytesToAnyTomlStruct(logger zerolog.Logger, filename, configurationName string, target any, content []byte) error` utility method useful for unmarshalling TOMLs read from env var or files into a struct -## Secrets in TOML config +## Test Secrets -For all values regarded as secrets, their keys should end with the `_secret` suffix. For example, use `basic_auth_secret="basic-auth"` instead of `basic_auth="basic-auth"`. +Test secrets are not stored directly within the `TestConfig` TOML due to security reasons. Instead, they are passed into `TestConfig` via environment variables. Below is a list of all available secrets. Set only the secrets required for your specific tests, like so: `E2E_TEST_CHAINLINK_IMAGE=qa_ecr_image_url`. + +### Default Secret Loading + +By default, secrets are loaded from the `~/.testsecrets` dotenv file. Example of a local `~/.testsecrets` file: + +```bash +E2E_TEST_CHAINLINK_IMAGE=qa_ecr_image_url +E2E_TEST_CHAINLINK_UPGRADE_IMAGE=qa_ecr_image_url +E2E_TEST_ARBITRUM_SEPOLIA_WALLET_KEY=wallet_key +``` + +### All E2E Test Secrets + +| Secret | Env Var | Example | +| ----------------------------- | ------------------------------------------------------------------- | --------------------------------------------------- | +| Chainlink Image | `E2E_TEST_CHAINLINK_IMAGE` | `E2E_TEST_CHAINLINK_IMAGE=qa_ecr_image_url` | +| Chainlink Upgrade Image | `E2E_TEST_CHAINLINK_UPGRADE_IMAGE` | `E2E_TEST_CHAINLINK_UPGRADE_IMAGE=qa_ecr_image_url` | +| Wallet Key per network | `E2E_TEST_(.+)_WALLET_KEY` or `E2E_TEST_(.+)_WALLET_KEY_(\d+)$` | `E2E_TEST_ARBITRUM_SEPOLIA_WALLET_KEY=wallet_key` | +| RPC HTTP URL per network | `E2E_TEST_(.+)_RPC_HTTP_URL` or `E2E_TEST_(.+)_RPC_HTTP_URL_(\d+)$` | `E2E_TEST_ARBITRUM_SEPOLIA_RPC_HTTP_URL=url` | +| RPC WebSocket URL per network | `E2E_TEST_(.+)_RPC_WS_URL` or `E2E_TEST_(.+)_RPC_WS_URL_(\d+)$` | `E2E_TEST_ARBITRUM_RPC_WS_URL=ws_url` | +| Loki Tenant ID | `E2E_TEST_LOKI_TENANT_ID` | `E2E_TEST_LOKI_TENANT_ID=tenant_id` | +| Loki Endpoint | `E2E_TEST_LOKI_ENDPOINT` | `E2E_TEST_LOKI_ENDPOINT=url` | +| Loki Basic Auth | `E2E_TEST_LOKI_BASIC_AUTH` | `E2E_TEST_LOKI_BASIC_AUTH=token` | +| Loki Bearer Token | `E2E_TEST_LOKI_BEARER_TOKEN` | `E2E_TEST_LOKI_BEARER_TOKEN=token` | +| Grafana Base URL | `E2E_TEST_GRAFANA_BASE_URL` | `E2E_TEST_GRAFANA_BASE_URL=base_url` | +| Grafana Dashboard URL | `E2E_TEST_GRAFANA_DASHBOARD_URL` | `E2E_TEST_GRAFANA_DASHBOARD_URL=url` | +| Grafana Bearer Token | `E2E_TEST_GRAFANA_BEARER_TOKEN` | `E2E_TEST_GRAFANA_BEARER_TOKEN=token` | +| Pyroscope Server URL | `E2E_TEST_PYROSCOPE_SERVER_URL` | `E2E_TEST_PYROSCOPE_SERVER_URL=url` | +| Pyroscope Key | `E2E_TEST_PYROSCOPE_KEY` | `E2E_TEST_PYROSCOPE_KEY=key` | +| Pyroscope Environment | `E2E_TEST_PYROSCOPE_ENVIRONMENT` | `E2E_TEST_PYROSCOPE_ENVIRONMENT=env` | +| Pyroscope Enabled | `E2E_TEST_PYROSCOPE_ENABLED` | `E2E_TEST_PYROSCOPE_ENABLED=true` | + +### Run GitHub Workflow with Your Test Secrets + +By default, GitHub workflows execute with a set of predefined secrets. However, you can use custom secrets by specifying a unique identifier for your secrets when running the `gh workflow` command. + +#### Steps to Use Custom Secrets + +1. **Upload Local Secrets to GitHub Secrets Vault:** + + - **Install `ghsecrets` tool:** + Install the `ghsecrets` tool to manage GitHub Secrets more efficiently. + ```bash + go install github.com/smartcontractkit/chainlink-testing-framework/tools/ghsecrets@latest + ``` + - **Upload Secrets:** + Use `ghsecrets set` to upload the content of your `~/.testsecrets` file to the GitHub Secrets Vault and generate a unique identifier (referred to as `your_ghsecret_id`). + ```bash + ghsecrets set + ``` + +2. **Execute the Workflow with Custom Secrets:** + - To use the custom secrets in your GitHub Actions workflow, pass the `-f test_secrets_override_key={your_ghsecret_id}` flag when running the `gh workflow` command. + ```bash + gh workflow run -f test_secrets_override_key={your_ghsecret_id} + ``` + +#### Default Secrets Handling + +If the `test_secrets_override_key` is not provided, the workflow will default to using the secrets preconfigured in the CI environment. ## Working example diff --git a/config/testconfig.go b/config/testconfig.go index 41279e22b..c101e47b8 100644 --- a/config/testconfig.go +++ b/config/testconfig.go @@ -1,7 +1,17 @@ package config import ( + "fmt" + "os" + "regexp" + "strconv" + "strings" + + "github.com/joho/godotenv" + "github.com/pkg/errors" "github.com/smartcontractkit/seth" + + "github.com/smartcontractkit/chainlink-testing-framework/logging" ) func (c *TestConfig) GetLoggingConfig() *LoggingConfig { @@ -39,3 +49,380 @@ type TestConfig struct { Seth *seth.Config `toml:"Seth"` NodeConfig *NodeConfig `toml:"NodeConfig"` } + +// Read config values from environment variables +func (c *TestConfig) ReadConfigValuesFromEnvVars() error { + logger := logging.GetTestLogger(nil) + + walletKeys := mergeMaps(loadEnvVarSingleMap(`E2E_TEST_(.+)_WALLET_KEY$`), loadEnvVarGroupedMap(`E2E_TEST_(.+)_WALLET_KEY_(\d+)$`)) + if len(walletKeys) > 0 { + if c.Network == nil { + c.Network = &NetworkConfig{} + } + c.Network.WalletKeys = walletKeys + } + rpcHttpUrls := mergeMaps(loadEnvVarSingleMap(`E2E_TEST_(.+)_RPC_HTTP_URL$`), loadEnvVarGroupedMap(`E2E_TEST_(.+)_RPC_HTTP_URL_(\d+)$`)) + if len(rpcHttpUrls) > 0 { + if c.Network == nil { + c.Network = &NetworkConfig{} + } + c.Network.RpcHttpUrls = rpcHttpUrls + } + rpcWsUrls := mergeMaps(loadEnvVarSingleMap(`E2E_TEST_(.+)_RPC_WS_URL$`), loadEnvVarGroupedMap(`E2E_TEST_(.+)_RPC_WS_URL_(\d+)$`)) + if len(rpcWsUrls) > 0 { + if c.Network == nil { + c.Network = &NetworkConfig{} + } + c.Network.RpcWsUrls = rpcWsUrls + } + + chainlinkImage, err := readEnvVarValue("E2E_TEST_CHAINLINK_IMAGE", String) + if err != nil { + return err + } + if chainlinkImage != nil && chainlinkImage.(string) != "" { + if c.ChainlinkImage == nil { + c.ChainlinkImage = &ChainlinkImageConfig{} + } + image := chainlinkImage.(string) + logger.Debug().Msgf("Using E2E_TEST_CHAINLINK_IMAGE env var to override ChainlinkImage.Image") + c.ChainlinkImage.Image = &image + } + + chainlinkUpgradeImage, err := readEnvVarValue("E2E_TEST_CHAINLINK_UPGRADE_IMAGE", String) + if err != nil { + return err + } + if chainlinkUpgradeImage != nil && chainlinkUpgradeImage.(string) != "" { + if c.ChainlinkUpgradeImage == nil { + c.ChainlinkUpgradeImage = &ChainlinkImageConfig{} + } + image := chainlinkUpgradeImage.(string) + logger.Debug().Msgf("Using E2E_TEST_CHAINLINK_UPGRADE_IMAGE env var to override ChainlinkUpgradeImage.Image") + c.ChainlinkUpgradeImage.Image = &image + } + + lokiTenantID, err := readEnvVarValue("E2E_TEST_LOKI_TENANT_ID", String) + if err != nil { + return err + } + if lokiTenantID != nil && lokiTenantID.(string) != "" { + if c.Logging == nil { + c.Logging = &LoggingConfig{} + } + if c.Logging.Loki == nil { + c.Logging.Loki = &LokiConfig{} + } + id := lokiTenantID.(string) + logger.Debug().Msgf("Using E2E_TEST_LOKI_TENANT_ID env var to override Logging.Loki.TenantId") + c.Logging.Loki.TenantId = &id + } + + lokiEndpoint, err := readEnvVarValue("E2E_TEST_LOKI_ENDPOINT", String) + if err != nil { + return err + } + if lokiEndpoint != nil && lokiEndpoint.(string) != "" { + if c.Logging == nil { + c.Logging = &LoggingConfig{} + } + if c.Logging.Loki == nil { + c.Logging.Loki = &LokiConfig{} + } + endpoint := lokiEndpoint.(string) + logger.Debug().Msgf("Using E2E_TEST_LOKI_ENDPOINT env var to override Logging.Loki.Endpoint") + c.Logging.Loki.Endpoint = &endpoint + } + + lokiBasicAuth, err := readEnvVarValue("E2E_TEST_LOKI_BASIC_AUTH", String) + if err != nil { + return err + } + if lokiBasicAuth != nil && lokiBasicAuth.(string) != "" { + if c.Logging == nil { + c.Logging = &LoggingConfig{} + } + if c.Logging.Loki == nil { + c.Logging.Loki = &LokiConfig{} + } + basicAuth := lokiBasicAuth.(string) + logger.Debug().Msgf("Using E2E_TEST_LOKI_BASIC_AUTH env var to override Logging.Loki.BasicAuth") + c.Logging.Loki.BasicAuth = &basicAuth + } + + lokiBearerToken, err := readEnvVarValue("E2E_TEST_LOKI_BEARER_TOKEN", String) + if err != nil { + return err + } + if lokiBearerToken != nil && lokiBearerToken.(string) != "" { + if c.Logging == nil { + c.Logging = &LoggingConfig{} + } + if c.Logging.Loki == nil { + c.Logging.Loki = &LokiConfig{} + } + bearerToken := lokiBearerToken.(string) + logger.Debug().Msgf("Using E2E_TEST_LOKI_BEARER_TOKEN env var to override Logging.Loki.BearerToken") + c.Logging.Loki.BearerToken = &bearerToken + } + + grafanaBaseUrl, err := readEnvVarValue("E2E_TEST_GRAFANA_BASE_URL", String) + if err != nil { + return err + } + if grafanaBaseUrl != nil && grafanaBaseUrl.(string) != "" { + if c.Logging == nil { + c.Logging = &LoggingConfig{} + } + if c.Logging.Grafana == nil { + c.Logging.Grafana = &GrafanaConfig{} + } + baseUrl := grafanaBaseUrl.(string) + logger.Debug().Msgf("Using E2E_TEST_GRAFANA_BASE_URL env var to override Logging.Grafana.BaseUrl") + c.Logging.Grafana.BaseUrl = &baseUrl + } + + grafanaDashboardUrl, err := readEnvVarValue("E2E_TEST_GRAFANA_DASHBOARD_URL", String) + if err != nil { + return err + } + if grafanaDashboardUrl != nil && grafanaDashboardUrl.(string) != "" { + if c.Logging == nil { + c.Logging = &LoggingConfig{} + } + if c.Logging.Grafana == nil { + c.Logging.Grafana = &GrafanaConfig{} + } + dashboardUrl := grafanaDashboardUrl.(string) + logger.Debug().Msgf("Using E2E_TEST_GRAFANA_DASHBOARD_URL env var to override Logging.Grafana.DashboardUrl") + c.Logging.Grafana.DashboardUrl = &dashboardUrl + } + + grafanaBearerToken, err := readEnvVarValue("E2E_TEST_GRAFANA_BEARER_TOKEN", String) + if err != nil { + return err + } + if grafanaBearerToken != nil && grafanaBearerToken.(string) != "" { + if c.Logging == nil { + c.Logging = &LoggingConfig{} + } + if c.Logging.Grafana == nil { + c.Logging.Grafana = &GrafanaConfig{} + } + bearerToken := grafanaBearerToken.(string) + logger.Debug().Msgf("Using E2E_TEST_GRAFANA_BEARER_TOKEN env var to override Logging.Grafana.BearerToken") + c.Logging.Grafana.BearerToken = &bearerToken + } + + pyroscopeServerUrl, err := readEnvVarValue("E2E_TEST_PYROSCOPE_SERVER_URL", String) + if err != nil { + return err + } + if pyroscopeServerUrl != nil && pyroscopeServerUrl.(string) != "" { + if c.Pyroscope == nil { + c.Pyroscope = &PyroscopeConfig{} + } + serverUrl := pyroscopeServerUrl.(string) + logger.Debug().Msgf("Using E2E_TEST_PYROSCOPE_SERVER_URL env var to override Pyroscope.ServerUrl") + c.Pyroscope.ServerUrl = &serverUrl + } + + pyroscopeKey, err := readEnvVarValue("E2E_TEST_PYROSCOPE_KEY", String) + if err != nil { + return err + } + if pyroscopeKey != nil && pyroscopeKey.(string) != "" { + if c.Pyroscope == nil { + c.Pyroscope = &PyroscopeConfig{} + } + key := pyroscopeKey.(string) + logger.Debug().Msgf("Using E2E_TEST_PYROSCOPE_KEY env var to override Pyroscope.Key") + c.Pyroscope.Key = &key + } + + pyroscopeEnvironment, err := readEnvVarValue("E2E_TEST_PYROSCOPE_ENVIRONMENT", String) + if err != nil { + return err + } + if pyroscopeEnvironment != nil && pyroscopeEnvironment.(string) != "" { + if c.Pyroscope == nil { + c.Pyroscope = &PyroscopeConfig{} + } + environment := pyroscopeEnvironment.(string) + logger.Debug().Msgf("Using E2E_TEST_PYROSCOPE_ENVIRONMENT env var to override Pyroscope.Environment") + c.Pyroscope.Environment = &environment + } + + pyroscopeEnabled, err := readEnvVarValue("E2E_TEST_PYROSCOPE_ENABLED", Boolean) + if err != nil { + return err + } + if pyroscopeEnabled != nil { + if c.Pyroscope == nil { + c.Pyroscope = &PyroscopeConfig{} + } + enabled := pyroscopeEnabled.(bool) + logger.Debug().Msgf("Using E2E_TEST_PYROSCOPE_ENABLED env var to override Pyroscope.Enabled") + c.Pyroscope.Enabled = &enabled + } + + return nil +} + +// loadEnvVarGroupedMap scans all environment variables, matches them against +// a specified pattern, and returns a map of grouped values based on the pattern. +// The grouping is defined by the first capture group of the regex. +func loadEnvVarGroupedMap(pattern string) map[string][]string { + logger := logging.GetTestLogger(nil) + re := regexp.MustCompile(pattern) + groupedVars := make(map[string][]string) + for _, env := range os.Environ() { + pair := strings.SplitN(env, "=", 2) + if len(pair) != 2 { + continue + } + key, value := pair[0], pair[1] + matches := re.FindStringSubmatch(key) + if len(matches) > 1 && value != "" { + group := matches[1] // Use the first capture group for grouping + groupedVars[group] = append(groupedVars[group], value) + logger.Debug().Msgf("Will override test config from env var '%s'", key) + } + } + return groupedVars +} + +func loadEnvVarSingleMap(pattern string) map[string]string { + logger := logging.GetTestLogger(nil) + re := regexp.MustCompile(pattern) + singleVars := make(map[string]string) + for _, env := range os.Environ() { + pair := strings.SplitN(env, "=", 2) + if len(pair) != 2 { + continue + } + key, value := pair[0], pair[1] + matches := re.FindStringSubmatch(key) + if len(matches) > 1 && value != "" { + group := matches[1] // Use the first capture group for grouping + singleVars[group] = value + logger.Debug().Msgf("Will override test config from env var '%s'", key) + } + } + return singleVars +} + +// Merges a map[string]string with a map[string][]string and returns a new map[string][]string. +// Elements from the single map are inserted at index 0 in the slice of the new map. +func mergeMaps(single map[string]string, multi map[string][]string) map[string][]string { + newMap := make(map[string][]string) + + // First, copy all elements from the multi map to the new map + for key, values := range multi { + newMap[key] = make([]string, len(values)) + copy(newMap[key], values) + } + + // Next, insert or prepend the elements from the single map + for key, value := range single { + if existingValues, exists := newMap[key]; exists { + // Prepend the value from the single map + newMap[key] = append([]string{value}, existingValues...) + } else { + // Initialize a new slice if the key does not exist + newMap[key] = []string{value} + } + } + + return newMap +} + +type EnvValueType int + +const ( + String EnvValueType = iota + Integer + Boolean + Float +) + +// readEnvVarValue reads an environment variable and returns the value parsed according to the specified type. +func readEnvVarValue(envVarName string, valueType EnvValueType) (interface{}, error) { + // Get the environment variable value + value, isSet := os.LookupEnv(envVarName) + if !isSet { + return nil, nil + } + if isSet && value == "" { + return "", nil // Return "" if the environment variable is not set + } + + // Parse the value according to the specified type + switch valueType { + case Integer: + intVal, err := strconv.Atoi(value) + if err != nil { + return nil, fmt.Errorf("error converting value to integer: %v", err) + } + return intVal, nil + case Boolean: + boolVal, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("error converting value to boolean: %v", err) + } + return boolVal, nil + case Float: + floatVal, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil, fmt.Errorf("error converting value to float: %v", err) + } + return floatVal, nil + default: // String or unrecognized type + return value, nil + } +} + +func LoadSecretEnvsFromFile() error { + logger := logging.GetTestLogger(nil) + + homeDir, err := os.UserHomeDir() + if err != nil { + return errors.Wrapf(err, "error getting user home directory") + } + path := fmt.Sprintf("%s/.testsecrets", homeDir) + + // Check if the file exists + info, err := os.Stat(path) + if os.IsNotExist(err) { + logger.Debug().Msgf("No test secrets file found at %s", path) + return nil + } + if info.IsDir() { + return errors.Errorf("%s file is a directory but should be a file", path) + } + + // Load existing environment variables into a map + existingEnv := make(map[string]string) + for _, env := range os.Environ() { + pair := strings.SplitN(env, "=", 2) + existingEnv[pair[0]] = pair[1] + } + + // Load variables from the env file + envMap, err := godotenv.Read(path) + if err != nil { + return errors.Wrapf(err, "error loading %s file with test secrets", path) + } + + // Set env vars from file only if they are not already set + for key, value := range envMap { + if _, exists := existingEnv[key]; !exists { + logger.Debug().Msgf("Setting env var %s from %s file", key, path) + os.Setenv(key, value) + } else { + logger.Debug().Msgf("Env var %s already set, not overriding it from %s file", key, path) + } + } + + return nil +} diff --git a/config/testconfig_test.go b/config/testconfig_test.go new file mode 100644 index 000000000..f94b46a7a --- /dev/null +++ b/config/testconfig_test.go @@ -0,0 +1,118 @@ +package config + +import ( + "errors" + "os" + "reflect" + "testing" + + "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" +) + +func TestReadConfigValuesFromEnvVars(t *testing.T) { + tests := []struct { + name string + setupFunc func() + cleanupFunc func() + expectedConfig TestConfig + expectedError error + }{ + { + name: "All configurations set correctly", + setupFunc: func() { + os.Setenv("E2E_TEST_GROUP1_WALLET_KEY_1", "walletValue1") + os.Setenv("E2E_TEST_GROUP2_RPC_HTTP_URL_1", "httpUrl1") + os.Setenv("E2E_TEST_GROUP3_RPC_WS_URL_1", "wsUrl1") + os.Setenv("E2E_TEST_CHAINLINK_IMAGE", "imageValue") + os.Setenv("E2E_TEST_PYROSCOPE_ENABLED", "true") + }, + cleanupFunc: func() { + os.Unsetenv("E2E_TEST_GROUP1_WALLET_KEY_1") + os.Unsetenv("E2E_TEST_GROUP2_RPC_HTTP_URL_1") + os.Unsetenv("E2E_TEST_GROUP3_RPC_WS_URL_1") + os.Unsetenv("E2E_TEST_CHAINLINK_IMAGE") + os.Unsetenv("E2E_TEST_PYROSCOPE_ENABLED") + }, + expectedConfig: TestConfig{ + Network: &NetworkConfig{ + WalletKeys: map[string][]string{"GROUP1": {"walletValue1"}}, + RpcHttpUrls: map[string][]string{"GROUP2": {"httpUrl1"}}, + RpcWsUrls: map[string][]string{"GROUP3": {"wsUrl1"}}, + }, + Pyroscope: &PyroscopeConfig{Enabled: ptr.Ptr[bool](true)}, + ChainlinkImage: &ChainlinkImageConfig{Image: newString("imageValue")}, + }, + }, + { + name: "Environment variables are empty strings", + setupFunc: func() { + os.Setenv("E2E_TEST_GROUP1_WALLET_KEY_1", "") + os.Setenv("E2E_TEST_GROUP2_RPC_HTTP_URL_1", "") + os.Setenv("E2E_TEST_GROUP3_RPC_WS_URL_1", "") + os.Setenv("E2E_TEST_CHAINLINK_IMAGE", "") + }, + cleanupFunc: func() { + os.Unsetenv("E2E_TEST_GROUP1_WALLET_KEY_1") + os.Unsetenv("E2E_TEST_GROUP2_RPC_HTTP_URL_1") + os.Unsetenv("E2E_TEST_GROUP3_RPC_WS_URL_1") + os.Unsetenv("E2E_TEST_CHAINLINK_IMAGE") + }, + expectedConfig: TestConfig{}, + }, + { + name: "Environment variables set with mixed numeric suffixes", + setupFunc: func() { + os.Setenv("E2E_TEST_ARBITRUM_SEPOLIA_RPC_HTTP_URL", "url") + os.Setenv("E2E_TEST_ARBITRUM_SEPOLIA_RPC_HTTP_URL_1", "url1") + os.Setenv("E2E_TEST_ARBITRUM_SEPOLIA_RPC_WS_URL", "wsurl") + os.Setenv("E2E_TEST_ARBITRUM_SEPOLIA_RPC_WS_URL_1", "wsurl1") + os.Setenv("E2E_TEST_OPTIMISM_SEPOLIA_WALLET_KEY", "wallet") + os.Setenv("E2E_TEST_OPTIMISM_SEPOLIA_WALLET_KEY_1", "wallet1") + os.Setenv("E2E_TEST_OPTIMISM_SEPOLIA_WALLET_KEY_2", "wallet2") + }, + cleanupFunc: func() { + os.Unsetenv("E2E_TEST_ARBITRUM_SEPOLIA_RPC_HTTP_URL_1") + os.Unsetenv("E2E_TEST_ARBITRUM_SEPOLIA_RPC_WS_URL") + os.Unsetenv("E2E_TEST_ARBITRUM_SEPOLIA_RPC_WS_URL_1") + os.Unsetenv("E2E_TEST_OPTIMISM_SEPOLIA_WALLET_KEY") + os.Unsetenv("E2E_TEST_OPTIMISM_SEPOLIA_WALLET_KEY_1") + os.Unsetenv("E2E_TEST_ARBITRUM_SEPOLIA_RPC_HTTP_URL_2") + os.Unsetenv("E2E_TEST_OPTIMISM_SEPOLIA_WALLET_KEY_2") + }, + expectedConfig: TestConfig{ + Network: &NetworkConfig{ + RpcHttpUrls: map[string][]string{"ARBITRUM_SEPOLIA": {"url", "url1"}}, + RpcWsUrls: map[string][]string{"ARBITRUM_SEPOLIA": {"wsurl", "wsurl1"}}, + WalletKeys: map[string][]string{"OPTIMISM_SEPOLIA": {"wallet", "wallet1", "wallet2"}}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + localTt := tt // Create a local copy of tt + localTt.setupFunc() // Setup the test environment + defer localTt.cleanupFunc() // Ensure cleanup after the test + + c := &TestConfig{} + + // Execute + err := c.ReadConfigValuesFromEnvVars() + + // Verify error handling + if !errors.Is(err, localTt.expectedError) { + t.Errorf("Expected error to be %v, got %v", localTt.expectedError, err) + } + + // Verify the config + if !reflect.DeepEqual(c, &localTt.expectedConfig) { + t.Errorf("Expected config to be %+v, got %+v", localTt.expectedConfig, c) + } + }) + } +} + +func newString(s string) *string { + return &s +} diff --git a/go.mod b/go.mod index c5a5306e2..fab115262 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/google/uuid v1.6.0 github.com/imdario/mergo v0.3.16 github.com/jmoiron/sqlx v1.3.5 + github.com/joho/godotenv v1.5.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/lib/pq v1.10.9 github.com/onsi/gomega v1.27.8 diff --git a/go.sum b/go.sum index 67b989eb0..8474f557e 100644 --- a/go.sum +++ b/go.sum @@ -695,6 +695,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= diff --git a/k8s/config/overrides.go b/k8s/config/overrides.go index 35204356d..88732ded9 100644 --- a/k8s/config/overrides.go +++ b/k8s/config/overrides.go @@ -10,6 +10,8 @@ import ( const ( EnvVarPrefix = "TEST_" + E2ETestEnvVarPrefix = "E2E_TEST_" + EnvVarSkipManifestUpdate = "SKIP_MANIFEST_UPDATE" EnvVarSkipManifestUpdateDescription = "Skip updating manifest when connecting to the namespace" EnvVarSkipManifestUpdateExample = "false" diff --git a/k8s/environment/runner.go b/k8s/environment/runner.go index d77469301..2ce8018ce 100644 --- a/k8s/environment/runner.go +++ b/k8s/environment/runner.go @@ -372,6 +372,10 @@ func jobEnvVars(props *Props) *[]*k8s.EnvVar { log.Debug().Str(e[:i], e[i+1:]).Msg("Forwarding generic Env Var") env[withoutPrefix] = e[i+1:] } + if strings.HasPrefix(e[:i], config.E2ETestEnvVarPrefix) { + log.Debug().Str("key", e[:i]).Msg("Forwarding E2E Test Env Var") + env[e[:i]] = e[i+1:] + } } } diff --git a/tools/ghsecrets/Makefile b/tools/ghsecrets/Makefile new file mode 100644 index 000000000..7a53b93be --- /dev/null +++ b/tools/ghsecrets/Makefile @@ -0,0 +1,5 @@ +lint: + golangci-lint --color=always run ./... --fix -v + +test_unit: + go test -timeout 5m -json -cover -covermode=count -coverprofile=unit-test-coverage.out ./... 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci diff --git a/tools/ghsecrets/README.md b/tools/ghsecrets/README.md new file mode 100644 index 000000000..e214761df --- /dev/null +++ b/tools/ghsecrets/README.md @@ -0,0 +1,56 @@ +# ghsecrets + +ghsecrets is a command-line tool designed to manage and set test secrets in GitHub via the GitHub CLI. + +## Installation + +To install ghsecrets CLI, you need to have Go installed on your machine. With Go installed, run the following command: + +```sh +go install github.com/smartcontractkit/chainlink-testing-framework/tools/ghsecrets@latest +``` + +Please install GitHub CLI to use this tool - https://cli.github.com/ + +## Usage + +Set default test secrets from ~/.testsecrets file: + +```sh +ghsecrets set +``` + +## FAQ + +### Q: What should I do if I get "command not found: ghsecrets" after installation? + +This error typically means that the directory where Go installs its binaries is not included in your system's PATH. The binaries are usually installed in $GOPATH/bin or $GOBIN. Here's how you can resolve this issue: + +1. Add Go bin directory to PATH: + +- First, find out where your Go bin directory is by running: + + ```sh + echo $(go env GOPATH)/bin + ``` + + This command will print the path where Go binaries are installed, typically something like /home/username/go/bin + +- Add the following line at the end of the file: + + ```sh + export PATH="$PATH:" + ``` + +- Apply the changes by sourcing the file: + ```sh + source ~/.bashrc # Use the appropriate file like .zshrc if needed + ``` + +2. Alternatively, run using the full path: + + If you prefer not to alter your PATH, or if you are troubleshooting temporarily, you can run the tool directly using its full path: + + ```sh + $(go env GOPATH)/bin/ghsecrets set + ``` diff --git a/tools/ghsecrets/go.mod b/tools/ghsecrets/go.mod new file mode 100644 index 000000000..9681ba629 --- /dev/null +++ b/tools/ghsecrets/go.mod @@ -0,0 +1,9 @@ +module github.com/smartcontractkit/chainlink-testing-framework/tools/ghsecrets + +go 1.21.9 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/tools/ghsecrets/go.sum b/tools/ghsecrets/go.sum new file mode 100644 index 000000000..912390a78 --- /dev/null +++ b/tools/ghsecrets/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/ghsecrets/main.go b/tools/ghsecrets/main.go new file mode 100644 index 000000000..b7954eaef --- /dev/null +++ b/tools/ghsecrets/main.go @@ -0,0 +1,121 @@ +package main + +import ( + "encoding/base64" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/spf13/cobra" +) + +func main() { + var filePath string + var customSecretID string + + var setCmd = &cobra.Command{ + Use: "set", + Short: "Set test secrets in GitHub", + Run: func(cmd *cobra.Command, args []string) { + if !isGHInstalled() { + fmt.Println("GitHub CLI not found. Please go to https://cli.github.com/ and install it to use this tool.") + return + } + + if err := validateFile(filePath); err != nil { + fmt.Println(err) + return + } + + secretID, err := getSecretID(customSecretID) + if err != nil { + log.Fatalf("Failed to obtain secret ID: %s", err) + } + + setSecret(filePath, secretID) + }, + } + + var rootCmd = &cobra.Command{ + Use: "ghsecrets", + Short: "A tool for managing GitHub test secrets", + } + + rootCmd.AddCommand(setCmd) + setCmd.PersistentFlags().StringVarP(&filePath, "file", "f", defaultSecretsPath(), "path to dotenv file with test secrets") + setCmd.PersistentFlags().StringVarP(&customSecretID, "secret-id", "s", "", "custom secret ID. Do not use unless you know what you are doing") + + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func defaultSecretsPath() string { + homeDir, err := os.UserHomeDir() + if err != nil { + log.Fatalf("Failed to get user home directory: %s", err) + } + return filepath.Join(homeDir, ".testsecrets") +} + +func isGHInstalled() bool { + _, err := exec.LookPath("gh") + return err == nil +} + +func validateFile(filePath string) error { + info, err := os.Stat(filePath) + if os.IsNotExist(err) { + return fmt.Errorf("file '%s' does not exist", filePath) + } + if info.Size() == 0 { + return fmt.Errorf("file '%s' is empty", filePath) + } + return nil +} + +func getSecretID(customID string) (string, error) { + if customID != "" { + return customID, nil + } + usernameCmd := exec.Command("gh", "api", "user", "--jq", ".login") + usernameOutput, err := usernameCmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to execute command: %s, output: %s", err, usernameOutput) + } + trimmedUsername := strings.TrimSpace(string(usernameOutput)) + secretID := fmt.Sprintf("BASE64_TESTSECRETS_%s", trimmedUsername) + return strings.ToUpper(secretID), nil +} + +func setSecret(filePath, secretID string) { + // Read the file content + data, err := os.ReadFile(filePath) + if err != nil { + log.Fatalf("Failed to read file: %s", err) + } + + // Base64 encode the file content + encoded := base64.StdEncoding.EncodeToString(data) + + // Construct the GitHub CLI command to set the secret + setSecretCmd := exec.Command("gh", "secret", "set", secretID, "--body", encoded) + setSecretCmd.Stdin = strings.NewReader(encoded) + + // Execute the command to set the secret + output, err := setSecretCmd.CombinedOutput() + if err != nil { + log.Fatalf("Failed to set secret: %s\nOutput: %s", err, string(output)) + } + + fmt.Printf( + "Test secret set successfully in Github with key: %s\n\n"+ + "To run a Github workflow with the test secrets, use the 'test_secrets_override_key' flag.\n"+ + "Example: gh workflow run ${workflow_name} -f test_secrets_override_key=%s\n", + secretID, secretID, + ) +} diff --git a/tools/ghsecrets/package.json b/tools/ghsecrets/package.json new file mode 100644 index 000000000..83819d3b0 --- /dev/null +++ b/tools/ghsecrets/package.json @@ -0,0 +1,5 @@ +{ + "name": "ghsecrets", + "description": "A tool for managing GitHub test secrets", + "version": "1.0.0" +}