diff --git a/Makefile b/Makefile index 9ef065ae77..61598ce5c0 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ clean-acceptance: build: $(all_modules) build-tests: build-test build-test: $(addprefix test_,$(go_modules)) -build-all: build build-test build-test-app ## Build all modules and tests +build-all: build build-test build-test-app mta-build ## Build all modules and tests db: target/db target/db: @echo "# building $@" @@ -260,13 +260,6 @@ build/autoscaler-test.tgz: @mkdir -p build @bosh create-release --force --timestamp-version --tarball=build/autoscaler-test.tgz -.PHONY: acceptance-release -acceptance-release: clean-acceptance go-mod-tidy go-mod-vendor build-test-app - @echo " - building acceptance test release '${VERSION}' to dir: '${DEST}' " - @mkdir -p ${DEST} - ${AUTOSCALER_DIR}/scripts/compile-acceptance-tests.sh - @tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance' - .PHONY: generate-fakes autoscaler.generate-fakes test-app.generate-fakes generate-fakes: autoscaler.generate-fakes test-app.generate-fakes autoscaler.generate-fakes: @@ -360,7 +353,21 @@ deploy-prometheus: ${prometheus-bosh-release-path}/manifests ${CI_DIR}/infrastructure/scripts/deploy-prometheus.sh; +.PHONY: mta-release +mta-release: mta-build + @echo " - building mtar release '${VERSION}' to dir: '${DEST}' " + +.PHONY: acceptance-release +acceptance-release: clean-acceptance go-mod-tidy go-mod-vendor build-test-app + @echo " - building acceptance test release '${VERSION}' to dir: '${DEST}' " + @mkdir -p ${DEST} + ${AUTOSCALER_DIR}/scripts/compile-acceptance-tests.sh + @tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance' +.PHONY: mta-build +mta-build: + @echo " - building mta" + @make --directory='./src/autoscaler' mta-build .PHONY: build-test-app build-test-app: diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index afb7141a8b..528af0340f 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -148,6 +148,14 @@ resources: regexp: releases/app-autoscaler-acceptance-tests-v(.*).tgz initial_path: releases/app-autoscaler-acceptance-tests-v0.0.1.tgz +- name: mtar-bucket + type: gcs-resource + source: + bucket: app-autoscaler-releases + json_key: ((autoscaler_releases_uploader_key)) + regexp: releases/app-autoscaler-v(.*).mtar + initial_path: releases/app-autoscaler-v0.0.1.mtar + - name: golang-release type: git icon: github @@ -518,6 +526,9 @@ jobs: - put: acceptance-tests-bucket params: file: "build/artifacts/app-autoscaler-acceptance-tests-v*.tgz" + - put: mtar-bucket + params: + file: "build/artifacts/app-autoscaler-release-v*.mtar" - put: gh-release params: name: build/name diff --git a/ci/autoscaler/scripts/release-autoscaler.sh b/ci/autoscaler/scripts/release-autoscaler.sh index f1e9c1760e..eb88e94ee0 100755 --- a/ci/autoscaler/scripts/release-autoscaler.sh +++ b/ci/autoscaler/scripts/release-autoscaler.sh @@ -1,21 +1,8 @@ #! /usr/bin/env bash # NOTE: you can run this locally for testing !!! -# beware that it adds a commit you need to drop each time also you need to remove dev_releases from root. # -# DEPLOYMENT=foo \ -# GITHUB_TOKEN="ghp_..." \ -# PREV_VERSION=12.2.1 \ -# DEST="${PWD}/../../../build" \ -# VERSION="12.3.0" \ -# BUILD_OPTS="--force" \ -# AUTOSCALER_CI_BOT_NAME="foo" \ -# AUTOSCALER_CI_BOT_EMAIL="foo@bar.baz" \ -# AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC="ssh-ed25519 AAAA... foo@bar.baz" \ -# AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE="-----BEGIN OPENSSH PRIVATE KEY----- -# b3Bl... -# -----END OPENSSH PRIVATE KEY-----" \ -# ./ci/autoscaler/scripts/release-autoscaler.sh +# ./script/local_release_autoscaler.sh [ -n "${DEBUG}" ] && set -x @@ -56,6 +43,17 @@ function create_release() { --tarball="${build_path}/artifacts/${release_file}" } +function create_mtar() { + set -e + mkdir -p "${build_path}/artifacts" + local version=$1 + local build_path=$2 + echo " - creating autorscaler mtar artifact" + pushd "${autoscaler_dir}" > /dev/null + make mta-release VERSION="${version}" DEST="${build_path}/artifacts/" + popd > /dev/null +} + function create_tests() { set -e mkdir -p "${build_path}/artifacts" @@ -136,19 +134,24 @@ pushd "${autoscaler_dir}" > /dev/null if [ "${PERFORM_BOSH_RELEASE}" == "true" ]; then RELEASE_TGZ="app-autoscaler-v${VERSION}.tgz" ACCEPTANCE_TEST_TGZ="app-autoscaler-acceptance-tests-v${VERSION}.tgz" + AUTOSCALER_MTAR="app-autoscaler-release-v${VERSION}.mtar" create_release "${VERSION}" "${build_path}" "${RELEASE_TGZ}" create_tests "${VERSION}" "${build_path}" + create_mtar "${VERSION}" "${build_path}" [ "${CI}" = "true" ] && commit_release sha256sum "${build_path}/artifacts/"* > "${build_path}/artifacts/files.sum.sha256" ACCEPTANCE_SHA256=$( grep "${ACCEPTANCE_TEST_TGZ}$" "${SUM_FILE}" | awk '{print $1}' ) RELEASE_SHA256=$( grep "${RELEASE_TGZ}$" "${SUM_FILE}" | awk '{print $1}') + MTAR_SHA256=$( grep "${AUTOSCALER_MTAR}$" "${SUM_FILE}" | awk '{print $1}') else ACCEPTANCE_SHA256="dummy-sha" RELEASE_SHA256="dummy-sha" + MTAR_SHA256="dummy-sha" fi export ACCEPTANCE_SHA256 export RELEASE_SHA256 + export MTAR_SHA256 cat >> "${build_path}/changelog.md" < /dev/null && pwd ) + +DEPLOYMENT=foo +DEBUG=true +DEST="${script_dir}/../build" +BUILD_OPTS="--force" +AUTOSCALER_CI_BOT_NAME="foo" +AUTOSCALER_CI_BOT_EMAIL="foo@bar.baz" +PREV_VERSION="$(yq ".properties.\"autoscaler.apiserver.info.build\".default" jobs/golangapiserver/spec)" + +VERSION="$(cat "${script_dir}/../VERSION")-pre" + +export DEPLOYMENT +export DEBUG +export DEST +export BUILD_OPTS +export AUTOSCALER_CI_BOT_NAME +export AUTOSCALER_CI_BOT_EMAIL +export PREV_VERSION +export VERSION + +# check for GITHUB_TOKEN +if [ -z "${GITHUB_TOKEN}" ]; then + echo "GITHUB_TOKEN is not set" + exit 1 +fi + +find_or_create_ssh_key() { + if [ -f ~/.ssh/id_ed25519 ]; then + echo "ssh key already exists" + return + fi + + ssh-keygen -t ed25519 -C "${AUTOSCALER_CI_BOT_EMAIL}" -f ~/.ssh/id_ed25519 -N "" +} + +prerelease() { + pushd "${script_dir}/.." > /dev/null + make clean generate-fakes generate-openapi-generated-clients-and-servers go-mod-tidy go-mod-vendor db scheduler + popd > /dev/null +} + +delete_dev_releases() { + rm -rf dev_releases +} + + +release_autoscaler() { + AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC=$(cat ~/.ssh/id_ed25519.pub) + AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE=$(cat ~/.ssh/id_ed25519) + export AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC + export AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE + + source "${script_dir}/../ci/autoscaler/scripts/release-autoscaler.sh" + echo "beware that it adds a commit you need to drop each time also you need to remove dev_releases from root." +} + +main() { + find_or_create_ssh_key + delete_dev_releases + prerelease + release_autoscaler +} + +main + diff --git a/src/acceptance/go.mod b/src/acceptance/go.mod index f1595f281a..67a82f1fed 100644 --- a/src/acceptance/go.mod +++ b/src/acceptance/go.mod @@ -6,18 +6,16 @@ require ( github.com/cloudfoundry/cf-test-helpers/v2 v2.9.0 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 - github.com/stretchr/testify v1.9.0 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d // indirect github.com/kr/pretty v0.3.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.8.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect diff --git a/src/acceptance/helpers/helpers_test.go b/src/acceptance/helpers/helpers_test.go deleted file mode 100644 index f428822df3..0000000000 --- a/src/acceptance/helpers/helpers_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package helpers_test - -import ( - "acceptance/config" - "acceptance/helpers" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestServicePlans_urlIsCorrect(t *testing.T) { - url := helpers.ServicePlansUrl(&config.Config{ServiceName: "autoscaler", ServiceBroker: "autoscaler"}, "GUID_UUID") - assert.Equal(t, url, "/v3/service_plans?available=true&fields%5Bservice_offering.service_broker%5D=name%2Cguid&include=service_offering&per_page=5000&service_broker_names=autoscaler&service_offering_names=autoscaler&space_guids=GUID_UUID") -} diff --git a/src/autoscaler/.gitignore b/src/autoscaler/.gitignore index 9f56ece60a..d3023bec01 100644 --- a/src/autoscaler/.gitignore +++ b/src/autoscaler/.gitignore @@ -1,3 +1,4 @@ build/ fakes/ mta_archives/ +mta.yaml diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index 79c59bb7ff..4a8ed8c098 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -3,6 +3,8 @@ SHELL := /bin/bash MAKEFLAGS := -s aes_terminal_font_yellow := \e[38;2;255;255;0m aes_terminal_reset := \e[0m +VERSION ?= 0.0.0-rc.1 +DEST ?= build GO_VERSION = $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') GO_DEPENDENCIES = $(shell find . -type f -name '*.go') @@ -29,6 +31,7 @@ GINKGO_OPTS = -r --race --require-suite --randomize-all --cover ${OPTS} GINKGO_VERSION = v$(shell cat ../../.tool-versions | grep ginkgo | cut --delimiter=' ' --fields='2') + # ogen generated OpenAPI clients and servers openapi-generated-clients-and-servers-dir := ./helpers/apis/scalinghistory openapi-spec-path := ../../api @@ -149,7 +152,8 @@ clean: mta-deploy: mta-build build-extension-file $(MAKE) -f metricsforwarder/Makefile set-security-group @echo "Deploying with extension file: $(EXTENSION_FILE)" - @cf deploy mta_archives/*.mtar -f --delete-services -e $(EXTENSION_FILE) + @cf deploy $(MAKEFILE_DIR)/$(DEST)/*.mtar -f --delete-services -e $(EXTENSION_FILE) + build-extension-file: echo "extension file at: $(EXTENSION_FILE)" @@ -162,7 +166,12 @@ mta-logs: .PHONY: mta-build mta-build: mta-build-clean + @echo "bulding mtar file for version: $(VERSION)" + cp mta.tpl.yaml mta.yaml + sed -i 's/VERSION/$(VERSION)/g' mta.yaml + mkdir -p $(DEST) mbt build + @mv mta_archives/com.github.cloudfoundry.app-autoscaler-release_$(VERSION).mtar $(DEST)/app-autoscaler-release-v$(VERSION).mtar mta-build-clean: rm -rf mta_archives diff --git a/src/autoscaler/build-extension-file.sh b/src/autoscaler/build-extension-file.sh index a35b1805a3..a1adb01176 100755 --- a/src/autoscaler/build-extension-file.sh +++ b/src/autoscaler/build-extension-file.sh @@ -39,6 +39,10 @@ _schema-version: 3.3.0 modules: - name: metricsforwarder + requires: + - name: config + - name: policydb + - name: syslog-client parameters: routes: - route: ${METRICSFORWARDER_APPNAME}.\${default-domain} diff --git a/src/autoscaler/helpers/json_redacter_with_url_creds.go b/src/autoscaler/helpers/json_redacter_with_url_creds.go index 1d3c764f6e..f787561577 100644 --- a/src/autoscaler/helpers/json_redacter_with_url_creds.go +++ b/src/autoscaler/helpers/json_redacter_with_url_creds.go @@ -40,13 +40,13 @@ func (r JSONRedacterWithURLCred) Redact(data []byte) []byte { } err := json.Unmarshal(data, &jsonBlob) if err != nil { - return handleError(err) + return errorToBytes(err) } r.redactValue(&jsonBlob) data, err = json.Marshal(jsonBlob) if err != nil { - return handleError(err) + return errorToBytes(err) } return r.jsonRedacter.Redact(data) @@ -82,7 +82,7 @@ func (r JSONRedacterWithURLCred) redactObject(data *map[string]interface{}) { } } -func handleError(err error) []byte { +func errorToBytes(err error) []byte { var content []byte var errType *json.UnsupportedTypeError if errors.As(err, &errType) { diff --git a/src/autoscaler/helpers/logger.go b/src/autoscaler/helpers/logger.go index 8a95e98dcd..f6531cfff5 100644 --- a/src/autoscaler/helpers/logger.go +++ b/src/autoscaler/helpers/logger.go @@ -2,8 +2,10 @@ package helpers import ( "fmt" + "log/slog" "os" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" "code.cloudfoundry.org/lager/v3" ) @@ -12,26 +14,27 @@ type LoggingConfig struct { } func InitLoggerFromConfig(conf *LoggingConfig, name string) lager.Logger { - logLevel, err := getLogLevel(conf.Level) + logLevel, err := parseLogLevel(conf.Level) if err != nil { - fmt.Fprintf(os.Stderr, "failed to initialize logger: %s\n", err.Error()) - os.Exit(1) + handleError("failed to initialize logger", err) } + logger := lager.NewLogger(name) - keyPatterns := []string{"[Pp]wd", "[Pp]ass", "[Ss]ecret", "[Tt]oken"} + vcapConfig, _ := configutil.NewVCAPConfigurationReader() - redactedSink, err := NewRedactingWriterWithURLCredSink(os.Stdout, logLevel, keyPatterns, nil) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create redacted sink: %s\n", err.Error()) - os.Exit(1) + if vcapConfig.IsRunningOnCF() { + plaintextFormatSink := createPlaintextSink() + logger.RegisterSink(plaintextFormatSink) + } else { + redactedSink := createRedactedSink(logLevel) + logger.RegisterSink(redactedSink) } - logger.RegisterSink(redactedSink) return logger } -func getLogLevel(level string) (lager.LogLevel, error) { +func parseLogLevel(level string) (lager.LogLevel, error) { switch level { case "debug": return lager.DEBUG, nil @@ -42,6 +45,25 @@ func getLogLevel(level string) (lager.LogLevel, error) { case "fatal": return lager.FATAL, nil default: - return -1, fmt.Errorf("Error: unsupported log level:%s", level) + return -1, fmt.Errorf("unsupported log level: %s", level) + } +} + +func createPlaintextSink() lager.Sink { + slogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + return lager.NewSlogSink(slogger) +} + +func createRedactedSink(logLevel lager.LogLevel) lager.Sink { + keyPatterns := []string{"[Pp]wd", "[Pp]ass", "[Ss]ecret", "[Tt]oken"} + redactedSink, err := NewRedactingWriterWithURLCredSink(os.Stdout, logLevel, keyPatterns, nil) + if err != nil { + handleError("failed to create redacted sink", err) } + return redactedSink +} + +func handleError(message string, err error) { + fmt.Fprintf(os.Stderr, "%s: %s\n", message, err.Error()) + os.Exit(1) } diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 743ff79b61..6536d0b95e 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -3,7 +3,9 @@ package config import ( "errors" "fmt" + "net/url" "os" + "strings" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" @@ -14,15 +16,10 @@ import ( "gopkg.in/yaml.v3" ) -// There are 3 type of errors that this package can return: -// - ErrReadYaml -// - ErrReadEnvironment -// - ErrReadVCAPEnvironment - var ( ErrReadYaml = errors.New("failed to read config file") ErrReadJson = errors.New("failed to read vcap_services json") - ErrMetricsforwarderConfigNotFound = errors.New("Configuration error: metricsforwarder config service not found") + ErrMetricsforwarderConfigNotFound = errors.New("metricsforwarder config service not found") ) const ( @@ -34,6 +31,16 @@ const ( DefaultValidDuration = 1 * time.Second ) +type LoggregatorConfig struct { + MetronAddress string `yaml:"metron_address"` + TLS models.TLSCerts `yaml:"tls"` +} +type SyslogConfig struct { + ServerAddress string `yaml:"server_address"` + Port int `yaml:"port"` + TLS models.TLSCerts `yaml:"tls"` +} + type Config struct { Logging helpers.LoggingConfig `yaml:"logging"` Server helpers.ServerConfig `yaml:"server"` @@ -49,66 +56,28 @@ type Config struct { StoredProcedureConfig *models.StoredProcedureConfig `yaml:"stored_procedure_binding_credential_config"` } -var defaultServerConfig = helpers.ServerConfig{ - Port: 6110, -} - -var defaultHealthConfig = helpers.HealthConfig{ - ServerConfig: helpers.ServerConfig{ - Port: 8081, - }, -} - -var defaultLoggingConfig = helpers.LoggingConfig{ - Level: "info", -} - -type LoggingConfig struct { - Level string `yaml:"level"` -} - -type LoggregatorConfig struct { - MetronAddress string `yaml:"metron_address"` - TLS models.TLSCerts `yaml:"tls"` -} - -type SyslogConfig struct { - ServerAddress string `yaml:"server_address"` - Port int `yaml:"port"` - TLS models.TLSCerts `yaml:"tls"` -} - -func decodeYamlFile(filepath string, c *Config) error { - r, err := os.Open(filepath) +func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) (*Config, error) { + conf := defaultConfig() - if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "failed to open config file '%s' : %s\n", filepath, err.Error()) - return err + if err := loadYamlFile(filepath, &conf); err != nil { + return nil, err } - dec := yaml.NewDecoder(r) - dec.KnownFields(true) - err = dec.Decode(c) - - if err != nil { - return fmt.Errorf("%w: %w", ErrReadYaml, err) + if err := loadVcapConfig(&conf, vcapReader); err != nil { + return nil, err } - defer r.Close() - return nil + return &conf, nil } -func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) (*Config, error) { - var conf Config - var err error - - conf = Config{ - Server: defaultServerConfig, - Logging: defaultLoggingConfig, +func defaultConfig() Config { + return Config{ + Server: helpers.ServerConfig{Port: 6110}, + Logging: helpers.LoggingConfig{Level: "info"}, LoggregatorConfig: LoggregatorConfig{ MetronAddress: DefaultMetronAddress, }, - Health: defaultHealthConfig, + Health: helpers.HealthConfig{ServerConfig: helpers.ServerConfig{Port: 8081}}, CacheTTL: DefaultCacheTTL, CacheCleanupInterval: DefaultCacheCleanupInterval, PolicyPollerInterval: DefaultPolicyPollerInterval, @@ -117,106 +86,194 @@ func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) ValidDuration: DefaultValidDuration, }, } +} - if filepath != "" { - err = decodeYamlFile(filepath, &conf) - if err != nil { - return nil, err - } +func loadYamlFile(filepath string, conf *Config) error { + if filepath == "" { + return nil + } + file, err := os.Open(filepath) + if err != nil { + fmt.Fprintf(os.Stdout, "failed to open config file '%s': %s\n", filepath, err) + return ErrReadYaml } + defer file.Close() - if vcapReader.IsRunningOnCF() { - conf.Server.Port = vcapReader.GetPort() + dec := yaml.NewDecoder(file) + dec.KnownFields(true) + if err := dec.Decode(conf); err != nil { + return fmt.Errorf("%w: %v", ErrReadYaml, err) + } + return nil +} - data, err := vcapReader.GetServiceCredentialContent("config", "metricsforwarder") - if err != nil { - return &conf, fmt.Errorf("%w: %w", ErrMetricsforwarderConfigNotFound, err) - } +func loadVcapConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + if !vcapReader.IsRunningOnCF() { + return nil + } - err = yaml.Unmarshal(data, &conf) - if err != nil { - return &conf, fmt.Errorf("%w: %w", ErrReadJson, err) - } + conf.Server.Port = vcapReader.GetPort() + if err := loadMetricsforwarderConfig(conf, vcapReader); err != nil { + return err + } - if conf.Db == nil { - conf.Db = make(map[string]db.DatabaseConfig) - } + if conf.Db == nil { + conf.Db = make(map[string]db.DatabaseConfig) + } - currentPolicyDb, ok := conf.Db[db.PolicyDb] - if !ok { - conf.Db[db.PolicyDb] = db.DatabaseConfig{} - } + if err := configurePolicyDb(conf, vcapReader); err != nil { + return err + } - currentPolicyDb.URL, err = vcapReader.MaterializeDBFromService(db.PolicyDb) - if err != nil { - return &conf, err - } - conf.Db[db.PolicyDb] = currentPolicyDb - - if conf.CredHelperImpl == "stored_procedure" { - currentStoredProcedureDb, ok := conf.Db[db.StoredProcedureDb] - if !ok { - conf.Db[db.StoredProcedureDb] = db.DatabaseConfig{} - } - currentStoredProcedureDb.URL, err = vcapReader.MaterializeDBFromService(db.StoredProcedureDb) - if err != nil { - return &conf, err - } - conf.Db[db.StoredProcedureDb] = currentStoredProcedureDb + if conf.CredHelperImpl == "stored_procedure" { + if err := configureStoredProcedureDb(conf, vcapReader); err != nil { + return err } + } + + if err := configureSyslogTLS(conf, vcapReader); err != nil { + return err + } + + return nil +} + +func loadMetricsforwarderConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + data, err := vcapReader.GetServiceCredentialContent("config", "metricsforwarder") + if err != nil { + return fmt.Errorf("%w: %v", ErrMetricsforwarderConfigNotFound, err) + } + return yaml.Unmarshal(data, conf) +} + +func configurePolicyDb(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + currentPolicyDb, ok := conf.Db[db.PolicyDb] + if !ok { + conf.Db[db.PolicyDb] = db.DatabaseConfig{} + } + + dbURL, err := vcapReader.MaterializeDBFromService(db.PolicyDb) + currentPolicyDb.URL = dbURL + if err != nil { + return err + } + conf.Db[db.PolicyDb] = currentPolicyDb + return nil +} - conf.SyslogConfig.TLS, err = vcapReader.MaterializeTLSConfigFromService("syslog-client") - if err != nil { - return &conf, err +func configureStoredProcedureDb(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + currentStoredProcedureDb, exists := conf.Db[db.StoredProcedureDb] + if !exists { + conf.Db[db.StoredProcedureDb] = db.DatabaseConfig{} + } + + dbURL, err := vcapReader.MaterializeDBFromService(db.StoredProcedureDb) + if err != nil { + return err + } + + currentStoredProcedureDb.URL = dbURL + parsedUrl, err := url.Parse(currentStoredProcedureDb.URL) + if err != nil { + return err + } + + if conf.StoredProcedureConfig != nil { + if conf.StoredProcedureConfig.Username != "" { + currentStoredProcedureDb.URL = strings.Replace(currentStoredProcedureDb.URL, parsedUrl.User.Username(), conf.StoredProcedureConfig.Username, 1) + } + if conf.StoredProcedureConfig.Password != "" { + bindingPassword, _ := parsedUrl.User.Password() + currentStoredProcedureDb.URL = strings.Replace(currentStoredProcedureDb.URL, bindingPassword, conf.StoredProcedureConfig.Password, 1) } } + conf.Db[db.StoredProcedureDb] = currentStoredProcedureDb - return &conf, nil + return nil } -func (c *Config) UsingSyslog() bool { - return c.SyslogConfig.ServerAddress != "" && c.SyslogConfig.Port != 0 +func configureSyslogTLS(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + tls, err := vcapReader.MaterializeTLSConfigFromService("syslog-client") + if err != nil { + return err + } + conf.SyslogConfig.TLS = tls + return nil } func (c *Config) Validate() error { + if err := c.validateDbConfig(); err != nil { + return err + } + if err := c.validateSyslogOrLoggregator(); err != nil { + return err + } + if err := c.validateRateLimit(); err != nil { + return err + } + if err := c.validateCredHelperImpl(); err != nil { + return err + } + return c.Health.Validate() +} + +func (c *Config) validateDbConfig() error { if c.Db[db.PolicyDb].URL == "" { - return fmt.Errorf("Configuration error: Policy DB url is empty") + return errors.New("Policy DB url is empty") } + return nil +} + +func (c *Config) validateSyslogOrLoggregator() error { if c.UsingSyslog() { - if c.SyslogConfig.TLS.CACertFile == "" { - return fmt.Errorf("Configuration error: SyslogServer Loggregator CACert is empty") - } - if c.SyslogConfig.TLS.CertFile == "" { - return fmt.Errorf("Configuration error: SyslogServer ClientCert is empty") - } - if c.SyslogConfig.TLS.KeyFile == "" { - return fmt.Errorf("Configuration error: SyslogServer ClientKey is empty") - } - } else { - if c.LoggregatorConfig.TLS.CACertFile == "" { - return fmt.Errorf("Configuration error: Loggregator CACert is empty") - } - if c.LoggregatorConfig.TLS.CertFile == "" { - return fmt.Errorf("Configuration error: Loggregator ClientCert is empty") - } - if c.LoggregatorConfig.TLS.KeyFile == "" { - return fmt.Errorf("Configuration error: Loggregator ClientKey is empty") - } + return c.validateSyslogConfig() } + return c.validateLoggregatorConfig() +} - if c.RateLimit.MaxAmount <= 0 { - return fmt.Errorf("Configuration error: RateLimit.MaxAmount is equal or less than zero") +func (c *Config) validateSyslogConfig() error { + if c.SyslogConfig.TLS.CACertFile == "" { + return errors.New("SyslogServer Loggregator CACert is empty") } - if c.RateLimit.ValidDuration <= 0*time.Nanosecond { - return fmt.Errorf("Configuration error: RateLimit.ValidDuration is equal or less than zero nanosecond") + if c.SyslogConfig.TLS.CertFile == "" { + return errors.New("SyslogServer ClientCert is empty") } - if c.CredHelperImpl == "" { - return fmt.Errorf("Configuration error: CredHelperImpl is empty") + if c.SyslogConfig.TLS.KeyFile == "" { + return errors.New("SyslogServer ClientKey is empty") + } + return nil +} + +func (c *Config) validateLoggregatorConfig() error { + if c.LoggregatorConfig.TLS.CACertFile == "" { + return errors.New("Loggregator CACert is empty") + } + if c.LoggregatorConfig.TLS.CertFile == "" { + return errors.New("Loggregator ClientCert is empty") } + if c.LoggregatorConfig.TLS.KeyFile == "" { + return errors.New("Loggregator ClientKey is empty") + } + return nil +} - if err := c.Health.Validate(); err != nil { - return err +func (c *Config) validateRateLimit() error { + if c.RateLimit.MaxAmount <= 0 { + return errors.New("RateLimit.MaxAmount is less than or equal to zero") + } + if c.RateLimit.ValidDuration <= 0 { + return errors.New("RateLimit.ValidDuration is less than or equal to zero") } + return nil +} +func (c *Config) validateCredHelperImpl() error { + if c.CredHelperImpl == "" { + return errors.New("CredHelperImpl is empty") + } return nil } + +func (c *Config) UsingSyslog() bool { + return c.SyslogConfig.ServerAddress != "" && c.SyslogConfig.Port != 0 +} diff --git a/src/autoscaler/metricsforwarder/config/config_test.go b/src/autoscaler/metricsforwarder/config/config_test.go index 01c00332a9..f61b8d9d82 100644 --- a/src/autoscaler/metricsforwarder/config/config_test.go +++ b/src/autoscaler/metricsforwarder/config/config_test.go @@ -63,12 +63,12 @@ var _ = Describe("Config", func() { When("service is empty", func() { var expectedErr error BeforeEach(func() { - expectedErr = fmt.Errorf("Configuration error: metricsforwarder config service not found") + expectedErr = fmt.Errorf("metricsforwarder config service not found") mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(""), expectedErr) }) It("should error with config service not found", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: metricsforwarder config service not found"))) + Expect(err).To(MatchError(MatchRegexp("metricsforwarder config service not found"))) }) }) @@ -93,7 +93,7 @@ var _ = Describe("Config", func() { When("VCAP_SERVICES has relational db service bind to app for policy db", func() { BeforeEach(func() { - mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("default"), nil) + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(`{ "cred_helper_impl": "default" }`), nil) // #nosec G101 expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 }) @@ -108,7 +108,7 @@ var _ = Describe("Config", func() { When("storedProcedure_db service is provided and cred_helper_impl is stored_procedure", func() { BeforeEach(func() { - mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("stored_procedure"), nil) + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(`{ "cred_helper_impl": "stored_procedure" }`), nil) // #nosec G101 expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 }) @@ -121,11 +121,39 @@ var _ = Describe("Config", func() { actualDbName := mockVCAPConfigurationReader.MaterializeDBFromServiceArgsForCall(1) Expect(actualDbName).To(Equal(db.StoredProcedureDb)) }) + + When("storedProcedure_db config has username and password", func() { + var storedProcedureUsername, storedProcedurePassword string + + BeforeEach(func() { + storedProcedureUsername = "storedProcedureUsername" + storedProcedurePassword = "storedProcedurePassword" + + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte( + `{ "cred_helper_impl": "stored_procedure", + "stored_procedure_binding_credential_config": { + "username": "`+storedProcedureUsername+`", + "password": "`+storedProcedurePassword+`" + }, + }`), + nil, + ) // #nosec G101 + }) + + It("should prioritize the username and password from the config", func() { + // url should include the username and password from the config + Expect(err).NotTo(HaveOccurred()) + _, storeProcedureFound := conf.Db[db.StoredProcedureDb] + Expect(storeProcedureFound).To(BeTrue()) + Expect(conf.Db[db.StoredProcedureDb].URL).To(ContainSubstring(fmt.Sprintf("%s:%s", storedProcedureUsername, storedProcedurePassword))) + }) + }) }) When("storedProcedure_db service is provided and cred_helper_impl is default", func() { BeforeEach(func() { - mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("default"), nil) + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte( + `{ "cred_helper_impl": "default" }`), nil) // #nosec G101 expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 }) @@ -138,7 +166,6 @@ var _ = Describe("Config", func() { When("VCAP_SERVICES has metricsforwarder config", func() { BeforeEach(func() { - mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(` { "cache_cleanup_interval":"10h", "cache_ttl":"90s", @@ -321,7 +348,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer Loggregator CACert is empty"))) + Expect(err).To(MatchError(MatchRegexp("SyslogServer Loggregator CACert is empty"))) }) }) @@ -331,7 +358,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer ClientKey is empty"))) + Expect(err).To(MatchError(MatchRegexp("SyslogServer ClientKey is empty"))) }) }) @@ -341,7 +368,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer ClientCert is empty"))) + Expect(err).To(MatchError(MatchRegexp("SyslogServer ClientCert is empty"))) }) }) }) @@ -358,7 +385,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: Policy DB url is empty"))) + Expect(err).To(MatchError(MatchRegexp("Policy DB url is empty"))) }) }) @@ -368,7 +395,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: Loggregator CACert is empty"))) + Expect(err).To(MatchError(MatchRegexp("Loggregator CACert is empty"))) }) }) @@ -378,7 +405,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: Loggregator ClientCert is empty"))) + Expect(err).To(MatchError(MatchRegexp("Loggregator ClientCert is empty"))) }) }) @@ -388,7 +415,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: Loggregator ClientKey is empty"))) + Expect(err).To(MatchError(MatchRegexp("Loggregator ClientKey is empty"))) }) }) @@ -398,8 +425,7 @@ health: }) It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.MaxAmount is equal or less than zero"))) - + Expect(err).To(MatchError(MatchRegexp("RateLimit.MaxAmount is less than or equal to zero"))) }) }) @@ -409,12 +435,8 @@ health: }) It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.ValidDuration is equal or less than zero nanosecond"))) + Expect(err).To(MatchError(MatchRegexp("RateLimit.ValidDuration is less than or equal to zero"))) }) }) }) }) - -func getVcapConfigWithCredImplementation(credHelperImplementation string) []byte { - return []byte(`{ "cred_helper_impl": "` + credHelperImplementation + `" }`) // #nosec G101 -} diff --git a/src/autoscaler/models/stored_procedure.go b/src/autoscaler/models/stored_procedure.go index 0693105250..864120c2d9 100644 --- a/src/autoscaler/models/stored_procedure.go +++ b/src/autoscaler/models/stored_procedure.go @@ -6,4 +6,6 @@ type StoredProcedureConfig struct { DropBindingCredentialProcedureName string `yaml:"drop_binding_credential_procedure_name"` DropAllBindingCredentialProcedureName string `yaml:"drop_all_binding_credential_procedure_name"` ValidateBindingCredentialProcedureName string `yaml:"validate_binding_credential_procedure_name"` + Username string `yaml:"username"` + Password string `yaml:"password"` } diff --git a/src/autoscaler/mta.yaml b/src/autoscaler/mta.tpl.yaml similarity index 82% rename from src/autoscaler/mta.yaml rename to src/autoscaler/mta.tpl.yaml index 4f9b597ea2..0c5f5b5122 100644 --- a/src/autoscaler/mta.yaml +++ b/src/autoscaler/mta.tpl.yaml @@ -3,7 +3,7 @@ description: Application Autoscaler Release for Cloud Foundry _schema-version: "3.3.0" provider: Cloud Foundry Foundation copyright: Apache License 2.0 -version: 0.0.1 +version: VERSION modules: - name: metricsforwarder @@ -15,6 +15,7 @@ modules: - name: config - name: policydb - name: syslog-client + - name: app-autoscaler-application-logs parameters: memory: 1G disk-quota: 1G @@ -44,4 +45,9 @@ resources: parameters: service-tags: - syslog-client - +- name: app-autoscaler-application-logs + active: false + type: org.cloudfoundry.managed-service + parameters: + service: application-logs + service-plan: standard