From c3a2064297d501d6621e56dfd48a98aa48f8058c Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 2 Sep 2024 16:21:52 +0200 Subject: [PATCH 01/19] WIP --- Makefile | 23 +++++++++++------- src/autoscaler/.gitignore | 1 + src/autoscaler/Makefile | 7 ++++++ src/autoscaler/mta.tpl.yaml | 47 +++++++++++++++++++++++++++++++++++++ src/autoscaler/mta.yaml | 2 +- 5 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 src/autoscaler/mta.tpl.yaml diff --git a/Makefile b/Makefile index 9ef065ae77..61598ce5c0 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ clean-acceptance: build: $(all_modules) build-tests: build-test build-test: $(addprefix test_,$(go_modules)) -build-all: build build-test build-test-app ## Build all modules and tests +build-all: build build-test build-test-app mta-build ## Build all modules and tests db: target/db target/db: @echo "# building $@" @@ -260,13 +260,6 @@ build/autoscaler-test.tgz: @mkdir -p build @bosh create-release --force --timestamp-version --tarball=build/autoscaler-test.tgz -.PHONY: acceptance-release -acceptance-release: clean-acceptance go-mod-tidy go-mod-vendor build-test-app - @echo " - building acceptance test release '${VERSION}' to dir: '${DEST}' " - @mkdir -p ${DEST} - ${AUTOSCALER_DIR}/scripts/compile-acceptance-tests.sh - @tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance' - .PHONY: generate-fakes autoscaler.generate-fakes test-app.generate-fakes generate-fakes: autoscaler.generate-fakes test-app.generate-fakes autoscaler.generate-fakes: @@ -360,7 +353,21 @@ deploy-prometheus: ${prometheus-bosh-release-path}/manifests ${CI_DIR}/infrastructure/scripts/deploy-prometheus.sh; +.PHONY: mta-release +mta-release: mta-build + @echo " - building mtar release '${VERSION}' to dir: '${DEST}' " + +.PHONY: acceptance-release +acceptance-release: clean-acceptance go-mod-tidy go-mod-vendor build-test-app + @echo " - building acceptance test release '${VERSION}' to dir: '${DEST}' " + @mkdir -p ${DEST} + ${AUTOSCALER_DIR}/scripts/compile-acceptance-tests.sh + @tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance' +.PHONY: mta-build +mta-build: + @echo " - building mta" + @make --directory='./src/autoscaler' mta-build .PHONY: build-test-app build-test-app: diff --git a/src/autoscaler/.gitignore b/src/autoscaler/.gitignore index 9f56ece60a..37f4a3c002 100644 --- a/src/autoscaler/.gitignore +++ b/src/autoscaler/.gitignore @@ -1,3 +1,4 @@ build/ +mta.yaml fakes/ mta_archives/ diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index 79c59bb7ff..5907102341 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -3,6 +3,8 @@ SHELL := /bin/bash MAKEFLAGS := -s aes_terminal_font_yellow := \e[38;2;255;255;0m aes_terminal_reset := \e[0m +VERSION ?= 0.0.0-rc.1 +DEST ?= build GO_VERSION = $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') GO_DEPENDENCIES = $(shell find . -type f -name '*.go') @@ -29,6 +31,7 @@ GINKGO_OPTS = -r --race --require-suite --randomize-all --cover ${OPTS} GINKGO_VERSION = v$(shell cat ../../.tool-versions | grep ginkgo | cut --delimiter=' ' --fields='2') + # ogen generated OpenAPI clients and servers openapi-generated-clients-and-servers-dir := ./helpers/apis/scalinghistory openapi-spec-path := ../../api @@ -162,7 +165,11 @@ mta-logs: .PHONY: mta-build mta-build: mta-build-clean + cp mta.tpl.yaml mta.yaml + sed -i 's/VERSION/$(VERSION)/g' mta.yaml + mkdir -p $(DEST) mbt build + @mv mta_archives/com.github.cloudfoundry.app-autoscaler-release_$(VERSION).mtar $(DEST)/app-autoscaler-release-v$(VERSION).mtar mta-build-clean: rm -rf mta_archives diff --git a/src/autoscaler/mta.tpl.yaml b/src/autoscaler/mta.tpl.yaml new file mode 100644 index 0000000000..5e0cd1b48f --- /dev/null +++ b/src/autoscaler/mta.tpl.yaml @@ -0,0 +1,47 @@ +ID: com.github.cloudfoundry.app-autoscaler-release +description: Application Autoscaler Release for Cloud Foundry +_schema-version: "3.3.0" +provider: Cloud Foundry Foundation +copyright: Apache License 2.0 +version: VERSION + +modules: + - name: metricsforwarder + type: go + path: . + properties: + GO_INSTALL_PACKAGE_SPEC: code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/cmd/metricsforwarder + requires: + - name: config + - name: policydb + - name: syslog-client + parameters: + memory: 1G + disk-quota: 1G + instances: 2 + stack: cflinuxfs4 + routes: + build-parameters: + builder: custom + commands: + - make vendor + +resources: +- name: config + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - config + path: metricsforwarder/default_config.json +- name: policydb + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - policy_db + - relational +- name: syslog-client + type: org.cloudfoundry.user-provided-service + parameters: + service-tags: + - syslog-client + diff --git a/src/autoscaler/mta.yaml b/src/autoscaler/mta.yaml index 4f9b597ea2..a7c82483d6 100644 --- a/src/autoscaler/mta.yaml +++ b/src/autoscaler/mta.yaml @@ -3,7 +3,7 @@ description: Application Autoscaler Release for Cloud Foundry _schema-version: "3.3.0" provider: Cloud Foundry Foundation copyright: Apache License 2.0 -version: 0.0.1 +version: 0.0.0-rc.1 modules: - name: metricsforwarder From cf31fff623c6ed972dcf4900446aa1c372e5fb70 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 12 Sep 2024 16:00:11 +0200 Subject: [PATCH 02/19] Adds optional logs service --- src/autoscaler/.gitignore | 1 + src/autoscaler/mta.tpl.yaml | 11 ++++++++- src/autoscaler/mta.yaml | 47 ------------------------------------- 3 files changed, 11 insertions(+), 48 deletions(-) delete mode 100644 src/autoscaler/mta.yaml diff --git a/src/autoscaler/.gitignore b/src/autoscaler/.gitignore index 37f4a3c002..9951afd805 100644 --- a/src/autoscaler/.gitignore +++ b/src/autoscaler/.gitignore @@ -2,3 +2,4 @@ build/ mta.yaml fakes/ mta_archives/ +mta.yaml diff --git a/src/autoscaler/mta.tpl.yaml b/src/autoscaler/mta.tpl.yaml index 5e0cd1b48f..7dba14ba0b 100644 --- a/src/autoscaler/mta.tpl.yaml +++ b/src/autoscaler/mta.tpl.yaml @@ -15,6 +15,7 @@ modules: - name: config - name: policydb - name: syslog-client + - name: app-autoscaler-application-logs parameters: memory: 1G disk-quota: 1G @@ -44,4 +45,12 @@ resources: parameters: service-tags: - syslog-client - +- name: app-autoscaler-application-logs + active: false + type: application-logs + parameters: + service-broker: null + service-plan: standard + parameters-metadata: + service-broker: + optional: false diff --git a/src/autoscaler/mta.yaml b/src/autoscaler/mta.yaml deleted file mode 100644 index a7c82483d6..0000000000 --- a/src/autoscaler/mta.yaml +++ /dev/null @@ -1,47 +0,0 @@ -ID: com.github.cloudfoundry.app-autoscaler-release -description: Application Autoscaler Release for Cloud Foundry -_schema-version: "3.3.0" -provider: Cloud Foundry Foundation -copyright: Apache License 2.0 -version: 0.0.0-rc.1 - -modules: - - name: metricsforwarder - type: go - path: . - properties: - GO_INSTALL_PACKAGE_SPEC: code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/cmd/metricsforwarder - requires: - - name: config - - name: policydb - - name: syslog-client - parameters: - memory: 1G - disk-quota: 1G - instances: 2 - stack: cflinuxfs4 - routes: - build-parameters: - builder: custom - commands: - - make vendor - -resources: -- name: config - type: org.cloudfoundry.user-provided-service - parameters: - service-tags: - - config - path: metricsforwarder/default_config.json -- name: policydb - type: org.cloudfoundry.user-provided-service - parameters: - service-tags: - - policy_db - - relational -- name: syslog-client - type: org.cloudfoundry.user-provided-service - parameters: - service-tags: - - syslog-client - From 0fc1a17231fd9172990b13b2842345ec1e98031e Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 16 Sep 2024 14:55:32 +0200 Subject: [PATCH 03/19] Update Makefile to use MAKEFILE_DIR for mtar deployment path --- src/autoscaler/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index 5907102341..cd920e182a 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -152,7 +152,8 @@ clean: mta-deploy: mta-build build-extension-file $(MAKE) -f metricsforwarder/Makefile set-security-group @echo "Deploying with extension file: $(EXTENSION_FILE)" - @cf deploy mta_archives/*.mtar -f --delete-services -e $(EXTENSION_FILE) + @cf deploy $(MAKEFILE_DIR)/mta_archives/*.mtar -f --delete-services -e $(EXTENSION_FILE) + build-extension-file: echo "extension file at: $(EXTENSION_FILE)" From 4cc1a1d41d1dfdd570a406c00c90fe835b83cdd1 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 17 Sep 2024 11:32:24 +0200 Subject: [PATCH 04/19] Adds storeprocedure db to mtar --- src/autoscaler/Makefile | 1 + src/autoscaler/mta.tpl.yaml | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index cd920e182a..3318b9cb44 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -166,6 +166,7 @@ mta-logs: .PHONY: mta-build mta-build: mta-build-clean + @echo "bulding mtar file for version: $(VERSION)" cp mta.tpl.yaml mta.yaml sed -i 's/VERSION/$(VERSION)/g' mta.yaml mkdir -p $(DEST) diff --git a/src/autoscaler/mta.tpl.yaml b/src/autoscaler/mta.tpl.yaml index 7dba14ba0b..d3492aa4fd 100644 --- a/src/autoscaler/mta.tpl.yaml +++ b/src/autoscaler/mta.tpl.yaml @@ -14,6 +14,7 @@ modules: requires: - name: config - name: policydb + - name: storeproceduredb - name: syslog-client - name: app-autoscaler-application-logs parameters: @@ -40,6 +41,13 @@ resources: service-tags: - policy_db - relational +- name: storeprceduredb + type: org.cloudfoundry.user-provided-service + active: false + parameters: + service-tags: + - storedprocedure_db + - relational - name: syslog-client type: org.cloudfoundry.user-provided-service parameters: From 89ced3348ab8e22b68578a8811f15dea05155b02 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 17 Sep 2024 12:07:54 +0200 Subject: [PATCH 05/19] Set up plain text logger --- .../helpers/json_redacter_with_url_creds.go | 6 +-- src/autoscaler/helpers/logger.go | 41 +++++++++++++------ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/autoscaler/helpers/json_redacter_with_url_creds.go b/src/autoscaler/helpers/json_redacter_with_url_creds.go index 1d3c764f6e..f787561577 100644 --- a/src/autoscaler/helpers/json_redacter_with_url_creds.go +++ b/src/autoscaler/helpers/json_redacter_with_url_creds.go @@ -40,13 +40,13 @@ func (r JSONRedacterWithURLCred) Redact(data []byte) []byte { } err := json.Unmarshal(data, &jsonBlob) if err != nil { - return handleError(err) + return errorToBytes(err) } r.redactValue(&jsonBlob) data, err = json.Marshal(jsonBlob) if err != nil { - return handleError(err) + return errorToBytes(err) } return r.jsonRedacter.Redact(data) @@ -82,7 +82,7 @@ func (r JSONRedacterWithURLCred) redactObject(data *map[string]interface{}) { } } -func handleError(err error) []byte { +func errorToBytes(err error) []byte { var content []byte var errType *json.UnsupportedTypeError if errors.As(err, &errType) { diff --git a/src/autoscaler/helpers/logger.go b/src/autoscaler/helpers/logger.go index 8a95e98dcd..942e08dc5c 100644 --- a/src/autoscaler/helpers/logger.go +++ b/src/autoscaler/helpers/logger.go @@ -2,6 +2,7 @@ package helpers import ( "fmt" + "log/slog" "os" "code.cloudfoundry.org/lager/v3" @@ -12,26 +13,23 @@ type LoggingConfig struct { } func InitLoggerFromConfig(conf *LoggingConfig, name string) lager.Logger { - logLevel, err := getLogLevel(conf.Level) + logLevel, err := parseLogLevel(conf.Level) if err != nil { - fmt.Fprintf(os.Stderr, "failed to initialize logger: %s\n", err.Error()) - os.Exit(1) + handleError("failed to initialize logger", err) } - logger := lager.NewLogger(name) - keyPatterns := []string{"[Pp]wd", "[Pp]ass", "[Ss]ecret", "[Tt]oken"} + logger := lager.NewLogger(name) - redactedSink, err := NewRedactingWriterWithURLCredSink(os.Stdout, logLevel, keyPatterns, nil) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create redacted sink: %s\n", err.Error()) - os.Exit(1) - } + redactedSink := createRedactedSink(logLevel) logger.RegisterSink(redactedSink) + plaintextFormatSink := createPlaintextSink() + logger.RegisterSink(plaintextFormatSink) + return logger } -func getLogLevel(level string) (lager.LogLevel, error) { +func parseLogLevel(level string) (lager.LogLevel, error) { switch level { case "debug": return lager.DEBUG, nil @@ -42,6 +40,25 @@ func getLogLevel(level string) (lager.LogLevel, error) { case "fatal": return lager.FATAL, nil default: - return -1, fmt.Errorf("Error: unsupported log level:%s", level) + return -1, fmt.Errorf("unsupported log level: %s", level) } } + +func createPlaintextSink() lager.Sink { + slogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + return lager.NewSlogSink(slogger) +} + +func createRedactedSink(logLevel lager.LogLevel) lager.Sink { + keyPatterns := []string{"[Pp]wd", "[Pp]ass", "[Ss]ecret", "[Tt]oken"} + redactedSink, err := NewRedactingWriterWithURLCredSink(os.Stdout, logLevel, keyPatterns, nil) + if err != nil { + handleError("failed to create redacted sink", err) + } + return redactedSink +} + +func handleError(message string, err error) { + fmt.Fprintf(os.Stderr, "%s: %s\n", message, err.Error()) + os.Exit(1) +} From dc6c2c363299b1bfd81315a052ac88329c74588c Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 17 Sep 2024 12:39:43 +0200 Subject: [PATCH 06/19] Fix typo --- src/autoscaler/mta.tpl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/mta.tpl.yaml b/src/autoscaler/mta.tpl.yaml index d3492aa4fd..dc39717a06 100644 --- a/src/autoscaler/mta.tpl.yaml +++ b/src/autoscaler/mta.tpl.yaml @@ -41,7 +41,7 @@ resources: service-tags: - policy_db - relational -- name: storeprceduredb +- name: storeproceduredb type: org.cloudfoundry.user-provided-service active: false parameters: From ede42ac5420052470dc976dc5f6da600fcc2884f Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 17 Sep 2024 14:14:49 +0200 Subject: [PATCH 07/19] Remove storeproceduredb dependency from autoscaler module and resources in mta.tpl.yaml --- src/autoscaler/mta.tpl.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/autoscaler/mta.tpl.yaml b/src/autoscaler/mta.tpl.yaml index dc39717a06..7dba14ba0b 100644 --- a/src/autoscaler/mta.tpl.yaml +++ b/src/autoscaler/mta.tpl.yaml @@ -14,7 +14,6 @@ modules: requires: - name: config - name: policydb - - name: storeproceduredb - name: syslog-client - name: app-autoscaler-application-logs parameters: @@ -41,13 +40,6 @@ resources: service-tags: - policy_db - relational -- name: storeproceduredb - type: org.cloudfoundry.user-provided-service - active: false - parameters: - service-tags: - - storedprocedure_db - - relational - name: syslog-client type: org.cloudfoundry.user-provided-service parameters: From e753aa74b43c29d47275c1a92faa143f3096eff7 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 18 Sep 2024 13:21:16 +0200 Subject: [PATCH 08/19] Refactor metricsforwarder config loading and enhance DB URL customization - Implement parsing and customization of the stored procedure DB URL with configurable username and password. - Move the TLS config materialization for the syslog-client outside of the conditional block. - Update tests to reflect changes in config loading and to test new DB URL customization logic. - Add username and password fields to the StoredProcedureConfig struct. --- .../metricsforwarder/config/config.go | 29 ++++++++++++-- .../metricsforwarder/config/config_test.go | 39 +++++++++++++++---- src/autoscaler/models/stored_procedure.go | 2 + 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 743ff79b61..59979c980a 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -3,7 +3,9 @@ package config import ( "errors" "fmt" + "net/url" "os" + "strings" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" @@ -158,17 +160,36 @@ func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) if !ok { conf.Db[db.StoredProcedureDb] = db.DatabaseConfig{} } + currentStoredProcedureDb.URL, err = vcapReader.MaterializeDBFromService(db.StoredProcedureDb) if err != nil { return &conf, err } + + dbURL, err := url.Parse(currentStoredProcedureDb.URL) + if err != nil { + return &conf, err + } + + if conf.StoredProcedureConfig != nil { + if conf.StoredProcedureConfig.Username != "" { + currentStoredProcedureDb.URL = strings.Replace(currentStoredProcedureDb.URL, dbURL.User.Username(), conf.StoredProcedureConfig.Username, 1) + } + + if conf.StoredProcedureConfig.Password != "" { + bindingPassword, _ := dbURL.User.Password() + currentStoredProcedureDb.URL = strings.Replace(currentStoredProcedureDb.URL, bindingPassword, conf.StoredProcedureConfig.Password, 1) + } + } + conf.Db[db.StoredProcedureDb] = currentStoredProcedureDb } - conf.SyslogConfig.TLS, err = vcapReader.MaterializeTLSConfigFromService("syslog-client") - if err != nil { - return &conf, err - } + } + + conf.SyslogConfig.TLS, err = vcapReader.MaterializeTLSConfigFromService("syslog-client") + if err != nil { + return &conf, err } return &conf, nil diff --git a/src/autoscaler/metricsforwarder/config/config_test.go b/src/autoscaler/metricsforwarder/config/config_test.go index 01c00332a9..148729ec77 100644 --- a/src/autoscaler/metricsforwarder/config/config_test.go +++ b/src/autoscaler/metricsforwarder/config/config_test.go @@ -93,7 +93,7 @@ var _ = Describe("Config", func() { When("VCAP_SERVICES has relational db service bind to app for policy db", func() { BeforeEach(func() { - mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("default"), nil) + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(`{ "cred_helper_impl": "default" }`), nil) // #nosec G101 expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 }) @@ -108,7 +108,7 @@ var _ = Describe("Config", func() { When("storedProcedure_db service is provided and cred_helper_impl is stored_procedure", func() { BeforeEach(func() { - mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("stored_procedure"), nil) + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(`{ "cred_helper_impl": "stored_procedure" }`), nil) // #nosec G101 expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 }) @@ -121,11 +121,39 @@ var _ = Describe("Config", func() { actualDbName := mockVCAPConfigurationReader.MaterializeDBFromServiceArgsForCall(1) Expect(actualDbName).To(Equal(db.StoredProcedureDb)) }) + + When("storedProcedure_db config has username and password", func() { + var storedProcedureUsername, storedProcedurePassword string + + BeforeEach(func() { + storedProcedureUsername = "storedProcedureUsername" + storedProcedurePassword = "storedProcedurePassword" + + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte( + `{ "cred_helper_impl": "stored_procedure", + "stored_procedure_binding_credential_config": { + "username": "`+storedProcedureUsername+`", + "password": "`+storedProcedurePassword+`" + }, + }`), + nil, + ) // #nosec G101 + }) + + It("should prioritize the username and password from the config", func() { + // url should include the username and password from the config + Expect(err).NotTo(HaveOccurred()) + _, storeProcedureFound := conf.Db[db.StoredProcedureDb] + Expect(storeProcedureFound).To(BeTrue()) + Expect(conf.Db[db.StoredProcedureDb].URL).To(ContainSubstring(fmt.Sprintf("%s:%s", storedProcedureUsername, storedProcedurePassword))) + }) + }) }) When("storedProcedure_db service is provided and cred_helper_impl is default", func() { BeforeEach(func() { - mockVCAPConfigurationReader.GetServiceCredentialContentReturns(getVcapConfigWithCredImplementation("default"), nil) + mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte( + `{ "cred_helper_impl": "default" }`), nil) // #nosec G101 expectedDbUrl = "postgres://foo:bar@postgres.example.com:5432/policy_db?sslcert=%2Ftmp%2Fclient_cert.sslcert&sslkey=%2Ftmp%2Fclient_key.sslkey&sslrootcert=%2Ftmp%2Fserver_ca.sslrootcert" // #nosec G101 }) @@ -138,7 +166,6 @@ var _ = Describe("Config", func() { When("VCAP_SERVICES has metricsforwarder config", func() { BeforeEach(func() { - mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(` { "cache_cleanup_interval":"10h", "cache_ttl":"90s", @@ -414,7 +441,3 @@ health: }) }) }) - -func getVcapConfigWithCredImplementation(credHelperImplementation string) []byte { - return []byte(`{ "cred_helper_impl": "` + credHelperImplementation + `" }`) // #nosec G101 -} diff --git a/src/autoscaler/models/stored_procedure.go b/src/autoscaler/models/stored_procedure.go index 0693105250..864120c2d9 100644 --- a/src/autoscaler/models/stored_procedure.go +++ b/src/autoscaler/models/stored_procedure.go @@ -6,4 +6,6 @@ type StoredProcedureConfig struct { DropBindingCredentialProcedureName string `yaml:"drop_binding_credential_procedure_name"` DropAllBindingCredentialProcedureName string `yaml:"drop_all_binding_credential_procedure_name"` ValidateBindingCredentialProcedureName string `yaml:"validate_binding_credential_procedure_name"` + Username string `yaml:"username"` + Password string `yaml:"password"` } From 909b56c325b4ac9a3aa8561519a5e2fb75d3e30e Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 18 Sep 2024 14:02:57 +0200 Subject: [PATCH 09/19] Refactor mf config --- .../metricsforwarder/config/config.go | 324 ++++++++++-------- .../metricsforwarder/config/config_test.go | 23 +- 2 files changed, 190 insertions(+), 157 deletions(-) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 59979c980a..6228a3392e 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -16,15 +16,10 @@ import ( "gopkg.in/yaml.v3" ) -// There are 3 type of errors that this package can return: -// - ErrReadYaml -// - ErrReadEnvironment -// - ErrReadVCAPEnvironment - var ( ErrReadYaml = errors.New("failed to read config file") ErrReadJson = errors.New("failed to read vcap_services json") - ErrMetricsforwarderConfigNotFound = errors.New("Configuration error: metricsforwarder config service not found") + ErrMetricsforwarderConfigNotFound = errors.New("metricsforwarder config service not found") ) const ( @@ -36,6 +31,16 @@ const ( DefaultValidDuration = 1 * time.Second ) +type LoggregatorConfig struct { + MetronAddress string `yaml:"metron_address"` + TLS models.TLSCerts `yaml:"tls"` +} +type SyslogConfig struct { + ServerAddress string `yaml:"server_address"` + Port int `yaml:"port"` + TLS models.TLSCerts `yaml:"tls"` +} + type Config struct { Logging helpers.LoggingConfig `yaml:"logging"` Server helpers.ServerConfig `yaml:"server"` @@ -51,66 +56,28 @@ type Config struct { StoredProcedureConfig *models.StoredProcedureConfig `yaml:"stored_procedure_binding_credential_config"` } -var defaultServerConfig = helpers.ServerConfig{ - Port: 6110, -} - -var defaultHealthConfig = helpers.HealthConfig{ - ServerConfig: helpers.ServerConfig{ - Port: 8081, - }, -} - -var defaultLoggingConfig = helpers.LoggingConfig{ - Level: "info", -} - -type LoggingConfig struct { - Level string `yaml:"level"` -} - -type LoggregatorConfig struct { - MetronAddress string `yaml:"metron_address"` - TLS models.TLSCerts `yaml:"tls"` -} - -type SyslogConfig struct { - ServerAddress string `yaml:"server_address"` - Port int `yaml:"port"` - TLS models.TLSCerts `yaml:"tls"` -} - -func decodeYamlFile(filepath string, c *Config) error { - r, err := os.Open(filepath) +func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) (*Config, error) { + conf := defaultConfig() - if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "failed to open config file '%s' : %s\n", filepath, err.Error()) - return err + if err := loadYamlFile(filepath, &conf); err != nil { + return nil, err } - dec := yaml.NewDecoder(r) - dec.KnownFields(true) - err = dec.Decode(c) - - if err != nil { - return fmt.Errorf("%w: %w", ErrReadYaml, err) + if err := loadVcapConfig(&conf, vcapReader); err != nil { + return nil, err } - defer r.Close() - return nil + return &conf, nil } -func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) (*Config, error) { - var conf Config - var err error - - conf = Config{ - Server: defaultServerConfig, - Logging: defaultLoggingConfig, +func defaultConfig() Config { + return Config{ + Server: helpers.ServerConfig{Port: 6110}, + Logging: helpers.LoggingConfig{Level: "info"}, LoggregatorConfig: LoggregatorConfig{ MetronAddress: DefaultMetronAddress, }, - Health: defaultHealthConfig, + Health: helpers.HealthConfig{ServerConfig: helpers.ServerConfig{Port: 8081}}, CacheTTL: DefaultCacheTTL, CacheCleanupInterval: DefaultCacheCleanupInterval, PolicyPollerInterval: DefaultPolicyPollerInterval, @@ -119,125 +86,192 @@ func LoadConfig(filepath string, vcapReader configutil.VCAPConfigurationReader) ValidDuration: DefaultValidDuration, }, } +} - if filepath != "" { - err = decodeYamlFile(filepath, &conf) - if err != nil { - return nil, err - } +func loadYamlFile(filepath string, conf *Config) error { + if filepath == "" { + return nil } + file, err := os.Open(filepath) + if err != nil { + fmt.Fprintf(os.Stdout, "failed to open config file '%s': %s\n", filepath, err) + return ErrReadYaml + } + defer file.Close() - if vcapReader.IsRunningOnCF() { - conf.Server.Port = vcapReader.GetPort() + dec := yaml.NewDecoder(file) + dec.KnownFields(true) + if err := dec.Decode(conf); err != nil { + return fmt.Errorf("%w: %v", ErrReadYaml, err) + } + return nil +} - data, err := vcapReader.GetServiceCredentialContent("config", "metricsforwarder") - if err != nil { - return &conf, fmt.Errorf("%w: %w", ErrMetricsforwarderConfigNotFound, err) - } +func loadVcapConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + if !vcapReader.IsRunningOnCF() { + return nil + } - err = yaml.Unmarshal(data, &conf) - if err != nil { - return &conf, fmt.Errorf("%w: %w", ErrReadJson, err) - } + conf.Server.Port = vcapReader.GetPort() + if err := loadMetricsforwarderConfig(conf, vcapReader); err != nil { + return err + } - if conf.Db == nil { - conf.Db = make(map[string]db.DatabaseConfig) - } + if conf.Db == nil { + conf.Db = make(map[string]db.DatabaseConfig) + } - currentPolicyDb, ok := conf.Db[db.PolicyDb] - if !ok { - conf.Db[db.PolicyDb] = db.DatabaseConfig{} - } + if err := configurePolicyDb(conf, vcapReader); err != nil { + return err + } - currentPolicyDb.URL, err = vcapReader.MaterializeDBFromService(db.PolicyDb) - if err != nil { - return &conf, err - } - conf.Db[db.PolicyDb] = currentPolicyDb - - if conf.CredHelperImpl == "stored_procedure" { - currentStoredProcedureDb, ok := conf.Db[db.StoredProcedureDb] - if !ok { - conf.Db[db.StoredProcedureDb] = db.DatabaseConfig{} - } - - currentStoredProcedureDb.URL, err = vcapReader.MaterializeDBFromService(db.StoredProcedureDb) - if err != nil { - return &conf, err - } - - dbURL, err := url.Parse(currentStoredProcedureDb.URL) - if err != nil { - return &conf, err - } - - if conf.StoredProcedureConfig != nil { - if conf.StoredProcedureConfig.Username != "" { - currentStoredProcedureDb.URL = strings.Replace(currentStoredProcedureDb.URL, dbURL.User.Username(), conf.StoredProcedureConfig.Username, 1) - } - - if conf.StoredProcedureConfig.Password != "" { - bindingPassword, _ := dbURL.User.Password() - currentStoredProcedureDb.URL = strings.Replace(currentStoredProcedureDb.URL, bindingPassword, conf.StoredProcedureConfig.Password, 1) - } - } - - conf.Db[db.StoredProcedureDb] = currentStoredProcedureDb + if conf.CredHelperImpl == "stored_procedure" { + if err := configureStoredProcedureDb(conf, vcapReader); err != nil { + return err } + } + if err := configureSyslogTLS(conf, vcapReader); err != nil { + return err } - conf.SyslogConfig.TLS, err = vcapReader.MaterializeTLSConfigFromService("syslog-client") + return nil +} + +func loadMetricsforwarderConfig(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + data, err := vcapReader.GetServiceCredentialContent("config", "metricsforwarder") if err != nil { - return &conf, err + return fmt.Errorf("%w: %v", ErrMetricsforwarderConfigNotFound, err) } + return yaml.Unmarshal(data, conf) +} - return &conf, nil +func configurePolicyDb(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + + currentPolicyDb, ok := conf.Db[db.PolicyDb] + if !ok { + conf.Db[db.PolicyDb] = db.DatabaseConfig{} + } + + dbURL, err := vcapReader.MaterializeDBFromService(db.PolicyDb) + currentPolicyDb.URL = dbURL + if err != nil { + return err + } + conf.Db[db.PolicyDb] = currentPolicyDb + return nil } -func (c *Config) UsingSyslog() bool { - return c.SyslogConfig.ServerAddress != "" && c.SyslogConfig.Port != 0 +func configureStoredProcedureDb(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + currentStoredProcedureDb, exists := conf.Db[db.StoredProcedureDb] + if !exists { + conf.Db[db.StoredProcedureDb] = db.DatabaseConfig{} + } + + dbURL, err := vcapReader.MaterializeDBFromService(db.StoredProcedureDb) + + currentStoredProcedureDb.URL = dbURL + parsedUrl, err := url.Parse(currentStoredProcedureDb.URL) + if err != nil { + return err + } + + if conf.StoredProcedureConfig != nil { + if conf.StoredProcedureConfig.Username != "" { + currentStoredProcedureDb.URL = strings.Replace(currentStoredProcedureDb.URL, parsedUrl.User.Username(), conf.StoredProcedureConfig.Username, 1) + } + if conf.StoredProcedureConfig.Password != "" { + bindingPassword, _ := parsedUrl.User.Password() + currentStoredProcedureDb.URL = strings.Replace(currentStoredProcedureDb.URL, bindingPassword, conf.StoredProcedureConfig.Password, 1) + } + } + conf.Db[db.StoredProcedureDb] = currentStoredProcedureDb + + return nil +} + +func configureSyslogTLS(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { + tls, err := vcapReader.MaterializeTLSConfigFromService("syslog-client") + if err != nil { + return err + } + conf.SyslogConfig.TLS = tls + return nil } func (c *Config) Validate() error { + if err := c.validateDbConfig(); err != nil { + return err + } + if err := c.validateSyslogOrLoggregator(); err != nil { + return err + } + if err := c.validateRateLimit(); err != nil { + return err + } + if err := c.validateCredHelperImpl(); err != nil { + return err + } + return c.Health.Validate() +} + +func (c *Config) validateDbConfig() error { if c.Db[db.PolicyDb].URL == "" { - return fmt.Errorf("Configuration error: Policy DB url is empty") + return errors.New("Policy DB url is empty") } + return nil +} + +func (c *Config) validateSyslogOrLoggregator() error { if c.UsingSyslog() { - if c.SyslogConfig.TLS.CACertFile == "" { - return fmt.Errorf("Configuration error: SyslogServer Loggregator CACert is empty") - } - if c.SyslogConfig.TLS.CertFile == "" { - return fmt.Errorf("Configuration error: SyslogServer ClientCert is empty") - } - if c.SyslogConfig.TLS.KeyFile == "" { - return fmt.Errorf("Configuration error: SyslogServer ClientKey is empty") - } - } else { - if c.LoggregatorConfig.TLS.CACertFile == "" { - return fmt.Errorf("Configuration error: Loggregator CACert is empty") - } - if c.LoggregatorConfig.TLS.CertFile == "" { - return fmt.Errorf("Configuration error: Loggregator ClientCert is empty") - } - if c.LoggregatorConfig.TLS.KeyFile == "" { - return fmt.Errorf("Configuration error: Loggregator ClientKey is empty") - } + return c.validateSyslogConfig() } + return c.validateLoggregatorConfig() +} - if c.RateLimit.MaxAmount <= 0 { - return fmt.Errorf("Configuration error: RateLimit.MaxAmount is equal or less than zero") +func (c *Config) validateSyslogConfig() error { + if c.SyslogConfig.TLS.CACertFile == "" { + return errors.New("SyslogServer Loggregator CACert is empty") } - if c.RateLimit.ValidDuration <= 0*time.Nanosecond { - return fmt.Errorf("Configuration error: RateLimit.ValidDuration is equal or less than zero nanosecond") + if c.SyslogConfig.TLS.CertFile == "" { + return errors.New("SyslogServer ClientCert is empty") } - if c.CredHelperImpl == "" { - return fmt.Errorf("Configuration error: CredHelperImpl is empty") + if c.SyslogConfig.TLS.KeyFile == "" { + return errors.New("SyslogServer ClientKey is empty") } + return nil +} - if err := c.Health.Validate(); err != nil { - return err +func (c *Config) validateLoggregatorConfig() error { + if c.LoggregatorConfig.TLS.CACertFile == "" { + return errors.New("Loggregator CACert is empty") } + if c.LoggregatorConfig.TLS.CertFile == "" { + return errors.New("Loggregator ClientCert is empty") + } + if c.LoggregatorConfig.TLS.KeyFile == "" { + return errors.New("Loggregator ClientKey is empty") + } + return nil +} + +func (c *Config) validateRateLimit() error { + if c.RateLimit.MaxAmount <= 0 { + return errors.New("RateLimit.MaxAmount is less than or equal to zero") + } + if c.RateLimit.ValidDuration <= 0 { + return errors.New("RateLimit.ValidDuration is less than or equal to zero") + } + return nil +} +func (c *Config) validateCredHelperImpl() error { + if c.CredHelperImpl == "" { + return errors.New("CredHelperImpl is empty") + } return nil } + +func (c *Config) UsingSyslog() bool { + return c.SyslogConfig.ServerAddress != "" && c.SyslogConfig.Port != 0 +} diff --git a/src/autoscaler/metricsforwarder/config/config_test.go b/src/autoscaler/metricsforwarder/config/config_test.go index 148729ec77..f61b8d9d82 100644 --- a/src/autoscaler/metricsforwarder/config/config_test.go +++ b/src/autoscaler/metricsforwarder/config/config_test.go @@ -63,12 +63,12 @@ var _ = Describe("Config", func() { When("service is empty", func() { var expectedErr error BeforeEach(func() { - expectedErr = fmt.Errorf("Configuration error: metricsforwarder config service not found") + expectedErr = fmt.Errorf("metricsforwarder config service not found") mockVCAPConfigurationReader.GetServiceCredentialContentReturns([]byte(""), expectedErr) }) It("should error with config service not found", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: metricsforwarder config service not found"))) + Expect(err).To(MatchError(MatchRegexp("metricsforwarder config service not found"))) }) }) @@ -348,7 +348,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer Loggregator CACert is empty"))) + Expect(err).To(MatchError(MatchRegexp("SyslogServer Loggregator CACert is empty"))) }) }) @@ -358,7 +358,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer ClientKey is empty"))) + Expect(err).To(MatchError(MatchRegexp("SyslogServer ClientKey is empty"))) }) }) @@ -368,7 +368,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: SyslogServer ClientCert is empty"))) + Expect(err).To(MatchError(MatchRegexp("SyslogServer ClientCert is empty"))) }) }) }) @@ -385,7 +385,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: Policy DB url is empty"))) + Expect(err).To(MatchError(MatchRegexp("Policy DB url is empty"))) }) }) @@ -395,7 +395,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: Loggregator CACert is empty"))) + Expect(err).To(MatchError(MatchRegexp("Loggregator CACert is empty"))) }) }) @@ -405,7 +405,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: Loggregator ClientCert is empty"))) + Expect(err).To(MatchError(MatchRegexp("Loggregator ClientCert is empty"))) }) }) @@ -415,7 +415,7 @@ health: }) It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: Loggregator ClientKey is empty"))) + Expect(err).To(MatchError(MatchRegexp("Loggregator ClientKey is empty"))) }) }) @@ -425,8 +425,7 @@ health: }) It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.MaxAmount is equal or less than zero"))) - + Expect(err).To(MatchError(MatchRegexp("RateLimit.MaxAmount is less than or equal to zero"))) }) }) @@ -436,7 +435,7 @@ health: }) It("should err", func() { - Expect(err).To(MatchError(MatchRegexp("Configuration error: RateLimit.ValidDuration is equal or less than zero nanosecond"))) + Expect(err).To(MatchError(MatchRegexp("RateLimit.ValidDuration is less than or equal to zero"))) }) }) }) From 0401505539c1c1d4e6c4250e5e71b41382a6c0ef Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Fri, 27 Sep 2024 18:07:07 +0200 Subject: [PATCH 10/19] Fix issue with acceptnace test --- src/autoscaler/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index 3318b9cb44..4a8ed8c098 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -152,7 +152,7 @@ clean: mta-deploy: mta-build build-extension-file $(MAKE) -f metricsforwarder/Makefile set-security-group @echo "Deploying with extension file: $(EXTENSION_FILE)" - @cf deploy $(MAKEFILE_DIR)/mta_archives/*.mtar -f --delete-services -e $(EXTENSION_FILE) + @cf deploy $(MAKEFILE_DIR)/$(DEST)/*.mtar -f --delete-services -e $(EXTENSION_FILE) build-extension-file: From a2bc272eb75bb4cafae8467a43040d9db83ab632 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 30 Sep 2024 10:56:44 +0200 Subject: [PATCH 11/19] Change type of mta resource for logs --- src/acceptance/go.mod | 4 +--- src/acceptance/helpers/helpers_test.go | 14 -------------- src/autoscaler/build-extension-file.sh | 4 ++++ src/autoscaler/helpers/logger.go | 13 +++++++++---- src/autoscaler/mta.tpl.yaml | 7 ++----- 5 files changed, 16 insertions(+), 26 deletions(-) delete mode 100644 src/acceptance/helpers/helpers_test.go diff --git a/src/acceptance/go.mod b/src/acceptance/go.mod index 8f8a49ddce..1a8cd1f44e 100644 --- a/src/acceptance/go.mod +++ b/src/acceptance/go.mod @@ -6,18 +6,16 @@ require ( github.com/cloudfoundry/cf-test-helpers/v2 v2.9.0 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 - github.com/stretchr/testify v1.9.0 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect github.com/kr/pretty v0.3.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.8.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect diff --git a/src/acceptance/helpers/helpers_test.go b/src/acceptance/helpers/helpers_test.go deleted file mode 100644 index f428822df3..0000000000 --- a/src/acceptance/helpers/helpers_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package helpers_test - -import ( - "acceptance/config" - "acceptance/helpers" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestServicePlans_urlIsCorrect(t *testing.T) { - url := helpers.ServicePlansUrl(&config.Config{ServiceName: "autoscaler", ServiceBroker: "autoscaler"}, "GUID_UUID") - assert.Equal(t, url, "/v3/service_plans?available=true&fields%5Bservice_offering.service_broker%5D=name%2Cguid&include=service_offering&per_page=5000&service_broker_names=autoscaler&service_offering_names=autoscaler&space_guids=GUID_UUID") -} diff --git a/src/autoscaler/build-extension-file.sh b/src/autoscaler/build-extension-file.sh index a35b1805a3..a1adb01176 100755 --- a/src/autoscaler/build-extension-file.sh +++ b/src/autoscaler/build-extension-file.sh @@ -39,6 +39,10 @@ _schema-version: 3.3.0 modules: - name: metricsforwarder + requires: + - name: config + - name: policydb + - name: syslog-client parameters: routes: - route: ${METRICSFORWARDER_APPNAME}.\${default-domain} diff --git a/src/autoscaler/helpers/logger.go b/src/autoscaler/helpers/logger.go index 942e08dc5c..f6531cfff5 100644 --- a/src/autoscaler/helpers/logger.go +++ b/src/autoscaler/helpers/logger.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/configutil" "code.cloudfoundry.org/lager/v3" ) @@ -20,11 +21,15 @@ func InitLoggerFromConfig(conf *LoggingConfig, name string) lager.Logger { logger := lager.NewLogger(name) - redactedSink := createRedactedSink(logLevel) - logger.RegisterSink(redactedSink) + vcapConfig, _ := configutil.NewVCAPConfigurationReader() - plaintextFormatSink := createPlaintextSink() - logger.RegisterSink(plaintextFormatSink) + if vcapConfig.IsRunningOnCF() { + plaintextFormatSink := createPlaintextSink() + logger.RegisterSink(plaintextFormatSink) + } else { + redactedSink := createRedactedSink(logLevel) + logger.RegisterSink(redactedSink) + } return logger } diff --git a/src/autoscaler/mta.tpl.yaml b/src/autoscaler/mta.tpl.yaml index 7dba14ba0b..0c5f5b5122 100644 --- a/src/autoscaler/mta.tpl.yaml +++ b/src/autoscaler/mta.tpl.yaml @@ -47,10 +47,7 @@ resources: - syslog-client - name: app-autoscaler-application-logs active: false - type: application-logs + type: org.cloudfoundry.managed-service parameters: - service-broker: null + service: application-logs service-plan: standard - parameters-metadata: - service-broker: - optional: false From 81878960db84f548a4a5da253cfc9b5a70526feb Mon Sep 17 00:00:00 2001 From: bonzofenix <317403+bonzofenix@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:07:38 +0100 Subject: [PATCH 12/19] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/eventgenerator/spec | 3 +++ packages/golangapiserver/spec | 3 +++ packages/operator/spec | 3 +++ packages/scalingengine/spec | 3 +++ 4 files changed, 12 insertions(+) diff --git a/packages/eventgenerator/spec b/packages/eventgenerator/spec index df0e15e382..d9cfe88dc7 100644 --- a/packages/eventgenerator/spec +++ b/packages/eventgenerator/spec @@ -12,6 +12,7 @@ files: - autoscaler/vendor/google.golang.org/protobuf/types/descriptorpb/* - autoscaler/cf/* # gosub - autoscaler/collection/* # gosub +- autoscaler/configutil/* # gosub - autoscaler/db/* # gosub - autoscaler/db/sqldb/* # gosub - autoscaler/envelopeprocessor/* # gosub @@ -43,6 +44,7 @@ files: - autoscaler/vendor/github.com/cenk/backoff/* # gosub - autoscaler/vendor/github.com/cenkalti/backoff/v4/* # gosub - autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub +- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub - autoscaler/vendor/github.com/facebookgo/clock/* # gosub - autoscaler/vendor/github.com/felixge/httpsnoop/* # gosub - autoscaler/vendor/github.com/go-logr/logr/* # gosub @@ -81,6 +83,7 @@ files: - autoscaler/vendor/github.com/klauspost/compress/internal/snapref/* # gosub - autoscaler/vendor/github.com/klauspost/compress/zstd/* # gosub - autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub +- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub - autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub - autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub - autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub diff --git a/packages/golangapiserver/spec b/packages/golangapiserver/spec index a39eaf8082..4b88477766 100644 --- a/packages/golangapiserver/spec +++ b/packages/golangapiserver/spec @@ -20,6 +20,7 @@ files: - autoscaler/api/publicapiserver/* # gosub - autoscaler/api/schedulerclient/* # gosub - autoscaler/cf/* # gosub +- autoscaler/configutil/* # gosub - autoscaler/cred_helper/* # gosub - autoscaler/db/* # gosub - autoscaler/db/sqldb/* # gosub @@ -40,6 +41,7 @@ files: - autoscaler/vendor/filippo.io/edwards25519/field/* # gosub - autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub - autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub +- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub - autoscaler/vendor/github.com/dlclark/regexp2/* # gosub - autoscaler/vendor/github.com/dlclark/regexp2/syntax/* # gosub - autoscaler/vendor/github.com/fatih/color/* # gosub @@ -85,6 +87,7 @@ files: - autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub - autoscaler/vendor/github.com/mattn/go-colorable/* # gosub - autoscaler/vendor/github.com/mattn/go-isatty/* # gosub +- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub - autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub - autoscaler/vendor/github.com/ogen-go/ogen/conv/* # gosub - autoscaler/vendor/github.com/ogen-go/ogen/http/* # gosub diff --git a/packages/operator/spec b/packages/operator/spec index df2d29b993..6671f4667c 100644 --- a/packages/operator/spec +++ b/packages/operator/spec @@ -9,6 +9,7 @@ files: - autoscaler/vendor/* - autoscaler/operator/db/* - autoscaler/cf/* # gosub +- autoscaler/configutil/* # gosub - autoscaler/db/* # gosub - autoscaler/db/sqldb/* # gosub - autoscaler/healthendpoint/* # gosub @@ -29,6 +30,7 @@ files: - autoscaler/vendor/filippo.io/edwards25519/field/* # gosub - autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub - autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub +- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub - autoscaler/vendor/github.com/go-logr/logr/* # gosub - autoscaler/vendor/github.com/go-logr/logr/funcr/* # gosub - autoscaler/vendor/github.com/go-logr/stdr/* # gosub @@ -62,6 +64,7 @@ files: - autoscaler/vendor/github.com/klauspost/compress/internal/snapref/* # gosub - autoscaler/vendor/github.com/klauspost/compress/zstd/* # gosub - autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub +- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub - autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub - autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub - autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub diff --git a/packages/scalingengine/spec b/packages/scalingengine/spec index 293a9cd454..3769e5739a 100644 --- a/packages/scalingengine/spec +++ b/packages/scalingengine/spec @@ -9,6 +9,7 @@ files: - autoscaler/vendor/* - autoscaler/scalingengine/db/* - autoscaler/cf/* # gosub +- autoscaler/configutil/* # gosub - autoscaler/db/* # gosub - autoscaler/db/sqldb/* # gosub - autoscaler/healthendpoint/* # gosub @@ -32,6 +33,7 @@ files: - autoscaler/vendor/filippo.io/edwards25519/field/* # gosub - autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub - autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub +- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub - autoscaler/vendor/github.com/dlclark/regexp2/* # gosub - autoscaler/vendor/github.com/dlclark/regexp2/syntax/* # gosub - autoscaler/vendor/github.com/fatih/color/* # gosub @@ -76,6 +78,7 @@ files: - autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub - autoscaler/vendor/github.com/mattn/go-colorable/* # gosub - autoscaler/vendor/github.com/mattn/go-isatty/* # gosub +- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub - autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub - autoscaler/vendor/github.com/ogen-go/ogen/conv/* # gosub - autoscaler/vendor/github.com/ogen-go/ogen/http/* # gosub From 5cb80fedd4d1884210044cabb7b808add62a8296 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 30 Sep 2024 15:31:29 +0200 Subject: [PATCH 13/19] Remove mta.yaml from .gitignore in autoscaler module --- src/autoscaler/.gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/src/autoscaler/.gitignore b/src/autoscaler/.gitignore index 9951afd805..d3023bec01 100644 --- a/src/autoscaler/.gitignore +++ b/src/autoscaler/.gitignore @@ -1,5 +1,4 @@ build/ -mta.yaml fakes/ mta_archives/ mta.yaml From bc83916fb0658685c55b100c0fb70326c821fa02 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 30 Sep 2024 18:33:00 +0200 Subject: [PATCH 14/19] Add VERSION file and update release scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Introduced a VERSION file to track the release version. • Simplified the release-autoscaler.sh script by removing hardcoded variables and adding a new create_mtar function for building mtar artifacts. • Added local_release_autoscaler.sh script to facilitate local testing and release process, including automatic SSH key generation and cleanup tasks. • Updated changelog generation to include mtar artifact details. --- VERSION | 1 + ci/autoscaler/scripts/release-autoscaler.sh | 37 ++++++++----- scripts/local_release_autoscaler.sh | 60 +++++++++++++++++++++ 3 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 VERSION create mode 100755 scripts/local_release_autoscaler.sh diff --git a/VERSION b/VERSION new file mode 100644 index 0000000000..726bfd5981 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +14.1.5 diff --git a/ci/autoscaler/scripts/release-autoscaler.sh b/ci/autoscaler/scripts/release-autoscaler.sh index f1e9c1760e..9570f1a933 100755 --- a/ci/autoscaler/scripts/release-autoscaler.sh +++ b/ci/autoscaler/scripts/release-autoscaler.sh @@ -1,21 +1,8 @@ #! /usr/bin/env bash # NOTE: you can run this locally for testing !!! -# beware that it adds a commit you need to drop each time also you need to remove dev_releases from root. # -# DEPLOYMENT=foo \ -# GITHUB_TOKEN="ghp_..." \ -# PREV_VERSION=12.2.1 \ -# DEST="${PWD}/../../../build" \ -# VERSION="12.3.0" \ -# BUILD_OPTS="--force" \ -# AUTOSCALER_CI_BOT_NAME="foo" \ -# AUTOSCALER_CI_BOT_EMAIL="foo@bar.baz" \ -# AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC="ssh-ed25519 AAAA... foo@bar.baz" \ -# AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE="-----BEGIN OPENSSH PRIVATE KEY----- -# b3Bl... -# -----END OPENSSH PRIVATE KEY-----" \ -# ./ci/autoscaler/scripts/release-autoscaler.sh +# ./script/local_release_autoscaler.sh [ -n "${DEBUG}" ] && set -x @@ -47,6 +34,8 @@ function create_release() { yq eval -i ".properties.\"autoscaler.apiserver.info.build\".default = \"${version}\"" jobs/golangapiserver/spec git add jobs/golangapiserver/spec + echo "${version}" VERSION + git add VERSION [ "${CI}" = "true" ] && git commit -S -m "Updated release version to ${version} in golangapiserver" # shellcheck disable=SC2086 @@ -56,6 +45,17 @@ function create_release() { --tarball="${build_path}/artifacts/${release_file}" } +function create_mtar() { + set -e + mkdir -p "${build_path}/artifacts" + local version=$1 + local build_path=$2 + echo " - creating autorscaler mtar artifact" + pushd "${autoscaler_dir}" > /dev/null + make mta-release VERSION="${version}" DEST="${build_path}/artifacts/" + popd > /dev/null +} + function create_tests() { set -e mkdir -p "${build_path}/artifacts" @@ -136,19 +136,24 @@ pushd "${autoscaler_dir}" > /dev/null if [ "${PERFORM_BOSH_RELEASE}" == "true" ]; then RELEASE_TGZ="app-autoscaler-v${VERSION}.tgz" ACCEPTANCE_TEST_TGZ="app-autoscaler-acceptance-tests-v${VERSION}.tgz" + AUTOSCALER_MTAR="app-autoscaler-release-v${VERSION}.mtar" create_release "${VERSION}" "${build_path}" "${RELEASE_TGZ}" create_tests "${VERSION}" "${build_path}" + create_mtar "${VERSION}" "${build_path}" [ "${CI}" = "true" ] && commit_release sha256sum "${build_path}/artifacts/"* > "${build_path}/artifacts/files.sum.sha256" ACCEPTANCE_SHA256=$( grep "${ACCEPTANCE_TEST_TGZ}$" "${SUM_FILE}" | awk '{print $1}' ) RELEASE_SHA256=$( grep "${RELEASE_TGZ}$" "${SUM_FILE}" | awk '{print $1}') + MTAR_SHA256=$( grep "${AUTOSCALER_MTAR}$" "${SUM_FILE}" | awk '{print $1}') else ACCEPTANCE_SHA256="dummy-sha" RELEASE_SHA256="dummy-sha" + MTAR_SHA256="dummy-sha" fi export ACCEPTANCE_SHA256 export RELEASE_SHA256 + export MTAR_SHA256 cat >> "${build_path}/changelog.md" < /dev/null && pwd ) + +DEPLOYMENT=foo +export DEBUG=true +export PREV_VERSION=12.2.1 +export DEST="${script_dir}/../build" +export VERSION="12.3.0" +export BUILD_OPTS="--force" +export AUTOSCALER_CI_BOT_NAME="foo" +export AUTOSCALER_CI_BOT_EMAIL="foo@bar.baz" +export PREV_VERSION=$(cat ${script_dir}/../VERSION) +export VERSION=$(cat ${script_dir}/../VERSION)-pre + + +# check for GITHUB_TOKEN +if [ -z "${GITHUB_TOKEN}" ]; then + echo "GITHUB_TOKEN is not set" + exit 1 +fi + + +find_or_create_ssh_key() { + if [ -f ~/.ssh/id_ed25519 ]; then + echo "ssh key already exists" + return + fi + + ssh-keygen -t ed25519 -C "${AUTOSCALER_CI_BOT_EMAIL}" -f ~/.ssh/id_ed25519 -N "" +} + +prerelease() { + pushd "${script_dir}/.." > /dev/null + make clean generate-fakes generate-openapi-generated-clients-and-servers go-mod-tidy go-mod-vendor db scheduler + popd > /dev/null +} + +delete_dev_releases() { + rm -rf dev_releases +} + + +release_autoscaler() { + export AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC=$(cat ~/.ssh/id_ed25519.pub) + export AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE=$(cat ~/.ssh/id_ed25519) + source "${script_dir}/../ci/autoscaler/scripts/release-autoscaler.sh" + echo "beware that it adds a commit you need to drop each time also you need to remove dev_releases from root." +} + +main() { + find_or_create_ssh_key + delete_dev_releases + prerelease + release_autoscaler +} + +main + From 56e6b99df8f5f52119fcf67b9c79d2098ef86a9e Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 1 Oct 2024 13:42:13 +0200 Subject: [PATCH 15/19] Fix lint --- src/autoscaler/metricsforwarder/config/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 81076d1eba..6536d0b95e 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -147,7 +147,6 @@ func loadMetricsforwarderConfig(conf *Config, vcapReader configutil.VCAPConfigur } func configurePolicyDb(conf *Config, vcapReader configutil.VCAPConfigurationReader) error { - currentPolicyDb, ok := conf.Db[db.PolicyDb] if !ok { conf.Db[db.PolicyDb] = db.DatabaseConfig{} From 35e30aa113657e4a1f4942254e9ebfb26c8b0a53 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 1 Oct 2024 16:13:24 +0200 Subject: [PATCH 16/19] Adds mtar bucket for release in pipeline --- ci/autoscaler/pipeline.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index afb7141a8b..528af0340f 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -148,6 +148,14 @@ resources: regexp: releases/app-autoscaler-acceptance-tests-v(.*).tgz initial_path: releases/app-autoscaler-acceptance-tests-v0.0.1.tgz +- name: mtar-bucket + type: gcs-resource + source: + bucket: app-autoscaler-releases + json_key: ((autoscaler_releases_uploader_key)) + regexp: releases/app-autoscaler-v(.*).mtar + initial_path: releases/app-autoscaler-v0.0.1.mtar + - name: golang-release type: git icon: github @@ -518,6 +526,9 @@ jobs: - put: acceptance-tests-bucket params: file: "build/artifacts/app-autoscaler-acceptance-tests-v*.tgz" + - put: mtar-bucket + params: + file: "build/artifacts/app-autoscaler-release-v*.mtar" - put: gh-release params: name: build/name From 7e9982eca6815723d55ec4b03922970cbfee91ca Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 1 Oct 2024 16:22:53 +0200 Subject: [PATCH 17/19] Fix lints --- scripts/local_release_autoscaler.sh | 33 +++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/local_release_autoscaler.sh b/scripts/local_release_autoscaler.sh index 1b8c40912c..ad03881fec 100755 --- a/scripts/local_release_autoscaler.sh +++ b/scripts/local_release_autoscaler.sh @@ -1,19 +1,27 @@ #! /usr/bin/env bash # +# shellcheck disable=SC1091 +# set -euo pipefail script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) DEPLOYMENT=foo -export DEBUG=true -export PREV_VERSION=12.2.1 -export DEST="${script_dir}/../build" -export VERSION="12.3.0" -export BUILD_OPTS="--force" -export AUTOSCALER_CI_BOT_NAME="foo" -export AUTOSCALER_CI_BOT_EMAIL="foo@bar.baz" -export PREV_VERSION=$(cat ${script_dir}/../VERSION) -export VERSION=$(cat ${script_dir}/../VERSION)-pre +DEBUG=true +DEST="${script_dir}/../build" +BUILD_OPTS="--force" +AUTOSCALER_CI_BOT_NAME="foo" +AUTOSCALER_CI_BOT_EMAIL="foo@bar.baz" +PREV_VERSION="$(cat "${script_dir}/../VERSION")" +VERSION="$(cat "${script_dir}/../VERSION")-pre" +export DEPLOYMENT +export DEBUG +export DEST +export BUILD_OPTS +export AUTOSCALER_CI_BOT_NAME +export AUTOSCALER_CI_BOT_EMAIL +export PREV_VERSION +export VERSION # check for GITHUB_TOKEN if [ -z "${GITHUB_TOKEN}" ]; then @@ -43,8 +51,11 @@ delete_dev_releases() { release_autoscaler() { - export AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC=$(cat ~/.ssh/id_ed25519.pub) - export AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE=$(cat ~/.ssh/id_ed25519) + AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC=$(cat ~/.ssh/id_ed25519.pub) + AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE=$(cat ~/.ssh/id_ed25519) + export AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC + export AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE + source "${script_dir}/../ci/autoscaler/scripts/release-autoscaler.sh" echo "beware that it adds a commit you need to drop each time also you need to remove dev_releases from root." } From c65298f953d2cf7492ba5fce1974f6c03a570edf Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 2 Oct 2024 16:13:22 +0200 Subject: [PATCH 18/19] Read version number from golangapiserver spec --- VERSION | 1 - ci/autoscaler/scripts/release-autoscaler.sh | 2 -- scripts/local_release_autoscaler.sh | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 VERSION diff --git a/VERSION b/VERSION deleted file mode 100644 index 726bfd5981..0000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -14.1.5 diff --git a/ci/autoscaler/scripts/release-autoscaler.sh b/ci/autoscaler/scripts/release-autoscaler.sh index 9570f1a933..99eda04e45 100755 --- a/ci/autoscaler/scripts/release-autoscaler.sh +++ b/ci/autoscaler/scripts/release-autoscaler.sh @@ -34,8 +34,6 @@ function create_release() { yq eval -i ".properties.\"autoscaler.apiserver.info.build\".default = \"${version}\"" jobs/golangapiserver/spec git add jobs/golangapiserver/spec - echo "${version}" VERSION - git add VERSION [ "${CI}" = "true" ] && git commit -S -m "Updated release version to ${version} in golangapiserver" # shellcheck disable=SC2086 diff --git a/scripts/local_release_autoscaler.sh b/scripts/local_release_autoscaler.sh index ad03881fec..675855af4a 100755 --- a/scripts/local_release_autoscaler.sh +++ b/scripts/local_release_autoscaler.sh @@ -11,7 +11,8 @@ DEST="${script_dir}/../build" BUILD_OPTS="--force" AUTOSCALER_CI_BOT_NAME="foo" AUTOSCALER_CI_BOT_EMAIL="foo@bar.baz" -PREV_VERSION="$(cat "${script_dir}/../VERSION")" +PREV_VERSION="$(yq ".properties.\"autoscaler.apiserver.info.build\".default" jobs/golangapiserver/spec)" + VERSION="$(cat "${script_dir}/../VERSION")-pre" export DEPLOYMENT @@ -29,7 +30,6 @@ if [ -z "${GITHUB_TOKEN}" ]; then exit 1 fi - find_or_create_ssh_key() { if [ -f ~/.ssh/id_ed25519 ]; then echo "ssh key already exists" From dadd4f6fbf69c30f0c6ebabba97e55c53cb580c7 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 2 Oct 2024 16:16:20 +0200 Subject: [PATCH 19/19] Fix typo --- ci/autoscaler/scripts/release-autoscaler.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/autoscaler/scripts/release-autoscaler.sh b/ci/autoscaler/scripts/release-autoscaler.sh index 99eda04e45..eb88e94ee0 100755 --- a/ci/autoscaler/scripts/release-autoscaler.sh +++ b/ci/autoscaler/scripts/release-autoscaler.sh @@ -169,7 +169,7 @@ releases: sha1: sha256:${ACCEPTANCE_SHA256} - name: app-autoscaler-mtar version: ${VERSION} - url: https://storage.googleapis.com/app-autoscaler-releases/releases/app-autoscaler-release-v${VERSION}.tgz + url: https://storage.googleapis.com/app-autoscaler-releases/releases/app-autoscaler-release-v${VERSION}.mtar sha1: sha256:${MTAR_SHA256} \`\`\` EOF