diff --git a/cmd/synchronization/app/BUILD.bazel b/cmd/synchronization/app/BUILD.bazel index 892f839d..4400865b 100644 --- a/cmd/synchronization/app/BUILD.bazel +++ b/cmd/synchronization/app/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//pkg/composition/client:go_default_library", "//pkg/k8s:go_default_library", "//pkg/k8s/updater:go_default_library", + "//pkg/opsgenie:go_default_library", "//pkg/options:go_default_library", "//pkg/releases:go_default_library", "//pkg/releases/deployinator/client:go_default_library", diff --git a/cmd/synchronization/app/controller_constructor.go b/cmd/synchronization/app/controller_constructor.go index 25258e83..48a7f4ec 100644 --- a/cmd/synchronization/app/controller_constructor.go +++ b/cmd/synchronization/app/controller_constructor.go @@ -7,6 +7,7 @@ import ( comp_v1_client "github.com/atlassian/voyager/pkg/composition/client" "github.com/atlassian/voyager/pkg/k8s" "github.com/atlassian/voyager/pkg/k8s/updater" + "github.com/atlassian/voyager/pkg/opsgenie" "github.com/atlassian/voyager/pkg/releases" "github.com/atlassian/voyager/pkg/releases/deployinator/client" "github.com/atlassian/voyager/pkg/servicecentral" @@ -79,6 +80,10 @@ func (cc *ControllerConstructor) New(config *ctrl.Config, cctx *ctrl.Context) (* scHTTPClient := util.HTTPClient() scClient := servicecentral.NewServiceCentralClient(config.Logger, scHTTPClient, opts.ASAPClientConfig, opts.Providers.ServiceCentralURL) + // create a client for talking to Opsgenie Integration Manager + ogHTTPClient := util.HTTPClient() + ogClient := opsgenie.New(config.Logger, ogHTTPClient, opts.ASAPClientConfig, opts.Providers.OpsgenieIntegrationsManagerURL) + scErrorCounter := prometheus.NewCounter( prometheus.CounterOpts{ Namespace: config.AppName, @@ -133,6 +138,7 @@ func (cc *ControllerConstructor) New(config *ctrl.Config, cctx *ctrl.Context) (* ServiceCentral: servicecentral.NewStore(config.Logger, scClient), ReleaseManagement: releases.NewReleaseManagement(deployinatorHTTPClient, config.Logger), ClusterLocation: opts.Location.ClusterLocation(), + Opsgenie: ogClient, ConfigMapUpdater: configMapObjectUpdater, RoleBindingUpdater: roleBindingObjectUpdater, diff --git a/cmd/synchronization/app/options.go b/cmd/synchronization/app/options.go index 0fb9e74d..671d625a 100644 --- a/cmd/synchronization/app/options.go +++ b/cmd/synchronization/app/options.go @@ -21,15 +21,17 @@ type Options struct { } type Providers struct { - ServiceCentralURL *url.URL // we use custom json marshalling to read it - DeployinatorURL *url.URL + ServiceCentralURL *url.URL // we use custom json marshalling to read it + DeployinatorURL *url.URL + OpsgenieIntegrationsManagerURL *url.URL } // UnmarshalJSON unmarshals our untyped config file into a typed struct including URLs func (p *Providers) UnmarshalJSON(data []byte) error { var rawProviders struct { - ServiceCentral string `json:"serviceCentral"` - Deployinator string `json:"deployinator"` + ServiceCentral string `json:"serviceCentral"` + Deployinator string `json:"deployinator"` + OpsgenieIntegrationsManager string `json:"opsgenieIntegrationsManager"` } if err := json.Unmarshal(data, &rawProviders); err != nil { @@ -37,13 +39,24 @@ func (p *Providers) UnmarshalJSON(data []byte) error { } scURL, err := url.Parse(rawProviders.ServiceCentral) - p.ServiceCentralURL = scURL if err != nil { return errors.Wrap(err, "unable to parse Service Central URL") } + p.ServiceCentralURL = scURL + depURL, err := url.Parse(rawProviders.Deployinator) + if err != nil { + return errors.Wrap(err, "unable to parse Deployinator URL") + } p.DeployinatorURL = depURL - return errors.Wrap(err, "unable to parse Deployinator URL") + + ogUrl, err := url.Parse(rawProviders.OpsgenieIntegrationsManager) + if err != nil { + return errors.Wrap(err, "unable to parse Opsgenie Integrations Manager URL") + } + p.OpsgenieIntegrationsManagerURL = ogUrl + + return nil } func (o *Options) DefaultAndValidate() []error { @@ -54,6 +67,10 @@ func (o *Options) DefaultAndValidate() []error { allErrors = append(allErrors, errors.New("providers.serviceCentral must be a valid URL")) } + if o.Providers.OpsgenieIntegrationsManagerURL == nil { + allErrors = append(allErrors, errors.New("providers.OpsgenieIntegrationsManagerURL must be a valid URL")) + } + return allErrors } diff --git a/pkg/apis/creator/v1/types.go b/pkg/apis/creator/v1/types.go index 9ed00d26..5278c2c5 100644 --- a/pkg/apis/creator/v1/types.go +++ b/pkg/apis/creator/v1/types.go @@ -59,6 +59,7 @@ func (ss *ServiceSpec) EmailAddress() string { // +k8s:deepcopy-gen=true type ServiceMetadata struct { PagerDuty *PagerDutyMetadata `json:"pagerDuty,omitempty"` + Opsgenie *OpsgenieMetadata `json:"opsgenie,omitempty"` Bamboo *BambooMetadata `json:"bamboo,omitempty"` } @@ -100,6 +101,10 @@ type PagerDutyIntegrationMetadata struct { IntegrationKey string `json:"integrationKey,omitempty"` } +type OpsgenieMetadata struct { + Team string `json:"team,omitempty"` +} + // +k8s:deepcopy-gen=true type Compliance struct { PRGBControl *bool `json:"prgbControl,omitempty"` diff --git a/pkg/apis/orchestration/meta/BUILD.bazel b/pkg/apis/orchestration/meta/BUILD.bazel index 8104cd8b..6d828eb3 100644 --- a/pkg/apis/orchestration/meta/BUILD.bazel +++ b/pkg/apis/orchestration/meta/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["types.go"], importpath = "github.com/atlassian/voyager/pkg/apis/orchestration/meta", visibility = ["//visibility:public"], - deps = ["//:go_default_library"], + deps = [ + "//:go_default_library", + "//pkg/opsgenie:go_default_library", + ], ) diff --git a/pkg/apis/orchestration/meta/types.go b/pkg/apis/orchestration/meta/types.go index b2cc2519..7fef22a4 100644 --- a/pkg/apis/orchestration/meta/types.go +++ b/pkg/apis/orchestration/meta/types.go @@ -1,6 +1,9 @@ package meta -import "github.com/atlassian/voyager" +import ( + "github.com/atlassian/voyager" + "github.com/atlassian/voyager/pkg/opsgenie" +) const ( ConfigMapConfigKey = "config" @@ -27,9 +30,10 @@ type ServiceProperties struct { // Notification is used in the ServiceProperties. type Notifications struct { - Email string `json:"email"` - LowPriorityPagerdutyEndpoint PagerDuty `json:"lowPriority"` - PagerdutyEndpoint PagerDuty `json:"main"` + Email string `json:"email"` + LowPriorityPagerdutyEndpoint PagerDuty `json:"lowPriority"` + PagerdutyEndpoint PagerDuty `json:"main"` + OpsgenieIntegrations []opsgenie.Integration `json:"opsgenieIntegrations"` } // PagerDuty is used in the ServiceProperties. diff --git a/pkg/opsgenie/client.go b/pkg/opsgenie/client.go index 0a5eabc7..92d854a1 100644 --- a/pkg/opsgenie/client.go +++ b/pkg/opsgenie/client.go @@ -41,7 +41,7 @@ func New(logger *zap.Logger, httpClient *http.Client, asap pkiutil.ASAP, baseURL } } -// Gets OpsGenie integrations +// Gets Opsgenie integrations // return codes: // - 400: Bad request to Opsgenie // - 401: Unauthorized diff --git a/pkg/opsgenie/client_test.go b/pkg/opsgenie/client_test.go index 204f67a2..1e6831f8 100644 --- a/pkg/opsgenie/client_test.go +++ b/pkg/opsgenie/client_test.go @@ -33,7 +33,7 @@ func TestGetIntegrations(t *testing.T) { defer ogIntManagerMockServer.Close() // when - ogIntManagerClient := mockOpsGenieIntegrationManagerClient(t, ogIntManagerMockServer.URL, pkitest.MockASAPClientConfig(t)) + ogIntManagerClient := mockOpsgenieIntegrationManagerClient(t, ogIntManagerMockServer.URL, pkitest.MockASAPClientConfig(t)) _, retriable, err := ogIntManagerClient.GetOrCreateIntegrations(context.Background(), teamName) // then @@ -58,7 +58,7 @@ func TestGetIntegrationsTeamNotFound(t *testing.T) { defer ogIntManagerMockServer.Close() // when - ogIntManagerClient := mockOpsGenieIntegrationManagerClient(t, ogIntManagerMockServer.URL, pkitest.MockASAPClientConfig(t)) + ogIntManagerClient := mockOpsgenieIntegrationManagerClient(t, ogIntManagerMockServer.URL, pkitest.MockASAPClientConfig(t)) _, retriable, err := ogIntManagerClient.GetOrCreateIntegrations(context.Background(), teamName) // then @@ -83,7 +83,7 @@ func TestGetIntegrationsRateLimited(t *testing.T) { defer ogIntManagerMockServer.Close() // when - ogIntManagerClient := mockOpsGenieIntegrationManagerClient(t, ogIntManagerMockServer.URL, pkitest.MockASAPClientConfig(t)) + ogIntManagerClient := mockOpsgenieIntegrationManagerClient(t, ogIntManagerMockServer.URL, pkitest.MockASAPClientConfig(t)) _, retriable, err := ogIntManagerClient.GetOrCreateIntegrations(context.Background(), teamName) // then @@ -108,7 +108,7 @@ func TestGetIntegrationsInternalServerError(t *testing.T) { defer ogIntManagerMockServer.Close() // when - ogIntManagerClient := mockOpsGenieIntegrationManagerClient(t, ogIntManagerMockServer.URL, pkitest.MockASAPClientConfig(t)) + ogIntManagerClient := mockOpsgenieIntegrationManagerClient(t, ogIntManagerMockServer.URL, pkitest.MockASAPClientConfig(t)) _, retriable, err := ogIntManagerClient.GetOrCreateIntegrations(context.Background(), teamName) // then @@ -117,7 +117,7 @@ func TestGetIntegrationsInternalServerError(t *testing.T) { require.True(t, retriable) } -func mockOpsGenieIntegrationManagerClient(t *testing.T, serverMockAddress string, asap pkiutil.ASAP) *Client { +func mockOpsgenieIntegrationManagerClient(t *testing.T, serverMockAddress string, asap pkiutil.ASAP) *Client { opsgenieIntegrationManagerURL, err := url.Parse(serverMockAddress) require.NoError(t, err) httpClient := util.HTTPClient() diff --git a/pkg/opsgenie/it/client_manual_test.go b/pkg/opsgenie/it/client_manual_test.go index 94d102f0..639a2e65 100644 --- a/pkg/opsgenie/it/client_manual_test.go +++ b/pkg/opsgenie/it/client_manual_test.go @@ -17,7 +17,7 @@ import ( ) const ( - opsGenieIntManURL = "https://micros.prod.atl-paas.net" + opsgenieIntManURL = "https://micros.prod.atl-paas.net" ) // NOTE: THIS WILL CREATE INTEGRATIONS IF NONE EXIST @@ -32,7 +32,7 @@ func TestGetIntegrations(t *testing.T) { require.NoError(t, asapErr) client := util.HTTPClient() - c := opsgenie.New(testLogger, client, asapConfig, parseURL(t, opsGenieIntManURL)) + c := opsgenie.New(testLogger, client, asapConfig, parseURL(t, opsgenieIntManURL)) // Get Service Attributes resp, _, err := c.GetOrCreateIntegrations(ctx, "Platform SRE") diff --git a/pkg/opsgenie/types.go b/pkg/opsgenie/types.go index 32fcecbc..ae09ea74 100644 --- a/pkg/opsgenie/types.go +++ b/pkg/opsgenie/types.go @@ -5,13 +5,22 @@ type IntegrationsResponse struct { } type Integration struct { - ID string `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - TeamID string `json:"teamId"` - TeamName string `json:"teamName"` - Priority string `json:"priority"` - APIKey string `json:"apiKey"` - Endpoint string `json:"endpoint"` - EnvType string `json:"envType"` + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + TeamID string `json:"teamId"` + TeamName string `json:"teamName"` + Priority string `json:"priority"` + APIKey string `json:"apiKey"` + Endpoint string `json:"endpoint"` + EnvType EnvType `json:"envType"` } + +type EnvType string + +const ( + EnvTypeDev EnvType = "dev" + EnvTypeStaging EnvType = "staging" + EnvTypeProd EnvType = "prod" + EnvTypeGlobal EnvType = "null" // Intentionally a string called "null" as this is the expected result from opsgenie int manager +) diff --git a/pkg/orchestration/wiring/BUILD.bazel b/pkg/orchestration/wiring/BUILD.bazel index 129ac666..7bdd16d4 100644 --- a/pkg/orchestration/wiring/BUILD.bazel +++ b/pkg/orchestration/wiring/BUILD.bazel @@ -34,6 +34,7 @@ go_test( "//cmd/smith/config:go_default_library", "//pkg/apis/orchestration/meta:go_default_library", "//pkg/apis/orchestration/v1:go_default_library", + "//pkg/opsgenie:go_default_library", "//pkg/orchestration/wiring/registry:go_default_library", "//pkg/orchestration/wiring/wiringplugin:go_default_library", "//pkg/orchestration/wiring/wiringutil/knownshapes:go_default_library", diff --git a/pkg/orchestration/wiring/entangler_test.go b/pkg/orchestration/wiring/entangler_test.go index c70921a9..f5c4a269 100644 --- a/pkg/orchestration/wiring/entangler_test.go +++ b/pkg/orchestration/wiring/entangler_test.go @@ -13,6 +13,7 @@ import ( smith_config "github.com/atlassian/voyager/cmd/smith/config" orch_meta "github.com/atlassian/voyager/pkg/apis/orchestration/meta" orch_v1 "github.com/atlassian/voyager/pkg/apis/orchestration/v1" + "github.com/atlassian/voyager/pkg/opsgenie" "github.com/atlassian/voyager/pkg/orchestration/wiring/registry" "github.com/atlassian/voyager/pkg/orchestration/wiring/wiringplugin" "github.com/atlassian/voyager/pkg/orchestration/wiring/wiringutil/knownshapes" @@ -303,6 +304,41 @@ func entangleTestState(t *testing.T, state *orch_v1.State, wiringPlugins map[voy CloudWatch: "https://events.pagerduty.com/adapter/cloudwatch_sns/v1/12312312312312312312312312312312", Generic: "123123123123123", }, + OpsgenieIntegrations: []opsgenie.Integration{ + { + APIKey: "SOME-API-KEY-HERE", + Endpoint: "https://api.opsgenie.com/v1/json/cloudwatch?apiKey=SOME-API-KEY-HERE", + EnvType: "dev", + ID: "6a33291e-a4d9-467e-a25d-f9d621fe2461", + Name: "micros_CloudWatch_high_dev", + Priority: "high", + TeamID: "01101000 01101001 00100000 01101101 01101111 01101101", + TeamName: "Platform SRE", + Type: "Cloudwatch", + }, + { + APIKey: "SOME-API-KEY-HERE", + Endpoint: "null", + EnvType: "dev", + ID: "6a33291e-a4d9-467e-a25d-f9d621fe2461", + Name: "micros_Platform SRE_Datadog", + Priority: "null", + TeamID: "01101000 01101001 00100000 01101101 01101111 01101101", + TeamName: "Platform SRE", + Type: "Datadog", + }, + { + APIKey: "SOME-API-KEY-HERE", + Endpoint: "null", + EnvType: "null", + ID: "6a33291e-a4d9-467e-a25d-f9d621fe2461", + Name: "micros_Platform SRE_API", + Priority: "null", + TeamID: "01101000 01101001 00100000 01101101 01101111 01101101", + TeamName: "Platform SRE", + Type: "API", + }, + }, }, SSAMAccessLevel: "access-level-from-configmap", LoggingID: "logging-id-from-configmap", diff --git a/pkg/servicecentral/client.go b/pkg/servicecentral/client.go index e48ce01c..c4cd28ff 100644 --- a/pkg/servicecentral/client.go +++ b/pkg/servicecentral/client.go @@ -300,14 +300,8 @@ func (c *Client) GetService(ctx context.Context, user auth.OptionalUser, service if err != nil { return nil, errors.Wrap(err, "failed to get attributes for service") } - ogTeamAttr, found, err := findOpsGenieTeamServiceAttribute(resp) - if err != nil { - return nil, errors.Wrap(err, "failed to get OpsGenie attributes for service") - } - if found { - service.Attributes = append(service.Attributes, ogTeamAttr) - } + service.Attributes = resp return service, nil } @@ -344,7 +338,7 @@ func (c *Client) DeleteService(ctx context.Context, user auth.User, serviceUUID } // GetServiceAttributes queries service central for the attributes of a given service. Can return an empty array if no attributes were found -func (c *Client) GetServiceAttributes(ctx context.Context, user auth.OptionalUser, serviceUUID string) ([]ServiceAttributeResponse, error) { +func (c *Client) GetServiceAttributes(ctx context.Context, user auth.OptionalUser, serviceUUID string) ([]ServiceAttribute, error) { req, err := c.rm.NewRequest( pkiutil.AuthenticateWithASAP(c.asap, asapAudience, user.NameOrElse(noUser)), restclient.Method(http.MethodGet), @@ -371,14 +365,13 @@ func (c *Client) GetServiceAttributes(ctx context.Context, user auth.OptionalUse message := fmt.Sprintf("failed to get attributes for service %q. Response: %s", serviceUUID, respBody) return nil, clientError(response.StatusCode, message) } - - var parsedBody []ServiceAttributeResponse - err = json.Unmarshal(respBody, &parsedBody) + var svcAttrResp []ServiceAttribute + err = json.Unmarshal(respBody, &svcAttrResp) if err != nil { return nil, errors.Wrap(err, "failed to unmarshal response body") } - return parsedBody, nil + return svcAttrResp, nil } func clientError(statusCode int, message string) error { @@ -445,27 +438,28 @@ func convertV2ServiceToV1(v2Service V2Service) ServiceDataRead { return service } -func findOpsGenieTeamServiceAttribute(attributes []ServiceAttributeResponse) (ServiceAttribute, bool /*found*/, error) { - const opsGenieSchemaName = "opsgenie" +// findOpsgenieAttribute searches a given list of ServiceAttributes for a single OpsgenieAttribute +func findOpsgenieAttribute(attributes []ServiceAttribute) (OpsgenieAttribute, bool /*found*/, error) { + const opsgenieSchemaName = "opsgenie" count := 0 found := false - ogTeamAttr := ServiceAttribute{} + ogTeamAttr := OpsgenieAttribute{} for _, attr := range attributes { - if attr.Schema.Name != opsGenieSchemaName { + if attr.Schema.Name != opsgenieSchemaName { continue } team, ok := attr.Value["team"] if !ok { - return ogTeamAttr, found, errors.Errorf("expected to find team name within schema of name %q", opsGenieSchemaName) + return ogTeamAttr, found, errors.Errorf("expected to find team name within schema of name %q", opsgenieSchemaName) } - ogTeamAttr = ServiceAttribute{Team: team} + ogTeamAttr = OpsgenieAttribute{Team: team} found = true count++ } if count > 1 { - return ogTeamAttr, found, errors.New("found more than one OpsGenie service attribute") + return ogTeamAttr, found, errors.New("found more than one Opsgenie service attribute") } return ogTeamAttr, found, nil } diff --git a/pkg/servicecentral/client_test.go b/pkg/servicecentral/client_test.go index 15d99f87..90437cdd 100644 --- a/pkg/servicecentral/client_test.go +++ b/pkg/servicecentral/client_test.go @@ -275,7 +275,7 @@ func TestGetService(t *testing.T) { require.Equal(t, 2, handler.RequestSnapshots.Calls()) } -func TestGetServiceWithOpsGenieAttribute(t *testing.T) { +func TestGetServiceWithOpsgenieAttribute(t *testing.T) { t.Parallel() // given handler := MockHandler(Match( @@ -302,11 +302,12 @@ func TestGetServiceWithOpsGenieAttribute(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, handler.RequestSnapshots.Calls()) - require.Equal(t, 1, len(service.Attributes)) - require.Equal(t, "Platform SRE", service.Attributes[0].Team) + require.Equal(t, 2, len(service.Attributes)) + require.Equal(t, "Platform SRE", service.Attributes[0].Value["team"]) + require.Equal(t, "Fake News", service.Attributes[1].Value["team"]) } -func TestGetServiceWithEmptyOpsGenieAttribute(t *testing.T) { +func TestGetServiceWithEmptyOpsgenieAttribute(t *testing.T) { t.Parallel() // given handler := MockHandler(Match( @@ -334,10 +335,10 @@ func TestGetServiceWithEmptyOpsGenieAttribute(t *testing.T) { require.Equal(t, 2, handler.RequestSnapshots.Calls()) require.Equal(t, 1, len(service.Attributes)) - require.Equal(t, "", service.Attributes[0].Team) + require.Equal(t, "", service.Attributes[0].Value["Team"]) } -func TestGetServiceWithoutOpsGenieAttribute(t *testing.T) { +func TestGetServiceWithoutOpsgenieAttribute(t *testing.T) { t.Parallel() // given handler := MockHandler(Match( @@ -393,34 +394,6 @@ func TestGetServiceWithFailedAttributesCall(t *testing.T) { require.Equal(t, 2, handler.RequestSnapshots.Calls()) } -func TestGetServiceWithMultipleOpsGenieAttribute(t *testing.T) { - t.Parallel() - // given - handler := MockHandler(Match( - Method(http.MethodGet), - Path(fmt.Sprintf("%s/%s", v1ServicesPath, testServiceName)), - ).Respond( - Status(http.StatusOK), - JSONFromFile(t, "get_service.rsp.json"), - ), - Match( - Method(http.MethodGet), - Path(fmt.Sprintf("%s/%s/attributes", v2ServicesPath, testServiceName)), - ).Respond( - Status(http.StatusOK), - JSONFromFile(t, "get_service_attributes_multiple_opsgenie.rsp.json"), - )) - serviceCentralServerMock := httptest.NewServer(handler) - defer serviceCentralServerMock.Close() - // when - serviceCentralClient := testServiceCentralClient(t, serviceCentralServerMock.URL, pkitest.MockASAPClientConfig(t)) - _, err := serviceCentralClient.GetService(context.Background(), optionalUser, string(testServiceName)) - - // then - require.Error(t, err) - require.Equal(t, 2, handler.RequestSnapshots.Calls()) -} - func testServiceCentralClient(t *testing.T, serviceCentralServerMockAddress string, asap pkiutil.ASAP) *Client { serviceCentralURL, err := url.Parse(serviceCentralServerMockAddress) require.NoError(t, err) diff --git a/pkg/servicecentral/metadata.go b/pkg/servicecentral/metadata.go index de0e21a1..5fabe30a 100644 --- a/pkg/servicecentral/metadata.go +++ b/pkg/servicecentral/metadata.go @@ -29,6 +29,16 @@ func SetPagerDutyMetadata(serviceCentralData *ServiceDataWrite, m *creator_v1.Pa return setMetadata(serviceCentralData, PagerDutyMetadataKey, m) } +// GetOpsgenieAttribute reads the Opsgenie team attribute out of a service +func GetOpsgenieAttribute(serviceCentralData *ServiceDataRead) (*creator_v1.OpsgenieMetadata, error) { + attributes := serviceCentralData.Attributes + ogTeamAttr, found, err := findOpsgenieAttribute(attributes) + if err != nil || !found { + return nil, err + } + return &creator_v1.OpsgenieMetadata{Team: ogTeamAttr.Team}, nil +} + // GetBambooMetadata reads the allowed builds metadata out of a service func GetBambooMetadata(serviceCentralData *ServiceDataWrite) (*creator_v1.BambooMetadata, error) { var m creator_v1.BambooMetadata diff --git a/pkg/servicecentral/store.go b/pkg/servicecentral/store.go index 2ece71e8..197f37ab 100644 --- a/pkg/servicecentral/store.go +++ b/pkg/servicecentral/store.go @@ -306,6 +306,14 @@ func serviceDataToService(data *ServiceDataRead) (*creator_v1.Service, error) { service.Spec.Metadata.PagerDuty = pagerDutyMetadata } + ogMetadata, err := GetOpsgenieAttribute(data) + if err != nil { + return nil, err + } + if ogMetadata != nil { + service.Spec.Metadata.Opsgenie = ogMetadata + } + bambooMetadata, err := GetBambooMetadata(&data.ServiceDataWrite) if err != nil { return nil, err diff --git a/pkg/servicecentral/testdata/get_service_attributes_multiple_opsgenie.rsp.json b/pkg/servicecentral/testdata/get_service_attributes_multiple_opsgenie.rsp.json deleted file mode 100644 index f06c583f..00000000 --- a/pkg/servicecentral/testdata/get_service_attributes_multiple_opsgenie.rsp.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "createdBy": "fcobb", - "createdOn": "2019-01-29T04:08:15.810106Z", - "id": 23, - "modifiedBy": "fcobb", - "modifiedOn": "2019-01-29T04:08:15.810104Z", - "schema": { - "id": 6, - "name": "opsgenie", - "ref": "/api/v2/schemas/attributes/6" - }, - "service": { - "name": "slime", - "ref": "/api/v2/services/175096a2-6b18-4df4-8bd0-65fe6e0c56b4", - "uuid": "175096a2-6b18-4df4-8bd0-65fe6e0c56b4" - }, - "value": { - "team": "Platform SRE" - } - }, - { - "createdBy": "fcobb", - "createdOn": "2019-01-29T04:08:15.810106Z", - "id": 23, - "modifiedBy": "fcobb", - "modifiedOn": "2019-01-29T04:08:15.810104Z", - "schema": { - "id": 6, - "name": "opsgenie", - "ref": "/api/v2/schemas/attributes/6" - }, - "service": { - "name": "slime", - "ref": "/api/v2/services/175096a2-6b18-4df4-8bd0-65fe6e0c56b4", - "uuid": "175096a2-6b18-4df4-8bd0-65fe6e0c56b4" - }, - "value": { - "team": "Voyager" - } - } -] diff --git a/pkg/servicecentral/types.go b/pkg/servicecentral/types.go index a721fbe1..e42e02dc 100644 --- a/pkg/servicecentral/types.go +++ b/pkg/servicecentral/types.go @@ -44,10 +44,6 @@ type ServiceDataRead struct { } type ServiceAttribute struct { - Team string `json:"team,omitempty"` -} - -type ServiceAttributeResponse struct { ID int `json:"id"` Service ServiceAttributeService `json:"service"` Schema ServiceAttributeSchema `json:"schema"` @@ -70,6 +66,10 @@ type ServiceAttributeSchema struct { Name string `json:"name"` } +type OpsgenieAttribute struct { + Team string `json:"team,omitempty"` +} + // ServiceComplianceConf includes all service compliance related data type ServiceComplianceConf struct { // using prgb_control instead of prgbControl here to match the response field name diff --git a/pkg/synchronization/BUILD.bazel b/pkg/synchronization/BUILD.bazel index dc0ba44c..0e495d23 100644 --- a/pkg/synchronization/BUILD.bazel +++ b/pkg/synchronization/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "//pkg/composition/client:go_default_library", "//pkg/k8s:go_default_library", "//pkg/k8s/updater:go_default_library", + "//pkg/opsgenie:go_default_library", "//pkg/orchestration/wiring/k8scompute/api:go_default_library", "//pkg/pagerduty:go_default_library", "//pkg/releases:go_default_library", @@ -67,6 +68,7 @@ go_test( "//pkg/k8s:go_default_library", "//pkg/k8s/testing:go_default_library", "//pkg/k8s/updater:go_default_library", + "//pkg/opsgenie:go_default_library", "//pkg/pagerduty:go_default_library", "//pkg/releases:go_default_library", "//pkg/servicecentral:go_default_library", diff --git a/pkg/synchronization/controller.go b/pkg/synchronization/controller.go index 2a6e0c8c..25f2bcdb 100644 --- a/pkg/synchronization/controller.go +++ b/pkg/synchronization/controller.go @@ -17,6 +17,7 @@ import ( compClient "github.com/atlassian/voyager/pkg/composition/client" "github.com/atlassian/voyager/pkg/k8s" "github.com/atlassian/voyager/pkg/k8s/updater" + "github.com/atlassian/voyager/pkg/opsgenie" apik8scompute "github.com/atlassian/voyager/pkg/orchestration/wiring/k8scompute/api" "github.com/atlassian/voyager/pkg/pagerduty" "github.com/atlassian/voyager/pkg/releases" @@ -54,7 +55,9 @@ const ( // see https://github.com/jtblin/kube2iam#namespace-restrictions allowedRolesAnnotation = "iam.amazonaws.com/allowed-roles" - maxSyncWorkers = 10 + maxSyncWorkers = 10 + defaultPagerdutyGeneric = "5d11612f25b840faaf77422edeff9c76" + defaultPagerdutyCloudwatch = "https://events.pagerduty.com/adapter/cloudwatch_sns/v1/124e0f010f214a9b9f30b768e7b18e69" ) const ( @@ -73,6 +76,10 @@ type ServiceMetadataStore interface { ListModifiedServices(ctx context.Context, user auth.OptionalUser, modifiedSince time.Time) ([]creator_v1.Service, error) } +type OpsgenieIntegrationManagerClient interface { + GetOrCreateIntegrations(ctx context.Context, teamName string) (*opsgenie.IntegrationsResponse, bool /* retriable */, error) +} + type Controller struct { Logger *zap.Logger ReadyForWork func() @@ -85,6 +92,7 @@ type Controller struct { ServiceCentral ServiceMetadataStore ReleaseManagement releases.ReleaseManagementStore ClusterLocation voyager.ClusterLocation + Opsgenie OpsgenieIntegrationManagerClient RoleBindingUpdater updater.ObjectUpdater ConfigMapUpdater updater.ObjectUpdater @@ -404,9 +412,9 @@ func (c *Controller) createOrUpdateServiceMetadata(logger *zap.Logger, ns *core_ tags[k] = v } - notifications, err := c.buildNotifications(serviceData.Spec) + notifications, retriable, err := c.buildNotifications(serviceData.Spec) if err != nil { - return false, err + return retriable, err } metadata := orch_meta.ServiceProperties{ @@ -459,41 +467,73 @@ func (c *Controller) getServiceData(user auth.OptionalUser, name voyager.Service return c.ServiceCentral.GetService(context.Background(), user, servicecentral.ServiceName(name)) } -func (c *Controller) buildNotifications(spec creator_v1.ServiceSpec) (*orch_meta.Notifications, error) { +func (c *Controller) buildNotifications(spec creator_v1.ServiceSpec) (*orch_meta.Notifications, bool /* retriable */, error) { + // Default pagerduty values re-used from Micros config.js + notifications := orch_meta.Notifications{ + Email: spec.EmailAddress(), + PagerdutyEndpoint: orch_meta.PagerDuty{ + Generic: defaultPagerdutyGeneric, + CloudWatch: defaultPagerdutyCloudwatch, + }, + LowPriorityPagerdutyEndpoint: orch_meta.PagerDuty{ + Generic: defaultPagerdutyGeneric, + CloudWatch: defaultPagerdutyCloudwatch, + }, + } + pdEnvMetadata, ok, err := pagerDutyForEnvType(spec.Metadata.PagerDuty, c.ClusterLocation.EnvType) if err != nil { - return nil, errors.Wrap(err, "error building notifications for servicemetadata") + return nil, false, errors.Wrap(err, "error building pagerduty notifications for servicemetadata") } if ok { mainPD, err := convertPagerDuty(pdEnvMetadata.Main) if err != nil { - return nil, errors.Wrap(err, "cannot convert main pagerduty entry") + return nil, false, errors.Wrap(err, "cannot convert main pagerduty entry") } lowPriPD, err := convertPagerDuty(pdEnvMetadata.LowPriority) if err != nil { - return nil, errors.Wrap(err, "cannot convert low priority pagerduty entry") + return nil, false, errors.Wrap(err, "cannot convert low priority pagerduty entry") } - return &orch_meta.Notifications{ - Email: spec.EmailAddress(), - PagerdutyEndpoint: *mainPD, - LowPriorityPagerdutyEndpoint: *lowPriPD, - }, nil + notifications.PagerdutyEndpoint = *mainPD + notifications.LowPriorityPagerdutyEndpoint = *lowPriPD } - // Default values re-used from Micros config.js - return &orch_meta.Notifications{ - Email: spec.EmailAddress(), - PagerdutyEndpoint: orch_meta.PagerDuty{ - Generic: "5d11612f25b840faaf77422edeff9c76", - CloudWatch: "https://events.pagerduty.com/adapter/cloudwatch_sns/v1/124e0f010f214a9b9f30b768e7b18e69", - }, - LowPriorityPagerdutyEndpoint: orch_meta.PagerDuty{ - Generic: "5d11612f25b840faaf77422edeff9c76", - CloudWatch: "https://events.pagerduty.com/adapter/cloudwatch_sns/v1/124e0f010f214a9b9f30b768e7b18e69", - }, - }, nil + ogInts, retriable, err := c.buildOpsgenieNotifications(spec.Metadata) + if err != nil { + return nil, retriable, err + } + notifications.OpsgenieIntegrations = ogInts + + return ¬ifications, true, nil +} + +func (c *Controller) buildOpsgenieNotifications(metadata creator_v1.ServiceMetadata) ([]opsgenie.Integration, bool /* retriable */, error) { + ogInts, retriable, err := c.getOpsgenieIntegrations(metadata.Opsgenie) + if err != nil { + return nil, retriable, errors.Wrap(err, "failed to get Opsgenie notifications") + } + envOgInts, err := filterOpsgenieIntegrationsByEnv(ogInts, c.ClusterLocation.EnvType) + if err != nil { + return nil, false, errors.Wrap(err, "failed to build Opsgenie notifications") + } + return envOgInts, true, nil +} + +// getOpsgenieIntegrations attempts to get Opsgenie integrations from the opsgenie integration manager +func (c *Controller) getOpsgenieIntegrations(metadata *creator_v1.OpsgenieMetadata) ([]opsgenie.Integration, bool /* retriable */, error) { + // Opsgenie is optional + if metadata == nil { + return nil, true, nil + } + + resp, retriable, err := c.Opsgenie.GetOrCreateIntegrations(context.TODO(), metadata.Team) + if err != nil { + return nil, retriable, err + } + + return resp.Integrations, true, nil } func (c *Controller) setupDockerSecret(logger *zap.Logger, namespaceName string) (bool /* retriable */, error) { @@ -665,6 +705,42 @@ func pagerDutyForEnvType(pagerduty *creator_v1.PagerDutyMetadata, envType voyage } } +// filterOpsgenieIntegrationsByEnv filters a given list of integrations by the EnvType +func filterOpsgenieIntegrationsByEnv(integrations []opsgenie.Integration, envType voyager.EnvType) ([]opsgenie.Integration, error) { + if len(integrations) == 0 { + return nil, nil // Not an error as Opsgenie is optional + } + + filtered := make([]opsgenie.Integration, 0, 4) + + for _, integration := range integrations { + + if integration.EnvType == opsgenie.EnvTypeGlobal { + filtered = append(filtered, integration) + continue + } + + switch envType { + case voyager.EnvTypeStaging: + if integration.EnvType == opsgenie.EnvTypeStaging { + filtered = append(filtered, integration) + } + case voyager.EnvTypeProduction: + if integration.EnvType == opsgenie.EnvTypeProd { + filtered = append(filtered, integration) + } + case voyager.EnvTypeDev: + if integration.EnvType == opsgenie.EnvTypeDev { + filtered = append(filtered, integration) + } + default: + return nil, errors.Errorf("unexpected envType %q when filtering Opsgenie integrations", envType) + } + } + + return filtered, nil +} + func NsServiceLabelIndexFunc(obj interface{}) ([]string, error) { ns := obj.(*core_v1.Namespace) serviceName, err := layers.ServiceNameFromNamespaceLabels(ns.Labels) diff --git a/pkg/synchronization/controller_test.go b/pkg/synchronization/controller_test.go index de0eb2cc..42de0458 100644 --- a/pkg/synchronization/controller_test.go +++ b/pkg/synchronization/controller_test.go @@ -18,6 +18,7 @@ import ( "github.com/atlassian/voyager/pkg/k8s" k8s_testing "github.com/atlassian/voyager/pkg/k8s/testing" "github.com/atlassian/voyager/pkg/k8s/updater" + "github.com/atlassian/voyager/pkg/opsgenie" "github.com/atlassian/voyager/pkg/pagerduty" "github.com/atlassian/voyager/pkg/releases" "github.com/atlassian/voyager/pkg/servicecentral" @@ -72,6 +73,15 @@ func (m *fakeServiceCentral) ListModifiedServices(ctx context.Context, user auth return args.Get(0).([]creator_v1.Service), args.Error(1) } +type fakeOpsgenie struct { + mock.Mock +} + +func (m *fakeOpsgenie) GetOrCreateIntegrations(ctx context.Context, teamName string) (*opsgenie.IntegrationsResponse, bool /* retriable */, error) { + args := m.Called(ctx, teamName) + return args.Get(0).(*opsgenie.IntegrationsResponse), args.Bool(1), args.Error(2) +} + type fakeReleaseManagement struct { serviceName voyager.ServiceName serviceNames []voyager.ServiceName @@ -1272,6 +1282,253 @@ func TestGenerateIamRoleGlob(t *testing.T) { } } +func TestOpsGenieWhenIntegrationManagerFails(t *testing.T) { + t.Parallel() + ns := &core_v1.Namespace{ + TypeMeta: meta_v1.TypeMeta{ + Kind: k8s.NamespaceKind, + APIVersion: core_v1.SchemeGroupVersion.String(), + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: namespaceName, + Labels: map[string]string{ + voyager.ServiceNameLabel: serviceName, + }, + }, + } + + tc := testCase{ + serviceName: serviceNameVoy, + ns: ns, + mainClientObjects: []runtime.Object{ns, existingDefaultDockerSecret()}, + test: func(t *testing.T, cntrlr *Controller, ctx *ctrl.ProcessContext, tc *testCase) { + service := &creator_v1.Service{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: serviceName, + }, + Spec: creator_v1.ServiceSpec{ + ResourceOwner: "somebody", + BusinessUnit: "the unit", + LoggingID: "some-logging-id", + Metadata: creator_v1.ServiceMetadata{ + PagerDuty: &creator_v1.PagerDutyMetadata{}, + Opsgenie: &creator_v1.OpsgenieMetadata{Team: "Platform SRE"}, + }, + SSAMContainerName: "some-ssam-container", + ResourceTags: map[voyager.Tag]string{ + "foo": "bar", + "baz": "blah", + }, + }, + } + tc.scFake.On("GetService", mock.Anything, auth.NoUser(), serviceNameSc).Return(service, nil) + + var nilResp *opsgenie.IntegrationsResponse + // Return error when calling Opsgenie Integration Manager + tc.ogFake.On("GetOrCreateIntegrations", mock.Anything, mock.Anything).Return(nilResp, true, errors.New("some error")) + + _, err := cntrlr.Process(ctx) + require.Error(t, err) + }, + } + tc.run(t) +} + +func TestOpsGenieWhenNoTeam(t *testing.T) { + t.Parallel() + ns := &core_v1.Namespace{ + TypeMeta: meta_v1.TypeMeta{ + Kind: k8s.NamespaceKind, + APIVersion: core_v1.SchemeGroupVersion.String(), + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: namespaceName, + Labels: map[string]string{ + voyager.ServiceNameLabel: serviceName, + }, + }, + } + + tc := testCase{ + serviceName: serviceNameVoy, + ns: ns, + mainClientObjects: []runtime.Object{ns, existingDefaultDockerSecret()}, + test: func(t *testing.T, cntrlr *Controller, ctx *ctrl.ProcessContext, tc *testCase) { + service := &creator_v1.Service{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: serviceName, + }, + Spec: creator_v1.ServiceSpec{ + ResourceOwner: "somebody", + BusinessUnit: "the unit", + LoggingID: "some-logging-id", + Metadata: creator_v1.ServiceMetadata{ + PagerDuty: &creator_v1.PagerDutyMetadata{}, + }, + SSAMContainerName: "some-ssam-container", + ResourceTags: map[voyager.Tag]string{ + "foo": "bar", + "baz": "blah", + }, + }, + } + tc.scFake.On("GetService", mock.Anything, auth.NoUser(), serviceNameSc).Return(service, nil) + _, err := cntrlr.Process(ctx) + require.NoError(t, err) // Expect no error as Opsgenie team is optional + }, + } + tc.run(t) +} + +func TestBuildOpsGenieNotifications(t *testing.T) { + t.Parallel() + + fullPagerDutyMetadata := fullPagerDutyMetadata() + + envTypeCases := []struct { + envType voyager.EnvType + pagerDutyEnvMetadata creator_v1.PagerDutyEnvMetadata + expectedOgInts []opsgenie.Integration + }{ + { + voyager.EnvTypeDev, + creator_v1.PagerDutyEnvMetadata{ + Main: creator_v1.PagerDutyServiceMetadata{ + Integrations: creator_v1.PagerDutyIntegrations{ + CloudWatch: creator_v1.PagerDutyIntegrationMetadata{ + IntegrationKey: "124e0f010f214a9b9f30b768e7b18e69", + }, + Generic: creator_v1.PagerDutyIntegrationMetadata{ + IntegrationKey: defaultPagerdutyGeneric, + }, + }, + }, + LowPriority: creator_v1.PagerDutyServiceMetadata{ + Integrations: creator_v1.PagerDutyIntegrations{ + CloudWatch: creator_v1.PagerDutyIntegrationMetadata{ + IntegrationKey: "124e0f010f214a9b9f30b768e7b18e69", + }, + Generic: creator_v1.PagerDutyIntegrationMetadata{ + IntegrationKey: defaultPagerdutyGeneric, + }, + }, + }, + }, + []opsgenie.Integration{ + {EnvType: opsgenie.EnvTypeDev}, + {EnvType: opsgenie.EnvTypeGlobal}, + }, + }, + { + voyager.EnvTypeStaging, + fullPagerDutyMetadata.Staging, + []opsgenie.Integration{ + {EnvType: opsgenie.EnvTypeStaging}, + {EnvType: opsgenie.EnvTypeGlobal}, + }, + }, + { + voyager.EnvTypeProduction, + fullPagerDutyMetadata.Production, + []opsgenie.Integration{ + {EnvType: opsgenie.EnvTypeProd}, + {EnvType: opsgenie.EnvTypeGlobal}, + }, + }, + } + + for _, subCase := range envTypeCases { + t.Run(string(subCase.envType), func(t *testing.T) { + + ns := &core_v1.Namespace{ + TypeMeta: meta_v1.TypeMeta{ + Kind: k8s.NamespaceKind, + APIVersion: core_v1.SchemeGroupVersion.String(), + }, + ObjectMeta: meta_v1.ObjectMeta{ + Name: namespaceName, + Labels: map[string]string{ + voyager.ServiceNameLabel: serviceName, + }, + }, + } + + tc := testCase{ + ns: ns, + mainClientObjects: []runtime.Object{ns, existingDefaultDockerSecret()}, + test: func(t *testing.T, cntrlr *Controller, ctx *ctrl.ProcessContext, tc *testCase) { + service := &creator_v1.Service{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: serviceName, + }, + Spec: creator_v1.ServiceSpec{ + ResourceOwner: "somebody", + BusinessUnit: "the unit", + Metadata: creator_v1.ServiceMetadata{ + PagerDuty: fullPagerDutyMetadata, + Opsgenie: &creator_v1.OpsgenieMetadata{Team: "Platform SRE"}, + }, + }, + } + expected := basicServiceProperties(service, subCase.envType) + //expected Pagerduty Notifications + cwURL, err := pagerduty.KeyToCloudWatchURL(subCase.pagerDutyEnvMetadata.Main.Integrations.CloudWatch.IntegrationKey) + require.NoError(t, err) + expected.Notifications.PagerdutyEndpoint = orch_meta.PagerDuty{ + Generic: subCase.pagerDutyEnvMetadata.Main.Integrations.Generic.IntegrationKey, + CloudWatch: cwURL, + } + cwURL, err = pagerduty.KeyToCloudWatchURL(subCase.pagerDutyEnvMetadata.LowPriority.Integrations.CloudWatch.IntegrationKey) + require.NoError(t, err) + expected.Notifications.LowPriorityPagerdutyEndpoint = orch_meta.PagerDuty{ + Generic: subCase.pagerDutyEnvMetadata.LowPriority.Integrations.Generic.IntegrationKey, + CloudWatch: cwURL, + } + //expected Opsgenie Notifications + expected.Notifications.OpsgenieIntegrations = subCase.expectedOgInts + + tc.scFake.On("GetService", mock.Anything, auth.NoUser(), serviceNameSc).Return(service, nil) + ogResp := &opsgenie.IntegrationsResponse{ + Integrations: []opsgenie.Integration{ + {EnvType: opsgenie.EnvTypeDev}, + {EnvType: opsgenie.EnvTypeStaging}, + {EnvType: opsgenie.EnvTypeProd}, + {EnvType: opsgenie.EnvTypeGlobal}, + }, + } + + // Return error when calling Opsgenie Integration Manager + tc.ogFake.On("GetOrCreateIntegrations", mock.Anything, mock.Anything).Return(ogResp, true, nil) + + // make sure the controller knows we are our specific environment type + cntrlr.ClusterLocation = voyager.ClusterLocation{ + EnvType: subCase.envType, + } + _, err = cntrlr.Process(ctx) + require.NoError(t, err) + + actions := tc.mainFake.Actions() + + cm, _ := findCreatedConfigMap(actions, namespaceName, apisynchronization.DefaultServiceMetadataConfigMapName) + require.NotNil(t, cm) + + assert.Equal(t, cm.Name, apisynchronization.DefaultServiceMetadataConfigMapName) + + assert.Contains(t, cm.Data, orch_meta.ConfigMapConfigKey) + data := cm.Data[orch_meta.ConfigMapConfigKey] + + var actual orch_meta.ServiceProperties + err = yaml.UnmarshalStrict([]byte(data), &actual) + require.NoError(t, err) + + assert.Equal(t, expected, actual) + }, + } + tc.run(t) + }) + } +} + func basicServiceProperties(s *creator_v1.Service, envType voyager.EnvType) orch_meta.ServiceProperties { return orch_meta.ServiceProperties{ ResourceOwner: s.Spec.ResourceOwner, @@ -1303,6 +1560,7 @@ type testCase struct { mainFake *kube_testing.Fake compFake *kube_testing.Fake scFake *fakeServiceCentral + ogFake *fakeOpsgenie releasesFake *fakeReleaseManagement registry *prometheus.Registry serviceName voyager.ServiceName @@ -1322,6 +1580,7 @@ func (tc *testCase) run(t *testing.T) { tc.compFake = &compClient.Fake tc.scFake = new(fakeServiceCentral) + tc.ogFake = new(fakeOpsgenie) tc.releasesFake = &fakeReleaseManagement{serviceName: tc.serviceName, serviceNames: tc.releaseDataServiceNames} tc.registry = prometheus.NewRegistry() @@ -1415,6 +1674,7 @@ func (tc *testCase) newController(t *testing.T, mainClient *k8s_fake.Clientset, ConfigMapInformer: configMapInformer, ServiceCentral: tc.scFake, + Opsgenie: tc.ogFake, ClusterLocation: tc.clusterLocation, ReleaseManagement: tc.releasesFake,