Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Insights access logs and configs #3596

Merged
merged 6 commits into from
Oct 9, 2024
+1,780 −62
Merged

Support Insights access logs and configs #3596

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ad5a2ab
Support Insights access logs and configs
ruks Oct 3, 2024
2cc85b4
Fixed formatting issues
ruks Oct 3, 2024
4899fb4
Adding review changes and model changes
ruks Oct 7, 2024
381a5dd
Adding header
ruks Oct 7, 2024
de0ccbe
Fixed review suggestions
ruks Oct 9, 2024
366be3e
Fixed logging format issue
ruks Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions adapter/internal/discovery/xds/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,16 @@ func UpdateAPI(vHost string, apiProject mgw.ProjectAPI, deployedEnvironments []*
mgwSwagger.APIProvider = apiProject.APIYaml.Data.Provider
mgwSwagger.EnvironmentID = deployedEnvironments[0].ID
mgwSwagger.EnvironmentName = deployedEnvironments[0].Name

choreoComponentInfo := mgw.ChoreoComponentInfo{
OrganizationID: apiYaml.ChoreoComponentInfo.OrganizationID,
ProjectID: apiYaml.ChoreoComponentInfo.ProjectID,
ComponentID: apiYaml.ChoreoComponentInfo.ComponentID,
VersionID: apiYaml.ChoreoComponentInfo.VersionID,
}

mgwSwagger.ChoreoComponentInfo = &choreoComponentInfo

organizationID := apiProject.OrganizationID
apiHashValue := generateHashValue(apiYaml.Name, apiYaml.Version)

Expand Down
9 changes: 9 additions & 0 deletions adapter/internal/oasparser/config_generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,14 @@ func GetEnforcerAPI(mgwSwagger model.MgwSwagger, lifeCycleState string, vhost st
}
}

choreoComponentInfo := &api.ChoreoComponentInfo{}
if mgwSwagger.ChoreoComponentInfo != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this is something we expect to be mandatory for a given api, we can add a log and monitor from logs for a while.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

discussed and decided we won't require that sort of a log.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not critical atm

choreoComponentInfo.ComponentID = mgwSwagger.ChoreoComponentInfo.ComponentID
choreoComponentInfo.VersionID = mgwSwagger.ChoreoComponentInfo.VersionID
choreoComponentInfo.OrganizationID = mgwSwagger.ChoreoComponentInfo.OrganizationID
choreoComponentInfo.ProjectID = mgwSwagger.ChoreoComponentInfo.ProjectID
}

return &api.Api{
Id: mgwSwagger.GetID(),
Title: mgwSwagger.GetTitle(),
Expand All @@ -221,6 +229,7 @@ func GetEnforcerAPI(mgwSwagger model.MgwSwagger, lifeCycleState string, vhost st
DeploymentType: mgwSwagger.DeploymentType,
EnvironmentId: mgwSwagger.EnvironmentID,
EnvironmentName: mgwSwagger.EnvironmentName,
ChoreoComponentInfo: choreoComponentInfo,
}
}

Expand Down
86 changes: 70 additions & 16 deletions adapter/internal/oasparser/envoyconf/access_loggers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,28 +36,25 @@ import (
"google.golang.org/protobuf/types/known/wrapperspb"
)

// getAccessLogConfigs provides file access log configurations for envoy
func getFileAccessLogConfigs() *config_access_logv3.AccessLog {
var logFormat *file_accesslogv3.FileAccessLog_LogFormat
logpath := defaultAccessLogPath //default access log path

logConf := config.ReadLogConfigs()

if !logConf.AccessLogs.Enable {
logger.LoggerOasparser.Info("Accesslog Configurations are disabled.")
return nil
}

formatters := []*corev3.TypedExtensionConfig{
// getAccessLogConfigs provides default formatters
func getDefaultFormatters() []*corev3.TypedExtensionConfig {
return []*corev3.TypedExtensionConfig{
{
Name: "envoy.formatter.req_without_query",
TypedConfig: &anypb.Any{
TypeUrl: "type.googleapis.com/envoy.extensions.formatter.req_without_query.v3.ReqWithoutQuery",
},
},
}
// Set the default log format
logFormat = &file_accesslogv3.FileAccessLog_LogFormat{
}

// getDefaultTextLogFormat provides default text log format
func getDefaultTextLogFormat() *file_accesslogv3.FileAccessLog_LogFormat {
logConf := config.ReadLogConfigs()

formatters := getDefaultFormatters()

return &file_accesslogv3.FileAccessLog_LogFormat{
LogFormat: &corev3.SubstitutionFormatString{
Format: &corev3.SubstitutionFormatString_TextFormatSource{
ruks marked this conversation as resolved.
Show resolved Hide resolved
TextFormatSource: &corev3.DataSource{
Expand All @@ -70,6 +67,59 @@ func getFileAccessLogConfigs() *config_access_logv3.AccessLog {
Formatters: formatters,
},
}
}

// getAccessLogConfigs provides file access log configurations for envoy
func getInsightsAccessLogConfigs() *config_access_logv3.AccessLog {
var logFormat *file_accesslogv3.FileAccessLog_LogFormat
logpath := defaultAccessLogPath //default access log path

logConf := config.ReadLogConfigs()

if !logConf.InsightsLogs.Enable {
return nil
}

// Set the default log format
logFormat = getDefaultTextLogFormat()

logpath = logConf.InsightsLogs.LogFile
accessLogConf := &file_accesslogv3.FileAccessLog{
Path: logpath,
AccessLogFormat: logFormat,
}

accessLogTypedConf, err := anypb.New(accessLogConf)
if err != nil {
logger.LoggerOasparser.Error("Error marshaling access log configs. ", err)
return nil
}

accessLog := config_access_logv3.AccessLog{
Name: fileAccessLogName,
Filter: nil,
ConfigType: &config_access_logv3.AccessLog_TypedConfig{
TypedConfig: accessLogTypedConf,
},
}

return &accessLog
}

// getAccessLogConfigs provides file access log configurations for envoy
func getFileAccessLogConfigs() *config_access_logv3.AccessLog {
var logFormat *file_accesslogv3.FileAccessLog_LogFormat
logpath := defaultAccessLogPath //default access log path

logConf := config.ReadLogConfigs()

if !logConf.AccessLogs.Enable {
logger.LoggerOasparser.Info("Accesslog Configurations are disabled.")
return nil
}

// Set the default log format
logFormat = getDefaultTextLogFormat()

// Configure the log format based on the log type
switch logConf.AccessLogs.LogType {
Expand All @@ -95,7 +145,7 @@ func getFileAccessLogConfigs() *config_access_logv3.AccessLog {
Fields: logFields,
},
},
Formatters: formatters,
Formatters: getDefaultFormatters(),
},
}
logger.LoggerOasparser.Debug("Access log type is set to json.")
Expand Down Expand Up @@ -197,12 +247,16 @@ func getAccessLogs() []*config_access_logv3.AccessLog {
var accessLoggers []*config_access_logv3.AccessLog
fileAccessLog := getFileAccessLogConfigs()
grpcAccessLog := getGRPCAccessLogConfigs(conf)
insightsAccessLog := getInsightsAccessLogConfigs()
if fileAccessLog != nil {
accessLoggers = append(accessLoggers, fileAccessLog)
}
if grpcAccessLog != nil {
accessLoggers = append(accessLoggers, getGRPCAccessLogConfigs(conf))
}
if insightsAccessLog != nil {
accessLoggers = append(accessLoggers, insightsAccessLog)
}
return accessLoggers
}

Expand Down
15 changes: 12 additions & 3 deletions adapter/internal/oasparser/model/mgw_swagger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,18 @@ type MgwSwagger struct {
// APIProvider is required for analytics purposes as /apis call is avoided temporarily.
APIProvider string
// DeploymentType could be either "PRODUCTION" or "SANDBOX"
DeploymentType string
EnvironmentID string
EnvironmentName string
DeploymentType string
EnvironmentID string
EnvironmentName string
ChoreoComponentInfo *ChoreoComponentInfo
}

// ChoreoComponentInfo represents the information of the Choreo component
type ChoreoComponentInfo struct {
OrganizationID string
ProjectID string
ComponentID string
VersionID string
}

// EndpointCluster represent an upstream cluster
Expand Down
8 changes: 8 additions & 0 deletions adapter/internal/oasparser/model/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,14 @@ type apiData struct {
BackendJWTConfiguration backendJWTConfiguration `json:"backendJWTConfiguration,omitempty"`
EndpointConfig endpointConfigStruct `json:"endpointConfig,omitempty"`
Operations []OperationYaml `json:"Operations,omitempty"`
ChoreoComponentInfo choreoComponentInfo `json:"choreoComponentInfo,omitempty"`
}

type choreoComponentInfo struct {
OrganizationID string `json:"organizationId,omitempty"`
ProjectID string `json:"projectId,omitempty"`
ComponentID string `json:"componentId,omitempty"`
VersionID string `json:"versionId,omitempty"`
}

type backendJWTConfiguration struct {
Expand Down
18 changes: 16 additions & 2 deletions adapter/pkg/config/log_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ type accessLog struct {
JSONFormat map[string]string
}

type insightsLog struct {
Enable bool
LogFile string
LoggingFormat string
OmitEmptyValues bool
}

// AccessLogExcludes represents the configurations related to excludes from access logs.
type AccessLogExcludes struct {
SystemHost AccessLogExcludesSystemHost
Expand All @@ -60,8 +67,9 @@ type LogConfig struct {
Compress bool
}

Pkg []pkg
AccessLogs *accessLog
Pkg []pkg
AccessLogs *accessLog
InsightsLogs *insightsLog
}

func getDefaultLogConfig() *LogConfig {
Expand Down Expand Up @@ -115,6 +123,12 @@ func getDefaultLogConfig() *LogConfig {
"extAuthDtls": "%DYNAMIC_METADATA(envoy.filters.http.ext_authz:extAuthDetails)%",
},
},
InsightsLogs: &insightsLog{
Enable: false,
LogFile: "/dev/stdout",
LoggingFormat: "-",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it work if we simply provided like this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

even this is enabled it will just print '-'. but ideally it is not usable and proper config should apply

OmitEmptyValues: true,
},
}
adapterLogConfig.Rotation.MaxSize = 10
adapterLogConfig.Rotation.MaxAge = 2
Expand Down
48 changes: 34 additions & 14 deletions adapter/pkg/discovery/api/wso2/discovery/api/api.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading