Skip to content

Commit 7e1b528

Browse files
Kavindu-DodanKavindu-Dodan
authored
[Filebeat] [AWS] add support to source logs from AWS linked source accounts when using log_group_name_prefix (elastic#41206)
* configuration parsing to support arn & linked accounts Signed-off-by: Kavindu Dodanduwa <kavindu.dodanduwa@elastic.co> # Conflicts: # x-pack/filebeat/input/awscloudwatch/input.go * code review change - fix typo Signed-off-by: Kavindu Dodanduwa <kavindu.dodanduwa@elastic.co> * add support to linked accounts when using prefix mode Signed-off-by: Kavindu Dodanduwa <kavindu.dodanduwa@elastic.co> * add changelog entry Signed-off-by: Kavindu Dodanduwa <kavindu.dodanduwa@elastic.co> * review suggestion Signed-off-by: Kavindu Dodanduwa <kavindu.dodanduwa@elastic.co> * use non-pointer struct property Signed-off-by: Kavindu Dodanduwa <kavindu.dodanduwa@elastic.co> --------- Signed-off-by: Kavindu Dodanduwa <kavindu.dodanduwa@elastic.co>
1 parent 42f2d41 commit 7e1b528

File tree

6 files changed

+45
-23
lines changed

6 files changed

+45
-23
lines changed

‎CHANGELOG.next.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
328328
- Add support to source AWS cloudwatch logs from linked accounts. {pull}41188[41188]
329329
- Jounrald input now supports filtering by facilities {pull}41061[41061]
330330
- System module now supports reading from jounrald. {pull}41061[41061]
331+
- Add support to include AWS cloudwatch linked accounts when using log_group_name_prefix to define log group names. {pull}41206[41206]
331332

332333
*Auditbeat*
333334

‎x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,10 +144,15 @@
144144
#log_group_name: test
145145

146146
# The prefix for a group of log group names.
147+
# You can include linked source accounts by using the property `include_linked_accounts_for_prefix_mode`.
147148
# Note: `region_name` is required when `log_group_name_prefix` is given.
148149
# `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
149150
#log_group_name_prefix: /aws/
150151

152+
# State whether to include linked source accounts when obtaining log groups matching the prefix provided through `log_group_name_prefix`
153+
# This property works together with `log_group_name_prefix` and default value (if unset) is false
154+
#include_linked_accounts_for_prefix_mode: true
155+
151156
# Region that the specified log group or log group prefix belongs to.
152157
#region_name: us-east-1
153158

‎x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,13 +56,21 @@ Note: `region_name` is required when log_group_name is given.
5656

5757
[float]
5858
==== `log_group_name_prefix`
59-
The prefix for a group of log group names.
59+
The prefix for a group of log group names. See `include_linked_accounts_for_prefix_mode` option for linked source accounts behavior.
6060

6161
Note: `region_name` is required when
6262
`log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix`
6363
cannot be given at the same time. The number of workers that will process the
6464
log groups under this prefix is set through the `number_of_workers` config.
6565

66+
[float]
67+
==== `include_linked_accounts_for_prefix_mode`
68+
Configure whether to include linked source accounts that contains the prefix value defined through `log_group_name_prefix`.
69+
Accepts a boolean and this is by default disabled.
70+
71+
Note: Utilize `log_group_arn` if you desire to obtain logs from a known log group (including linked source accounts)
72+
You can read more about AWS account linking and cross account observability from the https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account.html[official documentation].
73+
6674
[float]
6775
==== `region_name`
6876
Region that the specified log group or log group prefix belongs to.

‎x-pack/filebeat/filebeat.reference.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3078,10 +3078,15 @@ filebeat.inputs:
30783078
#log_group_name: test
30793079

30803080
# The prefix for a group of log group names.
3081+
# You can include linked source accounts by using the property `include_linked_accounts_for_prefix_mode`.
30813082
# Note: `region_name` is required when `log_group_name_prefix` is given.
30823083
# `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
30833084
#log_group_name_prefix: /aws/
30843085

3086+
# State whether to include linked source accounts when obtaining log groups matching the prefix provided through `log_group_name_prefix`
3087+
# This property works together with `log_group_name_prefix` and default value (if unset) is false
3088+
#include_linked_accounts_for_prefix_mode: true
3089+
30853090
# Region that the specified log group or log group prefix belongs to.
30863091
#region_name: us-east-1
30873092

‎x-pack/filebeat/input/awscloudwatch/config.go

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,21 @@ import (
1313
)
1414

1515
type config struct {
16-
harvester.ForwarderConfig `config:",inline"`
17-
LogGroupARN string `config:"log_group_arn"`
18-
LogGroupName string `config:"log_group_name"`
19-
LogGroupNamePrefix string `config:"log_group_name_prefix"`
20-
RegionName string `config:"region_name"`
21-
LogStreams []*string `config:"log_streams"`
22-
LogStreamPrefix string `config:"log_stream_prefix"`
23-
StartPosition string `config:"start_position" default:"beginning"`
24-
ScanFrequency time.Duration `config:"scan_frequency" validate:"min=0,nonzero"`
25-
APITimeout time.Duration `config:"api_timeout" validate:"min=0,nonzero"`
26-
APISleep time.Duration `config:"api_sleep" validate:"min=0,nonzero"`
27-
Latency time.Duration `config:"latency"`
28-
NumberOfWorkers int `config:"number_of_workers"`
29-
AWSConfig awscommon.ConfigAWS `config:",inline"`
16+
harvester.ForwarderConfig `config:",inline"`
17+
LogGroupARN string `config:"log_group_arn"`
18+
LogGroupName string `config:"log_group_name"`
19+
LogGroupNamePrefix string `config:"log_group_name_prefix"`
20+
IncludeLinkedAccountsForPrefixMode bool `config:"include_linked_accounts_for_prefix_mode"`
21+
RegionName string `config:"region_name"`
22+
LogStreams []*string `config:"log_streams"`
23+
LogStreamPrefix string `config:"log_stream_prefix"`
24+
StartPosition string `config:"start_position" default:"beginning"`
25+
ScanFrequency time.Duration `config:"scan_frequency" validate:"min=0,nonzero"`
26+
APITimeout time.Duration `config:"api_timeout" validate:"min=0,nonzero"`
27+
APISleep time.Duration `config:"api_sleep" validate:"min=0,nonzero"`
28+
Latency time.Duration `config:"latency"`
29+
NumberOfWorkers int `config:"number_of_workers"`
30+
AWSConfig awscommon.ConfigAWS `config:",inline"`
3031
}
3132

3233
func defaultConfig() config {

‎x-pack/filebeat/input/awscloudwatch/input.go

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,9 @@ func (in *cloudwatchInput) Run(inputContext v2.Context, pipeline beat.Pipeline)
105105
})
106106

107107
if len(logGroupIDs) == 0 {
108-
// fallback to LogGroupNamePrefix to derive group IDs
109-
logGroupIDs, err = getLogGroupNames(svc, in.config.LogGroupNamePrefix)
108+
// We haven't extracted group identifiers directly from the input configurations,
109+
// now fallback to provided LogGroupNamePrefix and use derived service client to derive logGroupIDs
110+
logGroupIDs, err = getLogGroupNames(svc, in.config.LogGroupNamePrefix, in.config.IncludeLinkedAccountsForPrefixMode)
110111
if err != nil {
111112
return fmt.Errorf("failed to get log group names from LogGroupNamePrefix: %w", err)
112113
}
@@ -164,15 +165,16 @@ func fromConfig(cfg config, awsCfg awssdk.Config) (logGroupIDs []string, region
164165
return logGroupIDs, region, nil
165166
}
166167

167-
// getLogGroupNames uses DescribeLogGroups API to retrieve all log group names
168-
func getLogGroupNames(svc *cloudwatchlogs.Client, logGroupNamePrefix string) ([]string, error) {
168+
// getLogGroupNames uses DescribeLogGroups API to retrieve LogGroupArn entries that matches the provided logGroupNamePrefix
169+
func getLogGroupNames(svc *cloudwatchlogs.Client, logGroupNamePrefix string, withLinkedAccount bool) ([]string, error) {
169170
// construct DescribeLogGroupsInput
170171
describeLogGroupsInput := &cloudwatchlogs.DescribeLogGroupsInput{
171-
LogGroupNamePrefix: awssdk.String(logGroupNamePrefix),
172+
LogGroupNamePrefix: awssdk.String(logGroupNamePrefix),
173+
IncludeLinkedAccounts: awssdk.Bool(withLinkedAccount),
172174
}
173175

174176
// make API request
175-
var logGroupNames []string
177+
var logGroupIDs []string
176178
paginator := cloudwatchlogs.NewDescribeLogGroupsPaginator(svc, describeLogGroupsInput)
177179
for paginator.HasMorePages() {
178180
page, err := paginator.NextPage(context.TODO())
@@ -181,8 +183,8 @@ func getLogGroupNames(svc *cloudwatchlogs.Client, logGroupNamePrefix string) ([]
181183
}
182184

183185
for _, lg := range page.LogGroups {
184-
logGroupNames = append(logGroupNames, *lg.LogGroupName)
186+
logGroupIDs = append(logGroupIDs, *lg.LogGroupArn)
185187
}
186188
}
187-
return logGroupNames, nil
189+
return logGroupIDs, nil
188190
}

0 commit comments

Comments
 (0)