Skip to content

Commit 7df7ab5

Browse files
terrancedejesusterrancedejesus
authored
[Bug] Update Prebuilt Detection Rules Release Process (#3403)
* release fleet workflow updates; build package integration reference changes * updated commit hash extraction to output to env * adjusted bump-pkg-versions to only include release if necessary * fixed flake errors * add historical argument for build-release set to yes by default * Update detection_rules/devtools.py * fixed fleet workflow; updated registry data references * updated job names * removed extract commit hash job and consolidated into fleet pr job * added echo statement for current branch before checkout * removed id from extract commit hash
1 parent d41855a commit 7df7ab5

File tree

2 files changed

+130
-161
lines changed

2 files changed

+130
-161
lines changed

‎.github/workflows/release-fleet.yml

+120-154
Original file line numberDiff line numberDiff line change
@@ -28,162 +28,128 @@ on:
2828
type: choice
2929
description: 'New Package'
3030
required: true
31+
default: "true"
3132
options:
3233
- "true"
3334
- "false"
34-
add_historical:
35-
type: choice
36-
description: 'Add Historical Rules'
37-
required: true
38-
options:
39-
- "yes"
40-
- "no"
41-
commit_hash:
42-
description: 'Commit hash'
43-
required: true
4435

4536
jobs:
46-
check-commit:
47-
name: Check Commit Hash
48-
runs-on: ubuntu-latest
49-
outputs:
50-
is_locked_commit: ${{ steps.check_commit.outputs.check_message }}
51-
steps:
52-
- name: Checkout detection-rules
53-
uses: actions/checkout@v3
54-
with:
55-
path: detection-rules
56-
fetch-depth: 0
57-
58-
- name: Check commit message
59-
id: check_commit
60-
env:
61-
COMMIT_HASH: "${{github.event.inputs.commit_hash}}"
62-
run: |
63-
cd detection-rules
64-
COMMIT_MESSAGE=$(git show -s --format=%B $COMMIT_HASH | grep "Lock versions for releases" || true)
65-
if [ -z "$COMMIT_MESSAGE" ]; then
66-
echo "::set-output name=check_message::false"
67-
else
68-
echo "::set-output name=check_message::true"
69-
fi
70-
shell: bash
71-
72-
fleet-pr:
73-
name: Fleet PR
74-
needs: check-commit
75-
if: needs.check-commit.outputs.is_locked_commit == 'true'
76-
runs-on: ubuntu-latest
77-
steps:
78-
- name: Validate the source branch
79-
uses: actions/github-script@v3
80-
with:
81-
script: |
82-
if ('refs/heads/main' === '${{github.ref}}') {
83-
core.setFailed('Forbidden branch')
84-
}
85-
86-
- name: Checkout detection-rules
87-
uses: actions/checkout@v3
88-
with:
89-
path: detection-rules
90-
fetch-depth: 0
91-
92-
- name: Checkout elastic/integrations
93-
uses: actions/checkout@v3
94-
with:
95-
token: ${{ secrets.READ_WRITE_RELEASE_FLEET }}
96-
repository: ${{github.event.inputs.target_repo}}
97-
path: integrations
98-
99-
- name: Set up Python 3.8
100-
uses: actions/setup-python@v2
101-
with:
102-
python-version: 3.8
103-
104-
- name: Install Python dependencies
105-
run: |
106-
cd detection-rules
107-
python -m pip install --upgrade pip
108-
pip cache purge
109-
pip install .[dev]
110-
111-
- name: Checkout commit hash
112-
env:
113-
COMMIT_HASH: ${{github.event.inputs.commit_hash}}
114-
run: |
115-
cd detection-rules
116-
git checkout $COMMIT_HASH
117-
118-
- name: Bump prebuilt rules package version
119-
env:
120-
PACKAGE_MATURITY: "${{github.event.inputs.package_maturity}}"
121-
NEW_PACKAGE: "${{github.event.inputs.new_package}}"
122-
run: |
123-
cd detection-rules
124-
python -m detection_rules dev bump-pkg-versions \
125-
--patch-release \
126-
--new-package $NEW_PACKAGE \
127-
--maturity $PACKAGE_MATURITY
128-
129-
- name: Store release tag
130-
if: ${{github.event.inputs.package_maturity}} == "ga"
131-
id: packages-version
132-
run: |
133-
cd detection-rules
134-
output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+')
135-
echo "::set-output name=pkg_version::$output"
136-
137-
- name: Create release tag
138-
if: ${{github.event.inputs.package_maturity}} == "ga"
139-
env:
140-
RELEASE_TAG: "integration-v${{ steps.packages-version.outputs.pkg_version }}"
141-
run: |
142-
cd detection-rules
143-
git tag $RELEASE_TAG
144-
git push origin $RELEASE_TAG
145-
146-
- name: Build release package
147-
env:
148-
HISTORICAL: "${{github.event.inputs.add_historical}}"
149-
run: |
150-
cd detection-rules
151-
python -m detection_rules dev build-release --add-historical $HISTORICAL
152-
153-
- name: Set github config
154-
run: |
155-
git config --global user.email "72879786+protectionsmachine@users.noreply.github.com"
156-
git config --global user.name "protectionsmachine"
157-
158-
- name: Setup go
159-
uses: actions/setup-go@v3
160-
with:
161-
go-version: '^1.20.1'
162-
check-latest: true
163-
164-
- name: Build elastic-package
165-
run: |
166-
go install github.com/elastic/elastic-package@latest
167-
168-
- name: Create the PR to Integrations
169-
env:
170-
DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}"
171-
TARGET_REPO: "${{github.event.inputs.target_repo}}"
172-
TARGET_BRANCH: "${{github.event.inputs.target_branch}}"
173-
LOCAL_REPO: "../integrations"
174-
GITHUB_TOKEN: "${{ secrets.READ_WRITE_RELEASE_FLEET }}"
175-
run: |
176-
cd detection-rules
177-
python -m detection_rules dev integrations-pr \
178-
$LOCAL_REPO \
179-
--github-repo $TARGET_REPO \
180-
--base-branch $TARGET_BRANCH \
181-
--assign ${{github.actor}} \
182-
$DRAFT_ARGS
183-
184-
- name: Archive production artifacts
185-
uses: actions/upload-artifact@v3
186-
with:
187-
name: release-files
188-
path: |
189-
detection-rules/releases
37+
fleet-pr:
38+
name: Build package and create PR to integrations
39+
runs-on: ubuntu-latest
40+
steps:
41+
- name: Validate the source branch
42+
uses: actions/github-script@v3
43+
with:
44+
script: |
45+
if ('refs/heads/main' === '${{github.ref}}') {
46+
core.setFailed('Forbidden branch')
47+
}
48+
- name: Checkout detection-rules
49+
uses: actions/checkout@v3
50+
with:
51+
path: detection-rules
52+
fetch-depth: 0
53+
54+
- name: Extract version lock commit hash
55+
run: |
56+
cd detection-rules
57+
COMMIT_HASH=$(git log --grep='Lock versions for releases' -1 --format='%H')
58+
echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV
59+
echo "Extracted commit hash: $COMMIT_HASH"
60+
61+
- name: Checkout commit hash
62+
run: |
63+
cd detection-rules
64+
echo "Current branch is $GITHUB_REF"
65+
echo "Checking out commit hash $COMMIT_HASH"
66+
git checkout $COMMIT_HASH
67+
68+
- name: Checkout elastic/integrations
69+
uses: actions/checkout@v3
70+
with:
71+
token: ${{ secrets.READ_WRITE_RELEASE_FLEET }}
72+
repository: ${{github.event.inputs.target_repo}}
73+
path: integrations
74+
75+
- name: Set up Python 3.8
76+
uses: actions/setup-python@v2
77+
with:
78+
python-version: 3.8
79+
80+
- name: Install Python dependencies
81+
run: |
82+
cd detection-rules
83+
python -m pip install --upgrade pip
84+
pip cache purge
85+
pip install .[dev]
86+
87+
- name: Bump prebuilt rules package version
88+
env:
89+
PACKAGE_MATURITY: "${{github.event.inputs.package_maturity}}"
90+
NEW_PACKAGE: "${{github.event.inputs.new_package}}"
91+
run: |
92+
cd detection-rules
93+
python -m detection_rules dev bump-pkg-versions \
94+
--patch-release \
95+
--new-package $NEW_PACKAGE \
96+
--maturity $PACKAGE_MATURITY
97+
98+
- name: Store release tag
99+
if: github.event.inputs.package_maturity == 'ga'
100+
run: |
101+
cd detection-rules
102+
output=$(cat detection_rules/etc/packages.yml | grep -oP '(?<=\sversion: )\S+')
103+
echo "pkg_version=$output" >> $GITHUB_ENV
104+
105+
- name: Create release tag
106+
if: github.event.inputs.package_maturity == 'ga'
107+
run: |
108+
cd detection-rules
109+
RELEASE_TAG="integration-v${{ env.pkg_version }}"
110+
echo "Creating release tag: $RELEASE_TAG"
111+
git tag $RELEASE_TAG
112+
git push origin $RELEASE_TAG
113+
114+
- name: Build release package
115+
run: |
116+
cd detection-rules
117+
python -m detection_rules dev build-release
118+
119+
- name: Set github config
120+
run: |
121+
git config --global user.email "72879786+protectionsmachine@users.noreply.github.com"
122+
git config --global user.name "protectionsmachine"
123+
124+
- name: Setup go
125+
uses: actions/setup-go@v3
126+
with:
127+
go-version: '^1.20.1'
128+
check-latest: true
129+
130+
- name: Build elastic-package
131+
run: |
132+
go install github.com/elastic/elastic-package@latest
133+
134+
- name: Create the PR to Integrations
135+
env:
136+
DRAFT_ARGS: "${{startsWith(github.event.inputs.draft,'y') && '--draft' || ' '}}"
137+
TARGET_REPO: "${{github.event.inputs.target_repo}}"
138+
TARGET_BRANCH: "${{github.event.inputs.target_branch}}"
139+
LOCAL_REPO: "../integrations"
140+
GITHUB_TOKEN: "${{ secrets.READ_WRITE_RELEASE_FLEET }}"
141+
run: |
142+
cd detection-rules
143+
python -m detection_rules dev integrations-pr \
144+
$LOCAL_REPO \
145+
--github-repo $TARGET_REPO \
146+
--base-branch $TARGET_BRANCH \
147+
--assign ${{github.actor}} \
148+
$DRAFT_ARGS
149+
150+
- name: Archive production artifacts
151+
uses: actions/upload-artifact@v3
152+
with:
153+
name: release-files
154+
path: |
155+
detection-rules/releases

‎detection_rules/devtools.py

+10-7
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,14 @@ def dev_group():
8686
@click.option('--update-version-lock', '-u', is_flag=True,
8787
help='Save version.lock.json file with updated rule versions in the package')
8888
@click.option('--generate-navigator', is_flag=True, help='Generate ATT&CK navigator files')
89-
@click.option('--add-historical', type=str, required=True, default="no",
89+
@click.option('--add-historical', type=str, required=True, default="yes",
9090
help='Generate historical package-registry files')
9191
@click.option('--update-message', type=str, help='Update message for new package')
9292
def build_release(config_file, update_version_lock: bool, generate_navigator: bool, add_historical: str,
9393
update_message: str, release=None, verbose=True):
9494
"""Assemble all the rules into Kibana-ready release files."""
9595
config = load_dump(config_file)['package']
96+
registry_data = config['registry_data']
9697
add_historical = True if add_historical == "yes" else False
9798

9899
if generate_navigator:
@@ -111,17 +112,18 @@ def build_release(config_file, update_version_lock: bool, generate_navigator: bo
111112
package.save(verbose=verbose)
112113

113114
if add_historical:
114-
previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga", config['name'])
115+
previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga",
116+
registry_data['conditions']['kibana.version'].strip("^"))
115117
sde = SecurityDetectionEngine()
116118
historical_rules = sde.load_integration_assets(previous_pkg_version)
117119
historical_rules = sde.transform_legacy_assets(historical_rules)
118120

119-
docs = IntegrationSecurityDocsMDX(config['registry_data']['version'], Path(f'releases/{config["name"]}-docs'),
121+
docs = IntegrationSecurityDocsMDX(registry_data['version'], Path(f'releases/{config["name"]}-docs'),
120122
True, historical_rules, package, note=update_message)
121123
docs.generate()
122124

123125
click.echo(f'[+] Adding historical rules from {previous_pkg_version} package')
124-
package.add_historical_rules(historical_rules, config['registry_data']['version'])
126+
package.add_historical_rules(historical_rules, registry_data['version'])
125127

126128
if verbose:
127129
package.get_package_hash(verbose=verbose)
@@ -207,7 +209,7 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool,
207209
pkg_data["registry_data"]["version"] = str(pkg_ver.bump_minor().bump_prerelease("beta"))
208210
if patch_release:
209211
latest_patch_release_ver = find_latest_integration_version("security_detection_engine",
210-
maturity, pkg_data["name"])
212+
maturity, pkg_kibana_ver)
211213

212214
# if an existing minor or major does not have a package, bump from the last
213215
# example is 8.10.0-beta.1 is last, but on 9.0.0 major
@@ -219,13 +221,14 @@ def bump_versions(major_release: bool, minor_release: bool, patch_release: bool,
219221

220222
if maturity == "ga":
221223
pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_patch())
222-
pkg_data["registry_data"]["release"] = maturity
223224
else:
224225
# passing in true or false from GH actions; not using eval() for security purposes
225226
if new_package == "true":
226227
latest_patch_release_ver = latest_patch_release_ver.bump_patch()
227228
pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_prerelease("beta"))
228-
pkg_data["registry_data"]["release"] = maturity
229+
230+
if 'release' in pkg_data['registry_data']:
231+
pkg_data['registry_data']['release'] = maturity
229232

230233
click.echo(f"Kibana version: {pkg_data['name']}")
231234
click.echo(f"Package Kibana version: {pkg_data['registry_data']['conditions']['kibana.version']}")

0 commit comments

Comments
 (0)