forked from instructlab/instructlab
-
Notifications
You must be signed in to change notification settings - Fork 0
170 lines (149 loc) · 5.34 KB
/
e2e-nvidia-t4-x1.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
# SPDX-License-Identifier: Apache-2.0
name: E2E (NVIDIA Tesla T4 x1)
on:
# run against every merge commit to 'main' and release branches
push:
branches:
- main
- release-*
# only run on PRs that touch certain regex paths
pull_request_target:
branches:
- main
- release-*
paths:
# note this should match the merging criteria in 'mergify.yml'
- '**.py'
- 'pyproject.toml'
- 'requirements**.txt'
- 'scripts/basic-workflow-tests.sh' # Used by this workflow
- 'scripts/test-data/**'
- '.github/workflows/e2e-nvidia-t4-x1.yml' # This workflow
concurrency:
group: ${{ github.workflow }}-${{ github.event.number || github.ref }}
cancel-in-progress: true
env:
LC_ALL: en_US.UTF-8
defaults:
run:
shell: bash
permissions:
contents: read
jobs:
e2e-workflow-ready:
permissions:
checks: read
uses: ./.github/workflows/status-checks.yml
with:
job_ids: >- # Space-separated job ids to wait on for status checks
DCO
actionlint
markdown-lint
shellcheck
spellcheck
lint-workflow-complete
start-ec2-runner:
needs:
- e2e-workflow-ready
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Start EC2 runner
id: start-ec2-runner
uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e # v2.3.6
with:
mode: start
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
ec2-image-id: ami-01a89eee1adde309c
ec2-instance-type: g4dn.2xlarge
subnet-id: subnet-02d230cffd9385bd4
security-group-id: sg-06300447c4a5fbef3
iam-role-name: instructlab-ci-runner
aws-resource-tags: >
[
{"Key": "Name", "Value": "instructlab-ci-github-small-runner"},
{"Key": "GitHubRepository", "Value": "${{ github.repository }}"},
{"Key": "GitHubRef", "Value": "${{ github.ref }}"},
{"Key": "GitHubPR", "Value": "${{ github.event.number }}"}
]
e2e-test:
needs:
- start-ec2-runner
runs-on: ${{ needs.start-ec2-runner.outputs.label }}
# It is important that this job has no write permissions and has
# no access to any secrets. This part (e2e-test) is where we are running
# untrusted code from PRs.
permissions: {}
steps:
- name: Install Packages
run: |
cat /etc/os-release
sudo dnf install -y gcc gcc-c++ make git python3.11 python3.11-devel
- name: Checkout
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
# https://github.com/actions/checkout/issues/249
fetch-depth: 0
- name: Fetch and checkout PR
if: ${{ github.event_name == 'pull_request_target' }}
run: |
git fetch origin pull/${{ github.event.number }}/head:pr-${{ github.event.number }}
git checkout pr-${{ github.event.number }}
- name: Install ilab
run: |
export PATH="/home/ec2-user/.local/bin:/usr/local/cuda/bin:$PATH"
python3.11 -m venv --upgrade-deps venv
. venv/bin/activate
nvidia-smi
python3.11 -m pip cache remove llama_cpp_python
CMAKE_ARGS="-DLLAMA_CUDA=on" python3.11 -m pip install .
# https://github.com/instructlab/instructlab/issues/1821
# install with Torch and build dependencies installed
python3.11 -m pip install packaging wheel setuptools-scm
python3.11 -m pip install .[cuda]
- name: Check disk
run: |
df -h
- name: Run e2e test
run: |
. venv/bin/activate
./scripts/basic-workflow-tests.sh -msq
stop-ec2-runner:
needs:
- start-ec2-runner
- e2e-test
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@fcfb31a5760dad1314a64a0e172b78ec6fc8a17e # v2.3.6
with:
mode: stop
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
label: ${{ needs.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-ec2-runner.outputs.ec2-instance-id }}
e2e-workflow-complete:
# we don't want to block PRs on failed EC2 cleanup
# so not requiring "stop-ec2-runner" as well
permissions:
checks: read
uses: ./.github/workflows/status-checks.yml
with:
job_ids: >- # Space-separated job ids to wait on for status checks
e2e-workflow-ready
start-ec2-runner
e2e-test