-
Notifications
You must be signed in to change notification settings - Fork 5k
201 lines (179 loc) · 7.21 KB
/
nightly_build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
name: Nightly Build
on:
workflow_dispatch:
schedule:
# Run job at 6:30 UTC, 10.30pm PST, or 11.30pm PDT
- cron: "30 6 * * *"
env:
POETRY_VERSION: "1.8.3"
PYTHON_VERSION: "3.12"
jobs:
create-nightly-tag:
if: github.repository == 'langflow-ai/langflow'
runs-on: ubuntu-latest
defaults:
run:
shell: bash -ex -o pipefail {0}
permissions:
# Required to create tag
contents: write
outputs:
main_tag: ${{ steps.generate_main_tag.outputs.main_tag }}
base_tag: ${{ steps.set_base_tag.outputs.base_tag }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: true
- name: "Setup Environment"
uses: ./.github/actions/setup-uv
- name: Install the project
run: uv sync --dev
- name: Generate main nightly tag
id: generate_main_tag
run: |
# NOTE: This outputs the tag with the `v` prefix.
MAIN_TAG="$(uv run ./scripts/ci/pypi_nightly_tag.py main)"
echo "main_tag=$MAIN_TAG" >> $GITHUB_OUTPUT
echo "main_tag=$MAIN_TAG"
- name: Check if main tag already exists
id: check_main_tag
run: |
git fetch --tags
if git rev-parse -q --verify "refs/tags/${{ steps.generate_main_tag.outputs.main_tag }}" >/dev/null; then
echo "main_tag_exists=true" >> $GITHUB_OUTPUT
else
echo "main_tag_exists=false" >> $GITHUB_OUTPUT
fi
- name: Generate base nightly tag
id: generate_base_tag
if: ${{ steps.check_main_tag.outputs.main_tag_exists == 'false' }}
run: |
# NOTE: This outputs the tag with the `v` prefix.
BASE_TAG="$(uv run ./scripts/ci/pypi_nightly_tag.py base)"
echo "base_tag=$BASE_TAG" >> $GITHUB_OUTPUT
echo "base_tag=$BASE_TAG"
- name: Commit tag
id: commit_tag
if: ${{ steps.check_main_tag.outputs.main_tag_exists == 'false' }}
run: |
# If the main tag does not exist in GH, we create the base tag from the existing codebase.
git config --global user.email "bot-nightly-builds@langflow.org"
git config --global user.name "Langflow Bot"
MAIN_TAG="${{ steps.generate_main_tag.outputs.main_tag }}"
BASE_TAG="${{ steps.generate_base_tag.outputs.base_tag }}"
echo "Updating base project version to $BASE_TAG and updating main project version to $MAIN_TAG"
uv run ./scripts/ci/update_pyproject_combined.py main $MAIN_TAG $BASE_TAG
uv lock
cd src/backend/base && uv lock && cd ../../..
git add pyproject.toml src/backend/base/pyproject.toml uv.lock src/backend/base/uv.lock
git commit -m "Update version and project name"
echo "Tagging main with $MAIN_TAG"
if ! git tag -a $MAIN_TAG -m "Langflow nightly $MAIN_TAG"; then
echo "Tag creation failed. Exiting the workflow."
exit 1
fi
echo "Pushing main tag $MAIN_TAG"
if ! git push origin $MAIN_TAG; then
echo "Tag push failed. Check if the tag already exists. Exiting the workflow."
exit 1
fi
# TODO: notify on failure
- name: Checkout main nightly tag
uses: actions/checkout@v4
if: ${{ steps.check_main_tag.outputs.main_tag_exists == 'true' }}
with:
ref: ${{ steps.generate_main_tag.outputs.main_tag }}
- name: Retrieve Base Tag
id: retrieve_base_tag
if: ${{ steps.check_main_tag.outputs.main_tag_exists == 'true' }}
working-directory: src/backend/base
run: |
# If the main tag already exists, we need to retrieve the base version from the main tag codebase.
version=$(uv tree | grep 'langflow-base' | awk '{print $3}')
echo "base_tag=$version" >> $GITHUB_OUTPUT
echo "base_tag=$version"
- name: Set Base Tag
id: set_base_tag
run: |
if [ "${{ steps.retrieve_base_tag.conclusion }}" != "skipped" ] && [ "${{ steps.retrieve_base_tag.outputs.base_tag }}" ]; then
BASE_TAG="${{ steps.retrieve_base_tag.outputs.base_tag }}"
echo "base_tag=$BASE_TAG" >> $GITHUB_OUTPUT
echo "base_tag=$BASE_TAG"
elif [ "${{ steps.commit_tag.conclusion }}" != "skipped" ] && [ "${{ steps.generate_base_tag.outputs.base_tag }}" ]; then
BASE_TAG="${{ steps.generate_base_tag.outputs.base_tag }}"
echo "base_tag=$BASE_TAG" >> $GITHUB_OUTPUT
echo "base_tag=$BASE_TAG"
else
echo "No base tag found. Exiting the workflow."
exit 1
fi
frontend-tests:
if: github.repository == 'langflow-ai/langflow'
name: Run Frontend Tests
needs: create-nightly-tag
uses: ./.github/workflows/typescript_test.yml
with:
tests_folder: "tests"
secrets:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
STORE_API_KEY: ${{ secrets.STORE_API_KEY }}
backend-unit-tests:
if: github.repository == 'langflow-ai/langflow'
name: Run Backend Unit Tests
needs: create-nightly-tag
uses: ./.github/workflows/python_test.yml
with:
python-versions: '["3.10", "3.11", "3.12"]'
# Not making nightly builds dependent on integration test success
# due to inherent flakiness of 3rd party integrations
# Revisit when https://github.com/langflow-ai/langflow/pull/3607 is merged.
# backend-integration-tests:
# name: Run Backend Integration Tests
# needs: create-nightly-tag
# uses: ./.github/workflows/integration_tests.yml
# with:
# python-versions: '["3.10", "3.11", "3.12"]'
# ref: ${{ needs.create-nightly-tag.outputs.tag }}
release-nightly-build:
if: github.repository == 'langflow-ai/langflow'
name: Run Nightly Langflow Build
needs: [frontend-tests, backend-unit-tests, create-nightly-tag]
uses: ./.github/workflows/release_nightly.yml
with:
build_docker_base: true
build_docker_main: true
nightly_tag_main: ${{ needs.create-nightly-tag.outputs.main_tag }}
nightly_tag_base: ${{ needs.create-nightly-tag.outputs.base_tag }}
secrets: inherit
# slack-notification:
# name: Send Slack Notification
# needs: run-nightly-build
# runs-on: ubuntu-latest
# steps:
# - name: Send success notification to Slack
# if: success()
# uses: slackapi/slack-github-action@v1.26.0
# with:
# payload: |
# {
# "channel": "#langflow-nightly-builds",
# "username": "GitHub Actions",
# "text": "Nightly Build Successful :white_check_mark:",
# "icon_emoji": ":rocket:"
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
# - name: Send failure notification to Slack
# if: failure()
# uses: slackapi/slack-github-action@v1.26.0
# with:
# payload: |
# {
# "channel": "#langflow-nightly-builds",
# "username": "GitHub Actions",
# "text": "Nightly Build Failed :x:",
# "icon_emoji": ":warning:"
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}