Skip to content

Commit

Permalink
Merge branch 'main' into upgrade_azurerm_provider
Browse files Browse the repository at this point in the history
  • Loading branch information
dcmcand authored Nov 6, 2024
2 parents 673b340 + cb8ac10 commit 9f7ba1f
Show file tree
Hide file tree
Showing 26 changed files with 458 additions and 159 deletions.
1 change: 0 additions & 1 deletion .github/ISSUE_TEMPLATE/bug-report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ body:
- "Azure"
- "GCP"
- "AWS"
- "Digital Ocean"
- "kind"
validations:
required: false
Expand Down
19 changes: 19 additions & 0 deletions .github/actions/health-check/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
name: health-check
description: "Check health of Nebari deployment"

inputs:
domain:
description: Domain name
required: true

runs:
using: composite

steps:
- name: List kubernetes components
shell: bash
run: kubectl get --all-namespaces all,cm,secret,pv,pvc,ing

- name: Check if JupyterHub login page is accessible
shell: bash
run: curl --insecure --include 'https://${{ inputs.domain }}/hub/home'
81 changes: 81 additions & 0 deletions .github/actions/init-local/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
name: init-local
description: "Initialize Nebari config for local deployment"

inputs:
directory:
description: "Path to directory to initialize in"
required: false
default: './local-deployment'

outputs:
directory:
description: "Path to config directory"
value: ${{ steps.metadata.outputs.directory }}
config:
description: "Path to Nebari config"
value: ${{ steps.metadata.outputs.config }}
project:
description: "Project name"
value: ${{ steps.metadata.outputs.project }}
domain:
description: "Domain name"
value: ${{ steps.metadata.outputs.domain }}

runs:
using: composite

steps:
- shell: bash
id: metadata
run: |
# Setup metadata
DIRECTORY=$(realpath '${{ inputs.directory }}')
mkdir --parents "${DIRECTORY}"
echo "directory=${DIRECTORY}" | tee --append "${GITHUB_OUTPUT}"
CONFIG="${DIRECTORY}/nebari-config.yaml"
echo "config=${CONFIG}" | tee --append "${GITHUB_OUTPUT}"
PROJECT='github-actions'
echo "project=${PROJECT}" | tee --append "${GITHUB_OUTPUT}"
DOMAIN='github-actions.nebari.dev'
nslookup "${DOMAIN}"
echo "domain=${DOMAIN}" | tee --append "${GITHUB_OUTPUT}"
- shell: bash -l {0}
id: init
working-directory: ${{ steps.metadata.outputs.directory }}
run: |
nebari init local \
--project-name '${{ steps.metadata.outputs.project }}' \
--domain-name '${{ steps.metadata.outputs.domain }}' \
--auth-provider password \
--output '${{ steps.metadata.outputs.config }}'
- shell: bash
run: |
# Update nebari config for CI
# Change default JupyterLab theme
cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
jupyterlab:
default_settings:
"@jupyterlab/apputils-extension:themes":
theme: JupyterLab Dark
EOM
# Change default value for minio persistence size
cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
monitoring:
enabled: true
overrides:
minio:
persistence:
size: 1Gi
EOM
- shell: bash
run: |
# Display Nebari config
cat '${{ steps.metadata.outputs.config }}'
30 changes: 30 additions & 0 deletions .github/actions/setup-local/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
name: setup-local
description: "Setup runner for local deployment"

inputs:
kubectl-version:
description: "Version of kubectl to install"
required: false
default: "1.19.16"

runs:
using: composite

steps:
- uses: azure/setup-kubectl@v4
with:
version: v${{ inputs.kubectl-version }}

- shell: bash
run: |
# Enable docker permissions for user
sudo docker ps
sudo usermod -aG docker $USER && newgrp docker
docker info
docker ps
- shell: bash
run: |
# Get routing table for docker pods
ip route
2 changes: 0 additions & 2 deletions .github/workflows/test-provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ jobs:
provider:
- aws
- azure
- do
- gcp
- local
- existing
Expand Down Expand Up @@ -88,7 +87,6 @@ jobs:
kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci client_id | ARM_CLIENT_ID;
kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci tenant_id | ARM_TENANT_ID;
kv/data/repository/nebari-dev/nebari/azure/nebari-dev-ci/github-nebari-dev-repo-ci subscription_id | ARM_SUBSCRIPTION_ID;
kv/data/repository/nebari-dev/nebari/shared_secrets DIGITALOCEAN_TOKEN | DIGITALOCEAN_TOKEN;
kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_ACCESS_KEY_ID | SPACES_ACCESS_KEY_ID;
kv/data/repository/nebari-dev/nebari/shared_secrets SPACES_SECRET_ACCESS_KEY | SPACES_SECRET_ACCESS_KEY;
Expand Down
89 changes: 19 additions & 70 deletions .github/workflows/test_local_integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ jobs:
with:
fetch-depth: 0

- name: Setup runner for local deployment
uses: ./.github/actions/setup-local

- name: Checkout the branch from the PR that triggered the job
if: ${{ github.event_name == 'issue_comment' }}
run: |
Expand All @@ -75,82 +78,29 @@ jobs:
pip install .[dev]
playwright install
- uses: azure/setup-kubectl@v4.0.0
with:
version: v1.19.16

- name: Enable docker permissions for user
run: |
sudo docker ps
sudo usermod -aG docker $USER && newgrp docker
docker info
docker ps
- name: Get routing table for docker pods
run: |
ip route
- name: Initialize Nebari Cloud
run: |
mkdir -p local-deployment
cd local-deployment
nebari init local --project=thisisatest --domain github-actions.nebari.dev --auth-provider=password
# Need smaller profiles on Local Kind
sed -i -E 's/(cpu_guarantee):\s+[0-9\.]+/\1: 0.25/g' "nebari-config.yaml"
sed -i -E 's/(mem_guarantee):\s+[A-Za-z0-9\.]+/\1: 0.25G/g' "nebari-config.yaml"
# Change default JupyterLab theme
cat >> nebari-config.yaml <<- EOM
jupyterlab:
default_settings:
"@jupyterlab/apputils-extension:themes":
theme: JupyterLab Dark
EOM
# Change default value for minio persistence size
cat >> nebari-config.yaml <<- EOM
monitoring:
enabled: true
overrides:
minio:
persistence:
size: 1Gi
EOM
cat nebari-config.yaml
- name: Initialize Nebari config for local deployment
id: init
uses: ./.github/actions/init-local

- name: Deploy Nebari
working-directory: local-deployment
run: |
nebari deploy --config nebari-config.yaml --disable-prompt
working-directory: ${{ steps.init.outputs.directory }}
run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt

- name: Basic kubectl checks after deployment
if: always()
run: |
kubectl get all,cm,secret,pv,pvc,ing -A
- name: Check github-actions.nebari.dev resolves
run: |
nslookup github-actions.nebari.dev
- name: Curl jupyterhub login page
run: |
curl -k https://github-actions.nebari.dev/hub/home -i
- name: Health check
uses: ./.github/actions/health-check
with:
domain: ${{ steps.init.outputs.domain }}

- name: Create example-user
working-directory: local-deployment
working-directory: ${{ steps.init.outputs.directory }}
run: |
nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config nebari-config.yaml
nebari keycloak listusers --config nebari-config.yaml
- name: Get nebari-config.yaml full path
run: echo "NEBARI_CONFIG_PATH=`realpath ./local-deployment/nebari-config.yaml`" >> "$GITHUB_ENV"
nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config ${{ steps.init.outputs.config }}
nebari keycloak listusers --config ${{ steps.init.outputs.config }}
### DEPLOYMENT TESTS
- name: Deployment Pytests
env:
NEBARI_CONFIG_PATH: ${{ steps.init.outputs.config }}
KEYCLOAK_USERNAME: ${{ env.TEST_USERNAME }}
KEYCLOAK_PASSWORD: ${{ env.TEST_PASSWORD }}
run: |
Expand All @@ -165,7 +115,7 @@ jobs:
env:
KEYCLOAK_USERNAME: ${{ env.TEST_USERNAME }}
KEYCLOAK_PASSWORD: ${{ env.TEST_PASSWORD }}
NEBARI_FULL_URL: https://github-actions.nebari.dev/
NEBARI_FULL_URL: "https://${{ steps.init.outputs.domain }}/"
working-directory: tests/tests_e2e/playwright
run: |
# create environment file
Expand All @@ -187,6 +137,5 @@ jobs:
# in the CI, it makes sense to only run on merge to main or workflow_dispatch to speed
# up feedback cycle
if: github.ref_name == 'main' || github.event_name == 'workflow_dispatch'
working-directory: local-deployment
run: |
nebari destroy --config nebari-config.yaml --disable-prompt
working-directory: ${{ steps.init.outputs.directory }}
run: nebari destroy --config ${{ steps.init.outputs.config }} --disable-prompt
85 changes: 85 additions & 0 deletions .github/workflows/test_local_upgrade.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
name: "Local Upgrade Tests"

on:
pull_request:
paths:
- ".github/actions/**"
- ".github/workflows/test_local_upgrade.yaml"
release:
types:
- prereleased
workflow_dispatch:


concurrency:
group: ${{ github.ref_name }}
cancel-in-progress: true

env:
NEBARI_IMAGE_TAG: "main"

jobs:
test-local-upgrade:
runs-on: "cirun-runner--${{ github.run_id }}"
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Setup runner for local deployment
uses: ./.github/actions/setup-local

- name: Setup Python
uses: conda-incubator/setup-miniconda@v3
env:
CONDA: /home/runnerx/miniconda3
with:
auto-update-conda: true
python-version: "3.11"
miniconda-version: "latest"

- name: Install latest stable Nebari release
run: pip install nebari

- name: Initialize Nebari config for local deployment
id: init
uses: ./.github/actions/init-local

- name: Extract old Nebari version
run: |
OLD_NEBARI_VERSION=$(grep 'nebari_version: ' ${{ steps.init.outputs.config }} | sed 's/nebari_version: //')
echo "OLD_NEBARI_VERSION=${OLD_NEBARI_VERSION}" | tee --append "${GITHUB_ENV}"
- name: Deploy Nebari
working-directory: ${{ steps.init.outputs.directory }}
run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt

- name: Health check before upgrade
id: health-check-before
uses: ./.github/actions/health-check
with:
domain: ${{ steps.init.outputs.domain }}

- name: Install current Nebari
run: pip install --upgrade .

- name: Upgrade Nebari config
run: |
git add --force ${{ steps.init.outputs.config }}
nebari upgrade --config ${{ steps.init.outputs.config }} --attempt-fixes
git diff
nebari validate --config ${{ steps.init.outputs.config }}
- name: Redeploy Nebari
working-directory: ${{ steps.init.outputs.directory }}
run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt

- name: Health check after upgrade
id: health-check-after
uses: ./.github/actions/health-check
with:
domain: ${{ steps.init.outputs.domain }}
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ repos:
exclude: "^src/_nebari/template/"

- repo: https://github.com/crate-ci/typos
rev: v1.26.0
rev: v1.27.0
hooks:
- id: typos

Expand All @@ -61,7 +61,7 @@ repos:
args: ["--line-length=88", "--exclude=/src/_nebari/template/"]

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.6.9
rev: v0.7.2
hooks:
- id: ruff
args: ["--fix"]
Expand All @@ -77,7 +77,7 @@ repos:

# terraform
- repo: https://github.com/antonbabenko/pre-commit-terraform
rev: v1.96.1
rev: v1.96.2
hooks:
- id: terraform_fmt
args:
Expand Down
Loading

0 comments on commit 9f7ba1f

Please sign in to comment.