diff --git a/.github/workflows/image-publish.yml b/.github/workflows/image-publish.yml index 2babbdc5..c32cf780 100644 --- a/.github/workflows/image-publish.yml +++ b/.github/workflows/image-publish.yml @@ -59,7 +59,7 @@ jobs: github_token: ${{ github.token }} workflow: ".github/workflows/import_packages.yml" workflow_conclusion: success - name: backup_weaviate + name: sqlite_vectordb_file name_is_regexp: true skip_unpack: false if_no_artifact_found: ignore @@ -67,8 +67,6 @@ jobs: id: fetch_commit_fe_sha run: | echo "LATEST_RELEASE=$(curl -s "https://api.github.com/repos/stacklok/codegate-ui/releases/latest" -H "Authorization: Bearer ${{ secrets.GH_CI_TOKEN }}" | grep '"zipball_url":' | cut -d '"' -f 4)" >> $GITHUB_ENV - - name: Rename to accommodate to image - run: mv ./backup_weaviate ./weaviate_backup - name: Download git lfs dependencies run: | git lfs install diff --git a/.github/workflows/import_packages.yml b/.github/workflows/import_packages.yml index e05a4b36..e07f0484 100644 --- a/.github/workflows/import_packages.yml +++ b/.github/workflows/import_packages.yml @@ -51,33 +51,31 @@ jobs: github_token: ${{ github.token }} workflow: ".github/workflows/import_packages.yml" workflow_conclusion: success - name: backup_weaviate + name: sqlite_vectordb_file path: /tmp/ name_is_regexp: true skip_unpack: false - if_no_artifact_found: ignore + if_no_artifact_found: ignore - - name: Create folder if artifact download is not enabled - if: ${{ github.event.inputs.enable_artifact_download == 'false' }} + - name: Install Poetry run: | - mkdir -p /tmp/backup_weaviate - echo "Folder ./backup_weaviate created because artifact download is disabled." + curl -sSL https://install.python-poetry.org | python3 - - - name: Run sync + - name: Add Poetry to PATH run: | - export PYTHONPATH=$PYTHONPATH:./ - export BACKUP_FILESYSTEM_PATH=/tmp/backup_weaviate/ - export BACKUP_FOLDER=backup - # Conditionally export the variables only if artifact download is enabled - if [ "${{ github.event.inputs.enable_artifact_download }}" == "true" ]; then - python scripts/import_packages.py --jsonl-dir /tmp/jsonl-files/ - else - python scripts/import_packages.py --restore-backup False --jsonl-dir /tmp/jsonl-files/ - fi + echo "source $HOME/.poetry/env" >> $GITHUB_ENV - - name: 'Upload Backup Files' + - name: Install dependencies with Poetry + run: | + poetry install + + - name: 'Run import_packages.py with poetry' + run: | + poetry run python scripts/import_packages.py --jsonl-dir /tmp --vec-db-path /tmp/sqlite_data/vectordb.db + + - name: 'Upload SQLite Vector DB File' uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4 with: - name: backup_weaviate - path: /tmp/backup_weaviate/backup* + name: sqlite_vectordb_file + path: /tmp/sqlite_data/vectordb.db retention-days: 90