diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index f10e7a9f1..4ee1eb40d 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -27,46 +27,62 @@ jobs: name: Build COMPAS runs-on: ubuntu-22.04 + container: teamcompas/compas:latest + # TODO: Switch to GHCR when package is made public + # container: ghcr.io/teamcompas/compas:latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - cache: pip - cache-dependency-path: setup.py - # - name: Install TeXLive - # uses: teatimeguest/setup-texlive-action@v3.3.4 - - name: Install dependencies on ubuntu - run: | - cd misc/cicd-scripts - chmod 755 linux-dependencies - ./linux-dependencies - - name: Build Compas + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build COMPAS from source run: | - cd src && make -j $(nproc) -f Makefile + echo "Building COMPAS from source..." + cd src + make clean 2>/dev/null || echo 'No existing build to clean' + make -j $(nproc) -f Makefile ./COMPAS -v - - name: Install python utils + echo "COMPAS build completed successfully!" + + - name: Install Python dependencies run: | - pip install --upgrade pip + echo "Installing Python dependencies..." + python3 -m pip install --upgrade pip pip install -e .[dev] + echo "Python dependencies installed!" + - name: Run example COMPAS job run: | + echo "Running example COMPAS job..." export COMPAS_EXECUTABLE_PATH=${GITHUB_WORKSPACE}/src/COMPAS cd ${GITHUB_WORKSPACE}/py_tests/test_data/ chmod 755 run.sh + echo "Contents of run.sh:" cat run.sh ./run.sh - - name: Run pytests + echo "Example job completed!" + + - name: Run pytest suite run: | + echo "Running pytest suite..." cd ${GITHUB_WORKSPACE} export COMPAS_ROOT_DIR=${GITHUB_WORKSPACE} - jupyter-kernelspec list + + if command -v jupyter-kernelspec &> /dev/null; then + echo "Available Jupyter kernels:" + jupyter-kernelspec list + fi + pytest --cov=compas_python_utils/ py_tests/ -m 'not webtest' pytest --cov=compas_python_utils/ --cov-append py_tests/ -m webtest - ls py_tests/test_artifacts + + echo "Test artifacts:" + ls py_tests/test_artifacts/ 2>/dev/null || echo "No test artifacts found" + coverage html coverage-badge -o coverage_badge.svg -f + echo "All tests completed successfully!" + - name: Archive code coverage results uses: actions/upload-artifact@v4 with: @@ -74,17 +90,67 @@ jobs: path: | htmlcov/ coverage_badge.svg + - name: Archive COMPAS detailed-evolution plot id: upload uses: actions/upload-artifact@v4 with: - name: ${{ env.ARTIFACT_NAME }} + name: evolution-plot-${{ github.run_number }} path: ${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_NAME }} if-no-files-found: error + - name: Test Summary run: | - echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- Compas Build: Success" >> $GITHUB_STEP_SUMMARY - echo "- Python Utils Installation: Success" >> $GITHUB_STEP_SUMMARY + echo "### COMPAS CI Results" >> $GITHUB_STEP_SUMMARY + echo "- COMPAS Build: Success" >> $GITHUB_STEP_SUMMARY + echo "- Python Dependencies: Success" >> $GITHUB_STEP_SUMMARY echo "- Example COMPAS Job: Success" >> $GITHUB_STEP_SUMMARY - echo "- Pytest Execution: Success" >> $GITHUB_STEP_SUMMARY + echo "- Pytest Suite: Success" >> $GITHUB_STEP_SUMMARY + + comment-with-plot: + name: Comment PR with Evolution Plot + runs-on: ubuntu-22.04 + container: docker://ghcr.io/iterative/cml:0-dvc2-base1 + needs: compas + if: github.event_name == 'pull_request' + + env: + ARTIFACT_NAME: detailedEvolutionPlot.png + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download evolution plot + uses: actions/download-artifact@v4 + with: + name: evolution-plot-${{ github.run_number }} + + - name: Create report with evolution plot + env: + REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "## ✅ COMPAS Build Successful!" >> report.md + echo "" >> report.md + echo "| Item | Value |" >> report.md + echo "|------|-------|" >> report.md + echo "| **Commit** | [\`$(echo $GITHUB_SHA | cut -c1-7)\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}) |" >> report.md + echo "| **Logs** | [View workflow](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) |" >> report.md + echo "" >> report.md + + if [ -f "${{ env.ARTIFACT_NAME }}" ]; then + echo "### Detailed Evolution Plot" >> report.md + echo "
Click to view evolution plot" >> report.md + echo "" >> report.md + echo "![](./${{ env.ARTIFACT_NAME }})" >> report.md + echo "
" >> report.md + else + echo "### ⚠️ Evolution plot not found" >> report.md + fi + + echo "" >> report.md + echo "---" >> report.md + echo "Generated by COMPAS CI " >> report.md + + # Post the report using CML + cml comment create report.md \ No newline at end of file diff --git a/.github/workflows/pr_artifact_url_commenter.yml b/.github/workflows/pr_artifact_url_commenter.yml deleted file mode 100644 index 90a3604d3..000000000 --- a/.github/workflows/pr_artifact_url_commenter.yml +++ /dev/null @@ -1,254 +0,0 @@ -name: Comment on Pull request - -on: - workflow_run: - workflows: ['COMPAS compile test'] - types: - - completed - -jobs: - comment: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: Get Artifact and Pull request info - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WORKFLOW_RUN_EVENT_OBJ: ${{ toJSON(github.event.workflow_run) }} - OWNER: ${{ github.repository_owner }} - REPO: ${{ github.event.repository.name }} - run: | - # Extract workflow run info - PREVIOUS_JOB_ID=$(jq -r '.id' <<< "$WORKFLOW_RUN_EVENT_OBJ") - echo "PREVIOUS_JOB_ID=$PREVIOUS_JOB_ID" >> "$GITHUB_ENV" - - HEAD_SHA="${{ github.event.workflow_run.head_sha }}" - echo "HEAD_SHA=$HEAD_SHA" >> "$GITHUB_ENV" - - # Get PR number from the workflow run - echo "Looking for PR associated with commit $HEAD_SHA..." - PR_NUMBER=$(gh api "/repos/$OWNER/$REPO/commits/$HEAD_SHA/pulls" \ - --jq '.[0].number // empty' 2>/dev/null || echo "") - - if [[ -n "$PR_NUMBER" && "$PR_NUMBER" != "null" ]]; then - echo "Found PR #$PR_NUMBER" - echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV" - - # Look for the specific artifact - ARTIFACT_NAME="detailedEvolutionPlot.png" - echo "Searching for artifact: $ARTIFACT_NAME from workflow run $PREVIOUS_JOB_ID..." - - # Retry loop with better error handling - ARTIFACT_ID="" - for i in {1..8}; do - echo "Attempt $i/8: Looking for artifact..." - - # Use more robust jq filter - ARTIFACT_ID=$(gh api "/repos/$OWNER/$REPO/actions/artifacts" \ - --jq --arg workflow_id "$PREVIOUS_JOB_ID" --arg name "$ARTIFACT_NAME" \ - '.artifacts[] | select(.workflow_run.id == ($workflow_id | tonumber) and .expired == false and .name == $name) | .id' \ - 2>/dev/null | head -n1 || echo "") - - if [[ -n "$ARTIFACT_ID" ]]; then - echo "Found artifact with ID: $ARTIFACT_ID" - break - fi - - echo "Artifact not available yet, waiting 12s before retry..." - sleep 12 - done - - if [[ -z "$ARTIFACT_ID" ]]; then - echo "Warning: Could not find artifact '$ARTIFACT_NAME' after 8 attempts" - echo "This might be expected if the artifact generation failed or is still processing." - fi - - echo "ARTIFACT_ID=${ARTIFACT_ID:-}" >> "$GITHUB_ENV" - else - echo "No PR found for this workflow run (this is normal for direct pushes to main/dev)" - echo "PR_NUMBER=" >> "$GITHUB_ENV" - echo "ARTIFACT_ID=" >> "$GITHUB_ENV" - fi - - - name: Download artifact - if: env.PR_NUMBER != '' && env.ARTIFACT_ID != '' - run: | - echo "Downloading artifact ${{ env.ARTIFACT_ID }}..." - - # Download with better error handling - if gh api "/repos/${{ github.repository }}/actions/artifacts/${{ env.ARTIFACT_ID }}/zip" --output artifact.zip; then - echo "Successfully downloaded artifact.zip" - - # Extract and verify - if unzip -q artifact.zip; then - rm artifact.zip - - if [[ -f "detailedEvolutionPlot.png" ]]; then - echo "Found detailedEvolutionPlot.png" - echo "File size: $(du -h detailedEvolutionPlot.png | cut -f1)" - # Verify it's actually a PNG file - if file detailedEvolutionPlot.png | grep -q "PNG image"; then - echo "Verified as valid PNG image" - else - echo "Warning: File may not be a valid PNG" - fi - else - echo "Error: detailedEvolutionPlot.png not found in extracted files" - echo "Available files:" - ls -la - exit 1 - fi - else - echo "Error: Failed to extract artifact.zip" - exit 1 - fi - else - echo "Error: Failed to download artifact" - exit 1 - fi - - - name: Create success comment with artifact - if: env.PR_NUMBER != '' && env.ARTIFACT_ID != '' - env: - JOB_PATH: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ env.PREVIOUS_JOB_ID }}" - ARTIFACT_DL: "${{ github.server_url }}/${{ github.repository }}/suites/${{ github.event.workflow_run.check_suite_id }}/artifacts/${{ env.ARTIFACT_ID }}" - uses: actions/github-script@v7 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const fs = require('fs'); - - try { - // Read and validate the image - if (!fs.existsSync('detailedEvolutionPlot.png')) { - throw new Error('detailedEvolutionPlot.png file not found'); - } - - const stats = fs.statSync('detailedEvolutionPlot.png'); - console.log(`Image file size: ${(stats.size / 1024).toFixed(1)} KB`); - - if (stats.size === 0) { - throw new Error('detailedEvolutionPlot.png is empty'); - } - - // GitHub has a ~65KB limit for image embeds, warn if approaching - if (stats.size > 50000) { - console.log('Warning: Image is quite large, may not display properly in GitHub'); - } - - const imageData = fs.readFileSync('detailedEvolutionPlot.png', {encoding: 'base64'}); - const shortSha = process.env.HEAD_SHA.substring(0, 7); - - const commentBody = \`![badge] - - ## Build Successful! - - Your COMPAS compilation and test suite completed successfully! - - | Item | Value | - |------|-------| - | **Commit** | [\\\`\${shortSha}\\\`](https://github.com/\${context.repo.owner}/\${context.repo.repo}/commit/\${process.env.HEAD_SHA}) | - | **Logs** | [View workflow logs](\${process.env.JOB_PATH}) | - | **Artifact** | [Download evolution plot](\${process.env.ARTIFACT_DL}) | - - ### Detailed Evolution Plot - -
- Click to view/hide evolution plot - - ![Detailed Evolution Plot](data:image/png;base64,\${imageData}) - -
- - --- - Generated by COMPAS CI • [View workflow run](\${process.env.JOB_PATH}) - - [badge]: https://img.shields.io/badge/Build_Success-28a745?style=for-the-badge&logo=github-actions&logoColor=white\`; - - await github.rest.issues.createComment({ - issue_number: parseInt(process.env.PR_NUMBER), - owner: context.repo.owner, - repo: context.repo.repo, - body: commentBody - }); - - console.log(`Successfully commented on PR #${process.env.PR_NUMBER} with artifact`); - - } catch (error) { - console.error('Error creating comment with artifact:', error); - - // Fallback: create comment without image - const fallbackBody = \`![badge] - - ## Build Successful! - - Your COMPAS compilation completed successfully, but there was an issue displaying the evolution plot. - - | Item | Value | - |------|-------| - | **Commit** | [\\\`\${process.env.HEAD_SHA.substring(0, 7)}\\\`](https://github.com/\${context.repo.owner}/\${context.repo.repo}/commit/\${process.env.HEAD_SHA}) | - | **Logs** | [View workflow logs](\${process.env.JOB_PATH}) | - | **Artifact** | [Download evolution plot](\${process.env.ARTIFACT_DL}) | - - **Note:** Evolution plot could not be embedded (\${error.message}) - - --- - Generated by COMPAS CI • [View workflow run](\${process.env.JOB_PATH}) - - [badge]: https://img.shields.io/badge/Build_Success-28a745?style=for-the-badge&logo=github-actions&logoColor=white\`; - - await github.rest.issues.createComment({ - issue_number: parseInt(process.env.PR_NUMBER), - owner: context.repo.owner, - repo: context.repo.repo, - body: fallbackBody - }); - - console.log(`Created fallback comment on PR #${process.env.PR_NUMBER}`); - } - - - name: Create success comment without artifact - if: env.PR_NUMBER != '' && env.ARTIFACT_ID == '' - env: - JOB_PATH: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ env.PREVIOUS_JOB_ID }}" - uses: actions/github-script@v7 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const shortSha = process.env.HEAD_SHA.substring(0, 7); - - const commentBody = \`![badge] - - ## Build Successful! - - Your COMPAS compilation and test suite completed successfully! - - | Item | Value | - |------|-------| - | **Commit** | [\\\`\${shortSha}\\\`](https://github.com/\${context.repo.owner}/\${context.repo.repo}/commit/\${process.env.HEAD_SHA}) | - | **Logs** | [View workflow logs](\${process.env.JOB_PATH}) | - - **Note:** The detailed evolution plot artifact was not found or is not yet available. This might be expected if the plot generation step was skipped or failed. - - --- - Generated by COMPAS CI • [View workflow run](\${process.env.JOB_PATH}) - - [badge]: https://img.shields.io/badge/Build_Success-28a745?style=for-the-badge&logo=github-actions&logoColor=white\`; - - await github.rest.issues.createComment({ - issue_number: parseInt(process.env.PR_NUMBER), - owner: context.repo.owner, - repo: context.repo.repo, - body: commentBody - }); - - console.log(`Successfully commented on PR #${process.env.PR_NUMBER} (no artifact)`); - - - name: Debug info for non-PR runs - if: env.PR_NUMBER == '' - run: | - echo "This workflow run was not associated with a pull request" - echo "Head SHA: ${{ github.event.workflow_run.head_sha }}" - echo "Event: ${{ github.event.workflow_run.event }}" - echo "Branch: ${{ github.event.workflow_run.head_branch }}" - echo "This is normal for direct pushes to main/dev branches" \ No newline at end of file diff --git a/compas_python_utils/cosmic_integration/binned_cosmic_integrator/io.py b/compas_python_utils/cosmic_integration/binned_cosmic_integrator/io.py index 135d7239a..32115f59a 100644 --- a/compas_python_utils/cosmic_integration/binned_cosmic_integrator/io.py +++ b/compas_python_utils/cosmic_integration/binned_cosmic_integrator/io.py @@ -25,14 +25,14 @@ def recursively_save_dict_contents_to_group(h5file: h5py.File, group: str, dic: def encode_for_hdf5(key, item): - if isinstance(item, np.int_): - item = int(item) - elif isinstance(item, np.float_): - item = float(item) - elif isinstance(item, np.complex_): - item = complex(item) + if isinstance(item, (np.generic, int, float, complex)): + if isinstance(item, np.integer): + item = int(item) + elif isinstance(item, np.floating): + item = float(item) + elif isinstance(item, np.complexfloating): + item = complex(item) if isinstance(item, np.ndarray): - # Numpy's wide unicode strings are not supported by hdf5 if item.dtype.kind == 'U': item = np.array(item, dtype='S') if isinstance(item, (np.ndarray, int, float, complex, str, bytes)): @@ -43,7 +43,7 @@ def encode_for_hdf5(key, item): if len(item) == 0: output = item elif isinstance(item[0], (str, bytes)) or item[0] is None: - output = list() + output = [] for value in item: if isinstance(value, str): output.append(value.encode("utf-8")) @@ -68,16 +68,17 @@ def decode_from_hdf5(item): elif isinstance(item, bytes) and item == b"__none__": output = None elif isinstance(item, (bytes, bytearray)): - output = item.decode() + output = item.decode("utf-8") elif isinstance(item, np.ndarray): if item.size == 0: output = item - elif "|S" in str(item.dtype) or isinstance(item[0], bytes): - output = [it.decode() for it in item] + elif "|S" in str(item.dtype) or (item.dtype.kind == 'S') or \ + (item.size > 0 and isinstance(item.flat[0], bytes)): + output = [it.decode("utf-8") for it in item] else: output = item - elif isinstance(item, np.bool_): + elif isinstance(item, (np.bool_, bool)): output = bool(item) else: output = item - return output + return output \ No newline at end of file