Merge pull request #1735 from oscal-compass/develop

chore: Trestle release

.github/workflows/python-push.yml

@@ -9,6 +9,8 @@ on:
     branches:
       - main
 
+permissions: {}
+
 jobs:
   set-versions:
     runs-on: ubuntu-latest
@@ -110,11 +112,16 @@ jobs:
       url: https://pypi.org/p/compliance-trestle
     if: github.ref == 'refs/heads/main' && github.repository == 'oscal-compass/compliance-trestle'
     steps:
+    - uses: actions/create-github-app-token@v1
+      id: app-token
+      with:
+        app-id: ${{ secrets.APP_ID }}
+        private-key: ${{ secrets.PRIVATE_KEY }}
     - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-        token: ${{ secrets.ADMIN_PAT }} 
+        token: ${{ steps.app-token.outputs.token }}
     - name: Set up Python ${{ needs.set-versions.outputs.max }}
       uses: actions/setup-python@v5
       with:
@@ -126,9 +133,9 @@ jobs:
     # This action uses Python Semantic Release v8
     - name: Python Semantic Release
       id: release
-      uses: python-semantic-release/python-semantic-release@v9.8.0
+      uses: python-semantic-release/python-semantic-release@v9.8.8
       with:
-        github_token: ${{ secrets.ADMIN_PAT }}
+        github_token: ${{ steps.app-token.outputs.token }}
 
     - name: Publish package distributions to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
@@ -137,10 +144,10 @@ jobs:
       if: steps.release.outputs.released == 'true'
 
     - name: Publish package distributions to GitHub Releases
-      uses: python-semantic-release/upload-to-gh-release@v9.8.0
+      uses: python-semantic-release/upload-to-gh-release@v9.8.9
       if: steps.release.outputs.released == 'true'
       with:
-        github_token: ${{ secrets.ADMIN_PAT }}
+        github_token: ${{ steps.app-token.outputs.token }}
 
   deploy-docs:
     runs-on: ubuntu-latest
@@ -151,11 +158,16 @@ jobs:
     # Temporary hack: allow develop as well as master to deploy docs.
     if: github.ref == 'refs/heads/main'
     steps:
+    - uses: actions/create-github-app-token@v1
+      id: app-token
+      with:
+        app-id: ${{ secrets.APP_ID }}
+        private-key: ${{ secrets.PRIVATE_KEY }}
     - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-        token: ${{ secrets.ADMIN_PAT }} 
+        token: ${{ steps.app-token.outputs.token }}
     - name: Set up Python ${{ needs.set-versions.outputs.max }}
       uses: actions/setup-python@v5
       # This is deliberately not using a custom credential as it relies on native github actions token to have push rights.
@@ -178,16 +190,31 @@ jobs:
       cancel-in-progress: true
     if: github.ref == 'refs/heads/main'
     steps:
+    - uses: actions/create-github-app-token@v1
+      id: app-token
+      with:
+        app-id: ${{ secrets.APP_ID }}
+        private-key: ${{ secrets.PRIVATE_KEY }}
     - uses: actions/checkout@v4
       with:
         submodules: true
         ref: main
         fetch-depth: 0
-        token: ${{ secrets.ADMIN_PAT }}
+        token: ${{ steps.app-token.outputs.token }}
+    - name: Get GitHub App User ID
+      id: get-user-id
+      run: echo "user-id=$(gh api "/users/${{ env.SLUG }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT"
+      env:
+        GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        SLUG: ${{ steps.app-token.outputs.app-slug }}
     - name: Configure Git
       run: |
-        git config user.name "Vikas Agarwal"
-        git config user.email "<>"
+        git config --global user.name '${{ env.SLUG }}[bot]'
+        git config --global user.email '${{ env.ID }}+${{ env.SLUG }}[bot]@users.noreply.github.com'
+      env:
+        SLUG: ${{ steps.app-token.outputs.app-slug }}
+        ID: ${{ steps.get-user-id.outputs.user-id }}
+      # https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
     - name: Merge Main to Develop
       run: |
         git checkout develop

.github/workflows/python-test.yml

@@ -151,7 +151,7 @@ jobs:
 
     - name: Upload artifact
       if: steps.core-version.outputs.core == 'true'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: coverage
         path: coverage.xml
@@ -184,7 +184,7 @@ jobs:
       run: |
         make develop
     - name: Get coverage
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4
       with:
         name: coverage
     - name: SonarCloud Scan

.github/workflows/stale.yml

@@ -0,0 +1,38 @@
+name: Stale Issues and PRs
+on:
+  schedule:
+  - cron: '17 1 * * *'
+  workflow_dispatch:
+
+permissions:
+    contents: read
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+        issues: write
+        pull-requests: write
+    env:
+        STALE_WARNING_DAYS: 90
+        STALE_CLOSURE_DAYS: 30
+    steps:
+    - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9
+      with:
+        stale-issue-label: stale
+        exempt-issue-labels: good-first-issue, help-wanted, exempt-from-stale
+        stale-issue-message: >
+          This issue has been automatically marked as stale because it has not had activity within ${{ env.STALE_WARNING_DAYS }} days.
+          It will be automatically closed if no further activity occurs within ${{ env.STALE_CLOSURE_DAYS }} days.
+        close-issue-message: >
+          This issue has been automatically closed due to inactivity.
+        days-before-issue-stale: ${{ env.STALE_WARNING_DAYS }}
+        days-before-issue-close: ${{ env.STALE_CLOSURE_DAYS }}
+        stale-pr-label: stale
+        stale-pr-message: >
+          This pull request has been automatically marked as stale because it has not had activity within ${{ env.STALE_WARNING_DAYS }} days.
+          It will be automatically closed if no further activity occurs within ${{ env.STALE_CLOSURE_DAYS }} days.
+        close-pr-message: >
+          This pull request has been automatically closed due to inactivity. Please reopen if this PR is still being worked on.
+        days-before-pr-stale: ${{ env.STALE_WARNING_DAYS }}
+        days-before-pr-close: ${{ env.STALE_CLOSURE_DAYS }}

.pre-commit-config.yaml

@@ -1,6 +1,7 @@
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0  # Use the ref you want to point at
+    rev: v5.0.0  # Use the ref you want to point at
     hooks:
       - id: check-merge-conflict
       - id: check-yaml
@@ -13,13 +14,13 @@ repos:
       - id: yapf
         args: [--in-place, --parallel, --recursive, --style, .yapf-config]
         files: "^(trestle|tests|scripts)"
-        stages: [commit]
+        stages: [pre-commit]
         additional_dependencies: [toml]
   - repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 7.1.1
     hooks:
       - id: flake8
-        args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028", "--illegal-import-packages=filecmp"]
+        args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028"]
         additional_dependencies:
           [
             flake8-2020,
@@ -37,14 +38,13 @@ repos:
             flake8-quotes,
             flake8-string-format,
             flake8-use-fstring,
-            flake8-illegal-import,
             pep8-naming,
           ]
         files: "^(tests|scripts)"
         exclude: "(oscal/|third_party)"
-        stages: [commit]
+        stages: [pre-commit]
       - id: flake8
-        args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028", "--illegal-import-packages=filecmp"]
+        args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028"]
         additional_dependencies:
           [
             flake8-2020,
@@ -62,16 +62,15 @@ repos:
             flake8-quotes,
             flake8-string-format,
             flake8-use-fstring,
-            flake8-illegal-import,
             pep8-naming,
             flake8-bandit,
             dlint  
           ]
         files: "^(trestle)"
         exclude: "(oscal/)"
-        stages: [commit]
+        stages: [pre-commit]
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.16
+    rev: 0.7.17
     hooks:
     - id: mdformat
       exclude: "CHANGELOG.md|docs/mkdocs_code_of_conduct.md|docs/maintainers.md|docs/api_reference|tests/data/author|docs/contributing/mkdocs_contributing.md|tests/data/jinja_markdown_include|tests/data/jinja_cmd/number_captions_data.md|tests/data/jinja_cmd/number_captions_expected_output.md"
@@ -80,3 +79,4 @@ repos:
       - mdformat-config
       - mdformat-frontmatter
       - mdformat-gfm
+

CONTRIBUTING.md

@@ -32,6 +32,17 @@ review to indicate acceptance. A change requires LGTMs from one of the maintaine
 
 For a list of the maintainers, see the [maintainers](https://oscal-compass.github.io/compliance-trestle/maintainers/) page.
 
+### Trestle updating, testing and release logistics
+
+Contributors should make a working copy (branch or fork) from the develop branch of `trestle`.
+Contributors should update the working copy with changes, then create a pull request to merge into the develop branch.
+Contributors must include test cases to meet at least the minimum code coverage requirements.
+Upon approval from reviewer(s), the working copy is squashed and merged into the develop branch.
+Upon a cadence established by the maintainers, the develop branch is merged into the main branch and a new release is uniquely numbered and pushed to [pypi](https://pypi.org/project/compliance-trestle/).
+
+`trestle` employs `semantic release` to automatically control release numbering.
+Code deliveries should be tagged with prefix `fix:` for changes that are bug fixes or `feat:` for changes that are new features.  See [allowed_tags](https://python-semantic-release.readthedocs.io/en/latest/commit-parsing.html#:~:text=The%20default%20configuration%20options%20for%20semantic_release.commit_parser.AngularCommitParser%20are%3A) for a list of supported tags.
+
 ### Trestle merging and release workflow
 
 `trestle` is operating on a simple, yet opinionated, method for continuous integration. It's designed to give developers a coherent understanding of the objectives of other past developers.
@@ -86,7 +97,7 @@ Software License 2.0. Using the SPDX format is the simplest approach.
 e.g.
 
 ```text
-# Copyright (c) 2020 IBM Corp. All rights reserved.
+# Copyright (c) 2024 The OSCAL Compass Authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -164,7 +175,7 @@ Test discovery should be automatic when you select a .py file for editing. After
 
 Sometimes the discovery fails - and you may need to resort to uninstalling the python extension and reinstalling it - perhaps also shutting down code and restarting.  This is a lightweight operation and seems to be safe and usually fixes any problems.
 
-Test disovery will fail or stop if any of the tests have errors in them - so be sure to monitor the Problems panel at the bottom for problems in the code.
+Test discovery will fail or stop if any of the tests have errors in them - so be sure to monitor the Problems panel at the bottom for problems in the code.
 
 Note that there are many panels available in Output - so be sure to check `Python Test Log` for errors and output from the tests.
 
@@ -179,8 +190,22 @@ Trestle relies on reference data from two NIST repositories for testing:
 
 Both of these repositories are submodules in the trestle project. In order to develop / test trestle the submodules must be checked out with `git submodule update --init` or `make submodules`.
 
+### Code testing
+
+Tests must exist for at least 96% of trestle Python code. To determine the code coverage locally during development:
+
+```bash
+make test-cov
+```
+
+A PR without sufficient test coverage will fail the trestle CI process and will not be approved or merged.
+
+Additional code scrutiny is applied in the trestle CI pipeline by [SonarCloud](https://sonarcloud.io/project/overview?id=compliance-trestle). Any failures must be addressed before code merging.
+
 ### Code style and formating
 
+Python code should generally follow [PEP 8](https://peps.python.org/pep-0008/).
+
 `trestle` uses [yapf](https://github.com/google/yapf) for code formatting and [flake8](https://flake8.pycqa.org/en/latest/) for code styling.  It also uses [pre-commit](https://pre-commit.com/) hooks that are integrated into the development process and the CI. When you run `make develop` you are ensuring that the pre-commit hooks are installed and updated to their latest versions for this repository. This ensures that all delivered code has been properly formatted
 and passes the linter rules.  See the [pre-commit configuration file](https://github.com/oscal-compass/compliance-trestle/blob/develop/.pre-commit-config.yaml) for details on
 `yapf` and `flake8` configurations.
@@ -248,4 +273,4 @@ ______________________________________________________________________
 
 ##### Overview of process to take OSCAL models and upgrade trestle Python code
 
-<img src="images/trestle-OSCAL-upgrade.png" style="width:600px;height:500px;border: 1px solid #000;padding:10px;">
+![](images/trestle-OSCAL-upgrade.png)

Makefile

@@ -86,7 +86,7 @@ docs-automation::
 	python ./scripts/website_automation.py
 
 docs-validate:: docs-automation
-	mkdocs build -c -s
+	mkdocs build -v -c -s
 	rm -rf site
 
 docs-serve: docs-automation
@@ -117,4 +117,4 @@ pylint-test:
 	pylint tests --rcfile=.pylintrc_tests
 
 check-for-changes:
-	python scripts/have_files_changed.py -u
+	python scripts/have_files_changed.py -u

README.md

@@ -6,7 +6,7 @@
 ![[Code Coverage](https://sonarcloud.io/dashboard?id=compliance-trestle)](https://sonarcloud.io/api/project_badges/measure?project=compliance-trestle&metric=coverage)
 ![[Quality gate](https://sonarcloud.io/dashboard?id=compliance-trestle)](https://sonarcloud.io/api/project_badges/measure?project=compliance-trestle&metric=alert_status)
 ![[Pypi](https://pypi.org/project/compliance-trestle/)](https://img.shields.io/pypi/dm/compliance-trestle)
-![GitHub Actions status](https://img.shields.io/github/workflow/status/oscal-compass/compliance-trestle/Trestle%20PR%20pipeline?event=push)
+![GitHub Actions status](https://github.com/oscal-compass/compliance-trestle/actions/workflows/python-test.yml/badge.svg?branch=develop)
 
 Trestle is an ensemble of tools that enable the creation, validation, and governance of documentation artifacts for compliance needs. It leverages NIST's [OSCAL](https://pages.nist.gov/OSCAL/) as a standard data format for interchange between tools and people, and provides an opinionated approach to OSCAL adoption.
 
@@ -112,7 +112,7 @@ If you would like to see the detailed LICENSE click [here](LICENSE).
 Consult [contributors](https://github.com/oscal-compass/compliance-trestle/graphs/contributors) for a list of authors and [maintainers](MAINTAINERS.md) for the core team.
 
 ```text
-# Copyright (c) 2020 IBM Corp. All rights reserved.
+# Copyright (c) 2024 The OSCAL Compass Authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -125,5 +125,17 @@ Consult [contributors](https://github.com/oscal-compass/compliance-trestle/graph
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 ```
+
+______________________________________________________________________
+
+We are a Cloud Native Computing Foundation sandbox project.
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://www.cncf.io/wp-content/uploads/2022/07/cncf-white-logo.svg">
+  <img src="https://www.cncf.io/wp-content/uploads/2022/07/cncf-color-bg.svg" width=300 />
+</picture>
+
+The Linux Foundation® (TLF) has registered trademarks and uses trademarks. For a list of TLF trademarks, see [Trademark Usage](https://www.linuxfoundation.org/legal/trademark-usage)".
+
+*Trestle was originally created by IBM.*

ROADMAP.md

@@ -18,6 +18,12 @@ We use 12-week iterations for high-level initiatives that must be broken down an
 
 Each roadmap item is represented as a GitHub Issue. Discussions or any feedback on current roadmap items can take place in threads on the corresponding issue. To stay up to date with roadmap items, please join the OSCAL-Compass community [meetings](https://github.com/oscal-compass/community).
 
+**Help Us Grow `compliance-trestle`!** Community contributions are essential to the project's success. To ensure we prioritize the most active and relevant issues, we're using `actions/stale` to automatically close older ones. Your participation is greatly appreciated!
+
+#### Our Triaging Process
+
+All new issues in `compliance-trestle` are added to the [project](https://github.com/orgs/oscal-compass/projects/2) with a status of New. All issues that need to be triaged are viewable [here](https://github.com/orgs/oscal-compass/projects/2/views/8). Once there is enough information to move forward on the issue, it can be moved to the `Backlog` by applying the `Backlog` label and moving the status to `Ready`.
+
 ### How to add an item to the roadmap?
 
 **Contributors are encouraged to get feedback early by submitting issues to new work they would like to complete and getting feedback.**

docs/api_reference/trestle.core.jinja.base.md

@@ -0,0 +1,2 @@
+::: trestle.core.jinja.base
+handler: python

docs/api_reference/trestle.core.jinja.ext.md

@@ -0,0 +1,2 @@
+::: trestle.core.jinja.ext
+handler: python

docs/api_reference/trestle.core.jinja.filters.md

@@ -0,0 +1,2 @@
+::: trestle.core.jinja.filters
+handler: python

docs/api_reference/trestle.core.jinja.tags.md

@@ -0,0 +1,2 @@
+::: trestle.core.jinja.tags
+handler: python

docs/api_reference/trestle.core.plugins.md

@@ -0,0 +1,2 @@
+::: trestle.core.plugins
+handler: python

docs/cli.md

@@ -113,11 +113,11 @@ Users can query the contents of files using `trestle describe`, and probe the co
 
 OSCAL models are rich and contain multiple nested data structures. Given this, a mechanism is required to address _elements_ /_attributes_ within an oscal object.
 
-This accessing method is called 'element path' and is similar to _jsonPath_. Commands provide element path by a `-e` argument where available, e.g. trestle split -f catalog.json -e 'catalog.metadata.\*'. This path is used whenever specifying an attribute or model, rather than exposing trestle's underlying object model name. Users can refer to [NIST's json outline](https://pages.nist.gov/OSCAL/reference/latest/complete/json-outline/) to understand object names in trestle.
+This accessing method is called 'element path' and is similar to _jsonPath_. Commands provide element path by a `-e` argument where available, e.g. trestle split -f catalog.json -e 'catalog.metadata.\*'. This path is used whenever specifying an attribute or model, rather than exposing trestle's underlying object model name. Users can refer to [NIST's json outline](https://pages.nist.gov/OSCAL-Reference/models/latest/complete/json-outline/) to understand object names in trestle.
 
 ### Rules for element path
 
-1. Element path is an expression of the attribute names, [in json form](https://pages.nist.gov/OSCAL/reference/latest/complete/json-outline/) , concatenated by a period (`.`).
+1. Element path is an expression of the attribute names, [in json form](https://pages.nist.gov/OSCAL-Reference/models/latest/complete/json-outline/) , concatenated by a period (`.`).
    1. E.g. The metadata in a catalog is referred to as `catalog.metadata`
 1. Element paths are relative to the file.
    1. e.g. For `metadata.json` roles would be referred to as `metadata.roles`, from the catalog file that would be `catalog.metadata.roles`