From 1dd2808e8b675b4b06fe44bef7d734deed884845 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Fri, 15 Oct 2021 12:17:22 -0700 Subject: [PATCH 001/128] Create azure-pipeline-00.yml --- code-scanning-workflows/azure-pipeline-00.yml | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 code-scanning-workflows/azure-pipeline-00.yml diff --git a/code-scanning-workflows/azure-pipeline-00.yml b/code-scanning-workflows/azure-pipeline-00.yml new file mode 100644 index 0000000..ef773c3 --- /dev/null +++ b/code-scanning-workflows/azure-pipeline-00.yml @@ -0,0 +1,54 @@ +steps: + # Download the CodeQL CLI and query packs... + # Check out the repository ... + + # Tasks prior to executing the build, e.g. restore NuGet dependencies... + + # Initialize the CodeQL database. + # In this example, the CodeQL CLI has been downloaded and placed on the PATH. + # If no language is specified, a GitHub Apps or personal access token must be passed through stdin + # to autodetect the language. + - task: CmdLine@1 + displayName: Initialize CodeQL database + inputs: + # Assumes the source code is checked out to the current working directory. + # Creates a database at `/db` + script: "codeql database init --language csharp --trace-process-level 3 --source-root . --begin-tracing db" + + # Read the generated environment variables and values, + # and set them so they are available for subsequent commands + # in the build pipeline. This is done in PowerShell in this example. + - task: PowerShell@1 + displayName: Set CodeQL environment variables + inputs: + targetType: inline + script: > + $json = Get-Content $(System.DefaultWorkingDirectory)/db/temp/tracingEnvironment/start-tracing.json | ConvertFrom-Json + $json.PSObject.Properties | ForEach-Object { + $template = "##vso[task.setvariable variable=" + $template += $_.Name + $template += "]" + $template += $_.Value + echo "$template" + } + + # Execute the pre-defined build step. Note the `msbuildArgs` variable. + - task: VSBuild@1 + inputs: + solution: '**/*.sln' + # Disable MSBuild shared compilation for C# builds. + msbuildArgs: /p:OutDir=$(Build.ArtifactStagingDirectory) /p:UseSharedCompilation=false + platform: Any CPU + configuration: Release + # Execute a clean build, in order to remove any existing build artifacts prior to the build. + clean: True + displayName: Visual Studio Build + + - task: CmdLine@2 + displayName: Finalize CodeQL database + inputs: + script: 'codeql database finalize db' + + # Other tasks go here, + # e.g. `codeql database analyze` + # and `codeql github upload-results` ... From b6c4b5a8fdd9281b7acb5721b56b0edcf07cb3ab Mon Sep 17 00:00:00 2001 From: Thibaud Lopez Schneider Date: Mon, 18 Oct 2021 13:19:19 -0700 Subject: [PATCH 002/128] Update advanced-security-reporting.md --- advanced-security-reporting.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced-security-reporting.md b/advanced-security-reporting.md index fb9eae2..b5eda1d 100644 --- a/advanced-security-reporting.md +++ b/advanced-security-reporting.md @@ -9,3 +9,5 @@ - Secret scanning - [ ] GHES 3.1+: https://github.com/cmboling/get-secret-scanning-alerts-in-org-sample/tree/ghes/base-url-included - [ ] dotcom/GHEC: https://github.com/cmboling/get-secret-scanning-alerts-in-org-sample +- Other + - [ ] https://github.com/ThibaudLopez/GHAS/blob/main/tracking.js From c3a9b37ca0cb2b09b424e247980f569a9e47ecaf Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Wed, 20 Oct 2021 14:25:28 -0700 Subject: [PATCH 003/128] Update setup-codeql-cli.md --- code-scanning-guides/setup-codeql-cli.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code-scanning-guides/setup-codeql-cli.md b/code-scanning-guides/setup-codeql-cli.md index d233c3f..232ba07 100644 --- a/code-scanning-guides/setup-codeql-cli.md +++ b/code-scanning-guides/setup-codeql-cli.md @@ -82,8 +82,8 @@ Note that this kind of depends on how you're running the builds (whether or not just make sure to have the same category value for subsequent scans, so that Code Scannning can easily figure out what the basline analysis is to compare subsequent analyses. The `--ref` and `--commit` flag combinations can be one of the following: -- `refs/pulls//merge` + HEAD commit -- `refs/heads/` + MERGE commit +- `refs/pulls//merge` + MERGE commit +- `refs/heads/` + HEAD commit - ` curl -H "Accept: application/vnd.github.v3+json" \\n -H "Authorization: token $GH_TOKEN" \\n https://api.github.com/repos///pulls/ | jq '.merge_commit_sha'` - The merge commit is a commit created to make sure PR checks are ran; this commit doesn't exist in the actual source tree/`git log`. From 09077ddea29e387c0361cb4ac7729b044625b23c Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 26 Oct 2021 11:32:27 -0700 Subject: [PATCH 004/128] Create Dockerfile.go-example --- Dockerfile.go-example | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 Dockerfile.go-example diff --git a/Dockerfile.go-example b/Dockerfile.go-example new file mode 100644 index 0000000..e38074e --- /dev/null +++ b/Dockerfile.go-example @@ -0,0 +1,33 @@ +FROM ubuntu +LABEL description="Security & Quality CodeQL Container Build for Cool Applications" +SHELL ["/bin/bash", "-c"] +ENV TZ=America/New_York + +# create directories +RUN mkdir /tools + +# setup tools +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN DEBIAN_FRONTEND="noninteractive" apt-get update && apt-get install -y golang zip wget +RUN wget -q https://github.com/github/codeql-action/releases/download/codeql-bundle-20211005/codeql-bundle-linux64.tar.gz +RUN tar xzf /codeql-bundle-linux64.tar.gz -C tools + +# copy source +COPY . /usr/src/myapp + +# set working directory +WORKDIR /usr/src/myapp + +# example repo used: https://github.com/ghas-bootcamp/ghas-bootcamp + +# codeql create +RUN /tools/codeql/codeql database create db --language=javascript, java --db-cluster --no-run-unnecessary-builds -vvvv + +# codeql analyze with default queries +RUN /tools/codeql/codeql database analyze codeql-database/go go-code-scanning.qls --format=sarif-latest --output=codeql-go-results.sarif --sarif-category=goiscool +RUN /tools/codeql/codeql database analyze db javascript-code-scanning.qls --format=sarif-latest --output=codeql-javascript-results.sarif --sarif-category=javascriptiscool + +# upload results +# remember to get the MERGE commit for a PR +RUN /tools/codeql/codeql github upload-results --github-url= --repository=oreos/miniature-invention --ref=refs/pull/1/merge --commit=778337f84a5abe2cda468c7abf6038b8a193cea2 --sarif=codeql-go-results.sarif --github-auth-stdin= +RUN /tools/codeql/codeql github upload-results --github-url= --repository=oreos/miniature-invention --ref=refs/pull/1/merge --commit=778337f84a5abe2cda468c7abf6038b8a193cea2 --sarif=codeql-javascript-results.sarif --github-auth-stdin= From 0c5c887702833a3b704900f727fe29aa4906f370 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 26 Oct 2021 11:36:13 -0700 Subject: [PATCH 005/128] Rename Dockerfile.go-example to Dockerfile.example --- Dockerfile.go-example => Dockerfile.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Dockerfile.go-example => Dockerfile.example (100%) diff --git a/Dockerfile.go-example b/Dockerfile.example similarity index 100% rename from Dockerfile.go-example rename to Dockerfile.example From 7fa4a936cdacdbd447d37ab1f76faf1d7adcdcf0 Mon Sep 17 00:00:00 2001 From: Michael Hohn Date: Tue, 26 Oct 2021 17:33:40 -0700 Subject: [PATCH 006/128] Create uniform-setup-for-cli-and-vs-code.md --- codeql/uniform-setup-for-cli-and-vs-code.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 codeql/uniform-setup-for-cli-and-vs-code.md diff --git a/codeql/uniform-setup-for-cli-and-vs-code.md b/codeql/uniform-setup-for-cli-and-vs-code.md new file mode 100644 index 0000000..dcca79c --- /dev/null +++ b/codeql/uniform-setup-for-cli-and-vs-code.md @@ -0,0 +1 @@ +An example of a setting up the codeql cli/library/vs code plugins together for version consistency can be found here: https://github.com/hohn/codeql-cli-vscode-setup From fcc66a9856358b3a6d679219cd6cd168901f674a Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 2 Nov 2021 08:28:10 -0700 Subject: [PATCH 007/128] Update run-pr-codeql-analysis.sh --- code-scanning-scripts/run-pr-codeql-analysis.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/code-scanning-scripts/run-pr-codeql-analysis.sh b/code-scanning-scripts/run-pr-codeql-analysis.sh index b0f539b..68c9bac 100644 --- a/code-scanning-scripts/run-pr-codeql-analysis.sh +++ b/code-scanning-scripts/run-pr-codeql-analysis.sh @@ -19,9 +19,6 @@ CODEQL_SARIF_CATEGORY=.github/workflows/codeql-analysis.yml:analyze/language:go # run a single language analysis for a PR -# remove db -rm -rf $CODEQL_DATABASE - # get mergit commit sha GH_MERGE_COMMIT_SHA=$(curl -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GH_TOKEN" https://api.github.com/repos/$GH_ORG/$GH_REPO/pulls/$GH_PULL_REQUEST_NUMBER | jq '.merge_commit_sha' | sed -e 's/^"//' -e 's/"$//') @@ -29,7 +26,7 @@ GH_MERGE_COMMIT_SHA=$(curl -H "Accept: application/vnd.github.v3+json" -H "Autho codeql --version # codeql database create -codeql database create $CODEQL_DATABASE --language=$CODEQL_LANGUAGE +codeql database create $CODEQL_DATABASE --language=$CODEQL_LANGUAGE --overwrite # codeql database analyze codeql database analyze $CODEQL_DATABASE $CODEQL_QUERY_SUITE --output=$CODEQL_SARIF_RESULTS --sarif-category=$CODEQL_SARIF_CATEGORY --format=sarif-latest From c841911a245030f9e06c84299f6cd2f3895de175 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 2 Nov 2021 08:28:24 -0700 Subject: [PATCH 008/128] Update run-pr-codeql-analysis.sh --- code-scanning-scripts/run-pr-codeql-analysis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code-scanning-scripts/run-pr-codeql-analysis.sh b/code-scanning-scripts/run-pr-codeql-analysis.sh index 68c9bac..67c66cd 100644 --- a/code-scanning-scripts/run-pr-codeql-analysis.sh +++ b/code-scanning-scripts/run-pr-codeql-analysis.sh @@ -19,7 +19,7 @@ CODEQL_SARIF_CATEGORY=.github/workflows/codeql-analysis.yml:analyze/language:go # run a single language analysis for a PR -# get mergit commit sha +# get merge commit sha GH_MERGE_COMMIT_SHA=$(curl -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GH_TOKEN" https://api.github.com/repos/$GH_ORG/$GH_REPO/pulls/$GH_PULL_REQUEST_NUMBER | jq '.merge_commit_sha' | sed -e 's/^"//' -e 's/"$//') # check codeql --version From fea067f5cd88ed644c251f3a5144c9eaa79dd62e Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Wed, 3 Nov 2021 09:13:03 -0700 Subject: [PATCH 009/128] Update advanced-security-reporting.md --- advanced-security-reporting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced-security-reporting.md b/advanced-security-reporting.md index b5eda1d..e08caeb 100644 --- a/advanced-security-reporting.md +++ b/advanced-security-reporting.md @@ -10,4 +10,4 @@ - [ ] GHES 3.1+: https://github.com/cmboling/get-secret-scanning-alerts-in-org-sample/tree/ghes/base-url-included - [ ] dotcom/GHEC: https://github.com/cmboling/get-secret-scanning-alerts-in-org-sample - Other - - [ ] https://github.com/ThibaudLopez/GHAS/blob/main/tracking.js + - [ ] https://github.com/ThibaudLopez/GHAS From bc12f23997d9ac09a2a4a761cf03bb34091666a4 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Wed, 3 Nov 2021 09:18:53 -0700 Subject: [PATCH 010/128] Update advanced-security-material.md --- advanced-security-material.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced-security-material.md b/advanced-security-material.md index 20a9913..f396f0a 100644 --- a/advanced-security-material.md +++ b/advanced-security-material.md @@ -46,4 +46,4 @@ https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerab - [ ] Jenkins + CodeQL CLI: https://github.com/kllund/sample-pipeline-files/blob/main/Jenkinsfile-template-with-codeql-cli-bundle #### OSS Issue Tracking -- [ ] GitHub Code Scanning + Jira: https://github.com/github/codescanning-jira-integration +- [ ] GitHub Code Scanning + Jira: https://github.com/github/ghas-jira-integration From d563a67b4ff5a1c95f87efa85226349b7d423914 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Wed, 3 Nov 2021 14:05:58 -0700 Subject: [PATCH 011/128] Create index.md --- code-scanning-scripts/index.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 code-scanning-scripts/index.md diff --git a/code-scanning-scripts/index.md b/code-scanning-scripts/index.md new file mode 100644 index 0000000..e902fe3 --- /dev/null +++ b/code-scanning-scripts/index.md @@ -0,0 +1,5 @@ +### Code scanning scripts + + +- [ ] [Code scanning bulke enable](https://github.com/mario-campos/gh-code-scanning) +- [ ] [Run CodeQL analysis on a pull request](https://github.com/advanced-security/advanced-security-material/blob/main/code-scanning-scripts/run-pr-codeql-analysis.sh) From e0c8af3850f26873d9a12b6a267051eb25a7a8dd Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Wed, 3 Nov 2021 14:06:12 -0700 Subject: [PATCH 012/128] Rename index.md to README.md --- code-scanning-scripts/{index.md => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename code-scanning-scripts/{index.md => README.md} (100%) diff --git a/code-scanning-scripts/index.md b/code-scanning-scripts/README.md similarity index 100% rename from code-scanning-scripts/index.md rename to code-scanning-scripts/README.md From 10b660f34553da4876b2ca1d4a346b2a1f216800 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 8 Nov 2021 15:51:22 +0100 Subject: [PATCH 013/128] Reusable workflow for Code Scanning A reusable workflow for Code Scanning dispatching to the right tool, based on the programming languages present in the repo. --- .../reusable_code_scanning-00.yml | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 code-scanning-workflows/reusable_code_scanning-00.yml diff --git a/code-scanning-workflows/reusable_code_scanning-00.yml b/code-scanning-workflows/reusable_code_scanning-00.yml new file mode 100644 index 0000000..3807ce7 --- /dev/null +++ b/code-scanning-workflows/reusable_code_scanning-00.yml @@ -0,0 +1,206 @@ +Sign in to GitHub-grid | Slack
+ + + + \ No newline at end of file From 0f1d40c5d403baaf48b66131f40186ecb7293fa2 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 8 Nov 2021 16:00:11 +0100 Subject: [PATCH 014/128] Update reusable_code_scanning-00.yml --- .../reusable_code_scanning-00.yml | 337 +++++++----------- 1 file changed, 131 insertions(+), 206 deletions(-) diff --git a/code-scanning-workflows/reusable_code_scanning-00.yml b/code-scanning-workflows/reusable_code_scanning-00.yml index 3807ce7..65165ec 100644 --- a/code-scanning-workflows/reusable_code_scanning-00.yml +++ b/code-scanning-workflows/reusable_code_scanning-00.yml @@ -1,206 +1,131 @@ -Sign in to GitHub-grid | Slack
- - - - \ No newline at end of file +name: "Code Analysis" + +on: + workflow_dispatch: #for testing + workflow_call: #for composition + +jobs: + detect-lang: + runs-on: ubuntu-latest + outputs: + linguist_languages: ${{ steps.linguist_languages.outputs.languages }} + codeql_languages: ${{ steps.codeql_languages.outputs.languages }} + steps: + - id: linguist_languages + run: echo "::set-output name=languages::$(gh api repos/${GITHUB_REPOSITORY}/languages)" + - id: codeql_languages + # builds the list of languages which are both present in the repo and supported by CodeQL + # remove from the dictionary the languages that should not be considered + run: | + echo "::set-output name=languages::$(gh api repos/${GITHUB_REPOSITORY}/languages -q '[ + {"C":"cpp", "C++":"cpp", "C#":"csharp", "Go":"go", "Java":"java", #"JavaScript":"javascript", + "TypeScript":"javascript", "Python":"python", "Ruby":"ruby"}[keys[]]] | unique | . -[null]' )" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + codeql-analysis: + needs: [detect-lang] + # skip the analysis when the list of languages is empty + if: needs.detect-lang.outputs.codeql_languages != '[]' + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ${{ fromJSON(needs.detect-lang.outputs.codeql_languages) }} + exclude: + # eventually exclude languages + - language: ruby + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # perform the analysis + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 + + tsqllint-analysis: + name: Codacy Security Scan + needs: [detect-lang] + if: contains(needs.detect-lang.outputs.linguist_languages, '"TSQL"') + runs-on: ubuntu-latest + steps: + # Checkout the repository to the GitHub Actions runner + - name: Checkout code + uses: actions/checkout@v2 + + # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis + - name: Run Codacy Analysis CLI + uses: codacy/codacy-analysis-cli-action@1.1.0 + with: + tool: tsqllint + verbose: true + output: ${{ runner.temp }}/results.sarif + format: sarif + # Adjust severity of non-security issues + gh-code-scanning-compat: true + # Force 0 exit code to allow SARIF file generation + # This will handover control about PR rejection to the GitHub side + max-allowed-issues: 2147483647 + + - name: Rewrite Codacy SARIF urls to relative paths + run: sed -i 's#"uri":"file:///codacy/#"uriBaseId":"%SRCROOT%","uri":"#g' ${{ runner.temp }}/results.sarif + + # Upload the SARIF file generated in the previous step + - name: Upload SARIF results file + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: ${{ runner.temp }}/results.sarif + + - uses: actions/upload-artifact@v2 + with: + name: results.sarif + path: ${{ runner.temp }}/results.sarif + + # + # Runs Rubocop for Ruby + # + rubocop-analysis: + name: Rubocop Security Scan + needs: [detect-lang] + if: contains(needs.detect-lang.outputs.linguist_languages, '"Ruby"') + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Checkout repository + uses: actions/checkout@v2 + # If running on a self-hosted runner, check it meets the requirements + # listed at https://github.com/ruby/setup-ruby#using-self-hosted-runners + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.6 + - name: Install Code Scanning integration + run: gem install 'code-scanning-rubocop' + - name: Rubocop run + run: | + bash -c " + rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif + [[ $? -ne 2 ]] + " + - name: Upload Sarif output + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: rubocop.sarif From 1d9879c18021f425cc96af79eb3adde0e27fdfa8 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 8 Nov 2021 16:08:46 +0100 Subject: [PATCH 015/128] Update code-scanning-third-party-integrations.md --- code-scanning-third-party-integrations.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code-scanning-third-party-integrations.md b/code-scanning-third-party-integrations.md index d98eaaa..9743227 100644 --- a/code-scanning-third-party-integrations.md +++ b/code-scanning-third-party-integrations.md @@ -102,6 +102,10 @@ - [MobSF](https://github.com/MobSF/Mobile-Security-Framework-MobSF) +### T-SQL + +- [TSQLLint](https://github.com/tsqllint/tsqllint) via [Codacy](https://github.com/codacy/codacy-analysis-cli-action) action + ### Visualforce - [CodeScan](https://github.com/codescan-io/codescan-scanner-action) From 43b4790a6f19514e4f8f44be9e2a2698f251924c Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 9 Nov 2021 15:17:39 +0100 Subject: [PATCH 016/128] Update code-scanning-third-party-integrations.md --- code-scanning-third-party-integrations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/code-scanning-third-party-integrations.md b/code-scanning-third-party-integrations.md index 9743227..fba5c40 100644 --- a/code-scanning-third-party-integrations.md +++ b/code-scanning-third-party-integrations.md @@ -132,6 +132,7 @@ ### CloudFormation - [KICS](https://github.com/Checkmarx/kics) +- [Action for CloudFormation Linter](https://github.com/ScottBrenner/cfn-lint-action) ### Docker From 6814251e1653f2b0514e29677badcf62ce7e2e51 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 17 Nov 2021 17:54:23 +0100 Subject: [PATCH 017/128] Update reusable_code_scanning-00.yml --- code-scanning-workflows/reusable_code_scanning-00.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code-scanning-workflows/reusable_code_scanning-00.yml b/code-scanning-workflows/reusable_code_scanning-00.yml index 65165ec..ba483f5 100644 --- a/code-scanning-workflows/reusable_code_scanning-00.yml +++ b/code-scanning-workflows/reusable_code_scanning-00.yml @@ -18,7 +18,7 @@ jobs: # remove from the dictionary the languages that should not be considered run: | echo "::set-output name=languages::$(gh api repos/${GITHUB_REPOSITORY}/languages -q '[ - {"C":"cpp", "C++":"cpp", "C#":"csharp", "Go":"go", "Java":"java", #"JavaScript":"javascript", + {"C":"cpp", "C++":"cpp", "C#":"csharp", "Go":"go", "Java":"java", "JavaScript":"javascript", "TypeScript":"javascript", "Python":"python", "Ruby":"ruby"}[keys[]]] | unique | . -[null]' )" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 9a6018aa82a1c0ca542103ba12b535d8642334de Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 17 Nov 2021 19:43:23 +0100 Subject: [PATCH 018/128] Update reusable_code_scanning-00.yml --- code-scanning-workflows/reusable_code_scanning-00.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/code-scanning-workflows/reusable_code_scanning-00.yml b/code-scanning-workflows/reusable_code_scanning-00.yml index ba483f5..61420bb 100644 --- a/code-scanning-workflows/reusable_code_scanning-00.yml +++ b/code-scanning-workflows/reusable_code_scanning-00.yml @@ -44,6 +44,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + with: + fetch-depth: 0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL From ef1324f77e1183dfd5b1c4607fb2201c583be3b1 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 22 Nov 2021 12:32:18 +0100 Subject: [PATCH 019/128] Update code-scanning-third-party-integrations.md --- code-scanning-third-party-integrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code-scanning-third-party-integrations.md b/code-scanning-third-party-integrations.md index fba5c40..b67bcc1 100644 --- a/code-scanning-third-party-integrations.md +++ b/code-scanning-third-party-integrations.md @@ -52,7 +52,7 @@ ### Kotlin - [Detekt](https://github.com/detekt/detekt) (Code Quality) - - [Kotlin Example](https://github.com/octodemo/KotlinGoat/blob/master/.github/workflows/detekt-analysis-yml) + - [Kotlin Example](https://github.com/octodemo/KotlinGoat/blob/master/.github/workflows/detekt-analysis.yml) - [MobSF](https://github.com/MobSF/Mobile-Security-Framework-MobSF) ### Lightning (Aura and LWC) From d9a8e48bac38ad9e819b0d0298439636b17e9616 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 22 Nov 2021 12:38:44 +0100 Subject: [PATCH 020/128] Update code-scanning-third-party-integrations.md --- code-scanning-third-party-integrations.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/code-scanning-third-party-integrations.md b/code-scanning-third-party-integrations.md index b67bcc1..055263b 100644 --- a/code-scanning-third-party-integrations.md +++ b/code-scanning-third-party-integrations.md @@ -54,6 +54,8 @@ - [Detekt](https://github.com/detekt/detekt) (Code Quality) - [Kotlin Example](https://github.com/octodemo/KotlinGoat/blob/master/.github/workflows/detekt-analysis.yml) - [MobSF](https://github.com/MobSF/Mobile-Security-Framework-MobSF) +- [ShiftLeft](https://github.com/ShiftLeftSecurity/scan-action) + - [Kotlin Example](https://github.com/octodemo/KotlinGoat/blob/master/.github/workflows/shiftleft-analysis.yml) ### Lightning (Aura and LWC) From 838e4fabdbe108ef105ea2fe676af6f374ff94e3 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Mon, 6 Dec 2021 10:13:39 -0800 Subject: [PATCH 021/128] Create troubleshooting.md --- .../sarif-upload/troubleshooting.md | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 troubleshooting/sarif-upload/troubleshooting.md diff --git a/troubleshooting/sarif-upload/troubleshooting.md b/troubleshooting/sarif-upload/troubleshooting.md new file mode 100644 index 0000000..305777b --- /dev/null +++ b/troubleshooting/sarif-upload/troubleshooting.md @@ -0,0 +1,46 @@ +### GHES 3.2.1 + CodeQL CLI 2.7.2 + +some intentional errors: + +:gift: wrong ref: +``` +codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/ --sarif=results.sarif --ref=ref/heads/main --commit=5f538e43c27e91cd3e31c5a12b136b69d61a744f --repository=santa-foss/jubilant-octo-pancake --github-auth-stdin=ghp_somethingsomethingsomething +A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/jubilant-octo-pancake/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 422 Unprocessable Entity:::{"message":"Invalid request.\n\nref/heads/main does not match /^refs\\/(heads|pull|tags)\\/.*$/.","documentation_url":"https://docs.github.com/enterprise/3.2/rest/reference/code-scanning#upload-a-sarif-file"} +``` + +:santa: bad credentials: +``` +codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/ --sarif=results.sarif --ref=refs/heads/main --commit=5f538e43c27e91cd3e31c5a12b136b69d61a744f --repository=santa-foss/jubilant-octo-pancake --github-auth-stdin=ghp_somethingsomethingsomethin +A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/jubilant-octo-pancake/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 401 Unauthorized:::{"message":"Bad credentials","documentation_url":"https://docs.github.com/enterprise/3.2/rest"} +``` + +:gift: missing token: +``` +codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/ --sarif=results.sarif --ref=refs/heads/main --commit=5f538e43c27e91cd3e31c5a12b136b69d61a744f --repository=santa-foss/jubilant-octo-pancake +A fatal error occurred: A GitHub token is required to upload SARIF results but none was specified. +(eventual cause: MissingTokenException "An operation was attempted that requires a GitHub token but one could not be fou..." +``` + +:santa: misspelled repo name: +``` +codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/ --sarif=results.sarif --ref=refs/heads/main --commit=5f538e43c27e91cd3e31c5a12b136b69d61a744f --repository=santa-foss/jubilant-octo-pancak --github-auth-stdin=ghp_somethingsomethingsomething +A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/jubilant-octo-pancak/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 404 Not Found:::{"message":"Not Found","documentation_url":"https://docs.github.com/enterprise/3.2/rest/reference/code-scanning#upload-a-sarif-file"} +``` + +:gift: bad token (no security event scope): +``` +codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/ --sarif=results.sarif --ref=refs/heads/main --commit=5f538e43c27e91cd3e31c5a12b136b69d61a744f --repository=santa-foss/jubilant-octo-pancake --github-auth-stdin=ghp_falalalala +A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/jubilant-octo-pancake/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 403 Forbidden:::{"message":"You are not authorized to write security events.","documentation_url":"https://docs.github.com/enterprise/3.2/rest/reference/code-scanning#upload-a-sarif-file"} +``` + +:santa: GHAS not enabled but have a valid token: +``` +codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/ --sarif=results.sarif --ref=refs/heads/main --commit=5f538e43c27e91cd3e31c5a12b136b69d61a744f --repository=santa-foss/jubilant-octo-pancake --github-auth-stdin=ghp_somethingsomethingsomething +A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/jubilant-octo-pancake/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 403 Forbidden:::{"message":"Advanced Security must be enabled for this repository to use code scanning.","documentation_url":"https://docs.github.com/enterprise/3.2/rest/reference/code-scanning#upload-a-sarif-file"} +``` + +:gift: posting SARIF to the wrong repo (where GHAS isn’t enabled): +``` +codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/ --sarif=results.sarif --ref=refs/heads/main --commit=5f538e43c27e91cd3e31c5a12b136b69d61a744f --repository=santa-foss/fluffy-potato --github-auth-stdin=ghp_somethingsomethingsomething +A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/fluffy-potato/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 403 Forbidden:::{"message":"Advanced Security must be enabled for this repository to use code scanning.","documentation_url":"https://docs.github.com/enterprise/3.2/rest/reference/code-scanning#upload-a-sarif-file"} +``` From 6450e3520513572f9d84d2d7c10342c54f30139c Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Mon, 6 Dec 2021 10:14:20 -0800 Subject: [PATCH 022/128] Update troubleshooting.md --- troubleshooting/sarif-upload/troubleshooting.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/troubleshooting/sarif-upload/troubleshooting.md b/troubleshooting/sarif-upload/troubleshooting.md index 305777b..4c55a65 100644 --- a/troubleshooting/sarif-upload/troubleshooting.md +++ b/troubleshooting/sarif-upload/troubleshooting.md @@ -1,6 +1,4 @@ -### GHES 3.2.1 + CodeQL CLI 2.7.2 - -some intentional errors: +### SARIF Upload Errors :gift: wrong ref: ``` @@ -44,3 +42,6 @@ A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff1 codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/ --sarif=results.sarif --ref=refs/heads/main --commit=5f538e43c27e91cd3e31c5a12b136b69d61a744f --repository=santa-foss/fluffy-potato --github-auth-stdin=ghp_somethingsomethingsomething A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/fluffy-potato/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 403 Forbidden:::{"message":"Advanced Security must be enabled for this repository to use code scanning.","documentation_url":"https://docs.github.com/enterprise/3.2/rest/reference/code-scanning#upload-a-sarif-file"} ``` + +### Test environments +- GHES 3.2.1 + CodeQL CLI 2.7.2 From ef51187d8848b06ef0b531ce07bbff136aa91578 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 7 Dec 2021 17:34:59 -0800 Subject: [PATCH 023/128] Update setup-codeql-cli.md --- code-scanning-guides/setup-codeql-cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code-scanning-guides/setup-codeql-cli.md b/code-scanning-guides/setup-codeql-cli.md index 232ba07..53d1a00 100644 --- a/code-scanning-guides/setup-codeql-cli.md +++ b/code-scanning-guides/setup-codeql-cli.md @@ -29,7 +29,7 @@ Clone this repository and `cd` into it. The first thing we gotta do when it comes to CodeQL analysis is to create a CodeQL database. When it comes to interpreted languages and Go, CodeQL will use an autobuild.sh script that will extract the source code and create a snapshot database. When it comes to compiled languages, we require to build the source code in order to trace the build and create a snapshot database of it. -You can rely on the autobuild.sh script as well, or you can supply your own build instructions via `--comand` flag, which can be used when invoking the `codeql database create` command. +You can rely on the autobuild.sh script as well, or you can supply your own build instructions via the `--command` flag, which can be used when invoking the `codeql database create` command. Please review this [list](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/) of currently supported languages and frameworks. From f9176f78dbba7b661451df0784e448f6ab3ee827 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 7 Dec 2021 17:39:08 -0800 Subject: [PATCH 024/128] Update setup-codeql-cli.md --- code-scanning-guides/setup-codeql-cli.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/code-scanning-guides/setup-codeql-cli.md b/code-scanning-guides/setup-codeql-cli.md index 53d1a00..be7fa96 100644 --- a/code-scanning-guides/setup-codeql-cli.md +++ b/code-scanning-guides/setup-codeql-cli.md @@ -1,6 +1,6 @@ ### Getting started with the CodeQL CLI -When you want to generate a CodeQL database locally and run the pre-compiled queries against, this is the way to go. +When you want to generate a CodeQL database locally and run the pre-compiled queries against it, this is the way to go. First let's download the CodeQL bundle! Head over [here](https://github.com/github/codeql-action/releases ) and download the approprate bundle for your operating system. Once it's downloaded, untar the content to a CodeQL home folder and you can add CodeQL to your path if you'd like @@ -15,9 +15,6 @@ Check to make sure you can use the CodeQL CLI codeql --version ``` -You can see in this example how the CodeQL CLI is used in a [workflow](https://github.com/advanced-security/javascript-codeql-cli-test-workflow/blob/main/.github/workflows/codeql-analysis.yml). -Note that it always downloads the latest CodeQL bundle for Linux. In your case, choose the bundle that best fits your operating system. - Now we need to use the CodeQL CLI on an actual repository. Let's start here with our [GHAS training material](https://github.com/ghas-bootcamp/ghas-bootcamp) There's multiple languages being used here, so for the purposes of this tutorial let's try to scan the Javascript portions of the codebase. @@ -41,9 +38,9 @@ CodeQL will create the `db` directory and will choose the autobuild.sh script fo CodeQL will also finalize the database at the specified `db` directory. Within your codeql database directory (in this case `db`) you should notice a db-javascript directory which contains the db schemes and a src.zip which contains the source that was extracted. -#### Importing the CodeQL database to Visual Studios +#### Optional: Importing the CodeQL database to Visual Studios You can actually take this database and import it to your Visual Studios workspace. -To get started on that, please go to this repository and follow the instructions on how to setup the CodeQL starter workspace, as well as installing the CodeQL plugin. +To get started on that, please go to this [repository](https://github.com/github/vscode-codeql-starter) and follow the instructions on how to setup the CodeQL starter workspace, as well as installing the CodeQL plugin. Once you have the CodeQL plugin installed, import the database you created in this step and try to run a javascript query against the database. @@ -77,7 +74,7 @@ Failure to do so, in particular on a pull request, can cause confusion in that C This step is typically used when you want to see the SARIF in the Code Scanning alerts UI. It's typically used when you want to post results to the default branch of a repository for the first time (baseline analysis) or to a pull request to see any security alert annotations. Here are some advanced things to note: -- When posting the analysis for the first time to a default analysis, make sure you define a `--sarif-category`. That way for the analyses for subsequent pull requests can also share the same category value. +- When posting the analysis for the first time to a default analysis, make sure you define a `--sarif-category`. That way the analyses for subsequent pull requests can also share the same category value. Note that this kind of depends on how you're running the builds (whether or not you've broken down a monorepo into separate analyses or you have multiple scans due to multiple languaages) but typically just starting out, just make sure to have the same category value for subsequent scans, so that Code Scannning can easily figure out what the basline analysis is to compare subsequent analyses. From 6628f63f935e4f17117e9bd77cdc016c7e5c219f Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 7 Dec 2021 21:53:04 -0800 Subject: [PATCH 025/128] Rename advanced-security-reporting.md to reporting/advanced-security-reporting.md --- .../advanced-security-reporting.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename advanced-security-reporting.md => reporting/advanced-security-reporting.md (100%) diff --git a/advanced-security-reporting.md b/reporting/advanced-security-reporting.md similarity index 100% rename from advanced-security-reporting.md rename to reporting/advanced-security-reporting.md From 066076ff939f009ba831ee81089bbadecb0c7acf Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 7 Dec 2021 22:01:15 -0800 Subject: [PATCH 026/128] Create ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 reporting/ghes-sql-queries.md diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md new file mode 100644 index 0000000..912eb6a --- /dev/null +++ b/reporting/ghes-sql-queries.md @@ -0,0 +1,26 @@ +### SQL queries for Advanced Security metrics +Normally you would use the APIs to get this data, use the Security Overview page to review GHAS rollout and/or use the webhooks to sync GHAS related information to Slack or some centralised security platform. +If you're on GHES, you can get into the `ghes-console` and run some SQL queries to get pretty much the same data. + +If at all possible use the available APIs and webhooks to get this data. The method described here is reserved for users who have access to the `ghe-dbconsole`. + +The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` + +#### Latest code scanning records +```sql +select + r.id, + r.name, + c.alert_number as 'number_of_code_scanning_alerts', + c.check_run_id, c.created_at as 'code_scanning_analysis_latest_date' +from + repositories as r, + code_scanning_alerts as c +where + r.id = c.repository_id + and c.check_run_id=( + select + max(c.check_run_id) + from + code_scanning_alerts as c) +``` From c0be84f72ffd52313fde6d04b4fddd373d5ca210 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 7 Dec 2021 22:38:21 -0800 Subject: [PATCH 027/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 912eb6a..46f1bba 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -24,3 +24,16 @@ where from code_scanning_alerts as c) ``` + +#### Security overview config for code scanning +```sql +select + r.repository_id, + r.organization_id, + r.name, + r.ghas_enabled, + r.last_push, + r.risk_level +from + repository_security_center_configs as r +``` From 689779792d7de24d108c8158ce2286ce0523cc5d Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 7 Dec 2021 22:38:43 -0800 Subject: [PATCH 028/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 46f1bba..02f637b 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -25,7 +25,7 @@ where code_scanning_alerts as c) ``` -#### Security overview config for code scanning +#### Security overview configs ```sql select r.repository_id, From b68f01d21706aedfa7607636627b1ffc2cc1ed38 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Wed, 8 Dec 2021 21:21:57 -0800 Subject: [PATCH 029/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 02f637b..d43ec20 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -37,3 +37,22 @@ select from repository_security_center_configs as r ``` + +#### List contributors of GHAS enabled repositories +```sql +select + r.owner_login as org_name, + r.name as repo_name, + u.login as gh_handle +from + ghas_repository_contributions as grc, + repositories as r, users as u +where + u.id = grc.user_id + and r.id = grc.repository_id; +``` + + +#### Helper tables +- github_enterprise.repositories +- github_enterprise.users From f7d76f79cc6f8630407bc6bdd6e58e47b5845189 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Wed, 8 Dec 2021 21:24:14 -0800 Subject: [PATCH 030/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index d43ec20..3ba968a 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -22,7 +22,7 @@ where select max(c.check_run_id) from - code_scanning_alerts as c) + code_scanning_alerts as c); ``` #### Security overview configs @@ -35,7 +35,7 @@ select r.last_push, r.risk_level from - repository_security_center_configs as r + repository_security_center_configs as r; ``` #### List contributors of GHAS enabled repositories From 10c03daee8f6ca7b3d0b7a7a65c2fa8580473f2a Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 00:03:37 -0800 Subject: [PATCH 031/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 89 ++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 3ba968a..61ef01f 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -4,7 +4,7 @@ If you're on GHES, you can get into the `ghes-console` and run some SQL queries If at all possible use the available APIs and webhooks to get this data. The method described here is reserved for users who have access to the `ghe-dbconsole`. -The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` +The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` OR run `ghe-dbconsole -y` and a `mysql` prompt will come up for you to run some queries. #### Latest code scanning records ```sql @@ -52,6 +52,93 @@ where and r.id = grc.repository_id; ``` +#### Lookup a particular dependency vulnerability +```sql +select + id, + status, + ghsa_id, + cve_id, + created_at +from + vulnerabilities +where + id = 4765; +``` + +``` ++------+-----------+---------------------+----------------+---------------------+ +| id | status | ghsa_id | cve_id | created_at | ++------+-----------+---------------------+----------------+---------------------+ +| 4765 | published | GHSA-68w8-qjq3-2gfm | CVE-2021-33203 | 2021-06-10 14:57:28 | ++------+-----------+---------------------+----------------+---------------------+ +``` + +#### Show dependabot alerts across enterprise + +```sql +select + rva.id, + r.owner_login as org_name, + r.name as repo_name, + rva.vulnerable_manifest_path, + rva.vulnerable_requirements, + v.cve_id, + v.ghsa_id, + v.status as cve_status, + rva.created_at, + rva.updated_at, + rva.last_detected_at, + rva.dismisser_id, + rva.dismiss_reason, + rva.dismissed_at +from + repository_vulnerability_alerts as rva, + repositories as r, + vulnerabilities as v +where + r.id = rva.repository_id + and v.id = rva.vulnerability_id; +``` + +``` ++----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+-----------------------------------------------------------------------------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ +| id | org_name | repo_name | vulnerable_manifest_path | vulnerable_requirements | cve_id | ghsa_id | external_reference | cve_status | created_at | updated_at | last_detected_at | dismisser_id | dismiss_reason | dismissed_at | ++----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+-----------------------------------------------------------------------------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ +| 1 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-33203 | GHSA-68w8-qjq3-2gfm | https://nvd.nist.gov/vuln/detail/CVE-2021-33203 | published | 2021-12-02 01:17:36 | 2021-12-02 01:17:36 | NULL | NULL | NULL | NULL | +| 2 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-33571 | GHSA-p99v-5w3c-jqq9 | https://nvd.nist.gov/vuln/detail/CVE-2021-33571 | published | 2021-12-02 01:17:37 | 2021-12-02 01:17:37 | NULL | NULL | NULL | NULL | +| 3 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-31542 | GHSA-rxjp-mfm9-w4wr | https://nvd.nist.gov/vuln/detail/CVE-2021-31542 | published | 2021-12-02 01:17:48 | 2021-12-02 01:17:48 | NULL | NULL | NULL | NULL | +| 4 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-32052 | GHSA-qm57-vhq3-3fwf | https://nvd.nist.gov/vuln/detail/CVE-2021-32052 | published | 2021-12-02 01:17:49 | 2021-12-02 01:17:49 | NULL | NULL | NULL | NULL | +| 5 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-28658 | GHSA-xgxc-v2qg-chmh | https://nvd.nist.gov/vuln/detail/CVE-2021-28658 | published | 2021-12-02 01:18:53 | 2021-12-02 01:18:53 | NULL | NULL | NULL | NULL | +| 6 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-3281 | GHSA-fvgf-6h6h-3322 | https://nvd.nist.gov/vuln/detail/CVE-2021-3281 | published | 2021-12-02 01:19:08 | 2021-12-02 01:19:08 | NULL | NULL | NULL | NULL | +| 7 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2020-24584 | GHSA-fr28-569j-53c4 | https://nvd.nist.gov/vuln/detail/CVE-2020-24584 | published | 2021-12-02 01:19:09 | 2021-12-02 01:19:09 | NULL | NULL | NULL | NULL | +| 8 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2020-24583 | GHSA-m6gj-h9gm-gw44 | https://nvd.nist.gov/vuln/detail/CVE-2020-24583 | published | 2021-12-02 01:19:09 | 2021-12-02 01:19:09 | NULL | NULL | NULL | NULL | +| 9 | elves | urban-computing-machine | requirements.txt | = 1.11 | CVE-2020-15105 | GHSA-vhr6-pvjm-9qwf | https://github.com/Bouke/django-two-factor-auth/security/advisories/GHSA-vhr6-pvjm-9qwf | published | 2021-12-02 01:20:10 | 2021-12-02 01:20:10 | NULL | NULL | NULL | NULL | +| 12 | elves | urban-computing-machine | requirements.txt | = 0.2.0 | CVE-2011-4103 | GHSA-pvhp-v9qp-xf5r | https://nvd.nist.gov/vuln/detail/CVE-2011-4103 | published | 2021-12-02 23:17:53 | 2021-12-02 23:17:53 | NULL | NULL | NULL | NULL | ++----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+-----------------------------------------------------------------------------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ +``` + +#### List secret scanning alerts across the enterprise +```sql +select + r.owner_login as org_name, + r.name as repo_name, + tsr.* +from + token_scan_results as tsr, + repositories as r +where + r.id = tsr.repository_id; +``` + +``` ++----------+-------------------------+----+---------------+---------------------+---------------------+------------+------------------------------------------------------------------+------------+-------------+-------------+--------+------------+ +| org_name | repo_name | id | repository_id | created_at | updated_at | token_type | token_signature | resolution | resolver_id | resolved_at | number | scan_scope | ++----------+-------------------------+----+---------------+---------------------+---------------------+------------+------------------------------------------------------------------+------------+-------------+-------------+--------+------------+ +| elves | urban-computing-machine | 1 | 2 | 2021-12-02 01:10:09 | 2021-12-02 01:10:09 | cp_1 | blah | NULL | NULL | NULL | 1 | 2 | ++----------+-------------------------+----+---------------+---------------------+---------------------+------------+------------------------------------------------------------------+------------+-------------+-------------+--------+------------+ +``` + #### Helper tables - github_enterprise.repositories From 8c84ae1ad04a5e4e2dd82f3975f958f900e0f929 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 11:36:19 -0800 Subject: [PATCH 032/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 61ef01f..4c4c9c9 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -2,7 +2,7 @@ Normally you would use the APIs to get this data, use the Security Overview page to review GHAS rollout and/or use the webhooks to sync GHAS related information to Slack or some centralised security platform. If you're on GHES, you can get into the `ghes-console` and run some SQL queries to get pretty much the same data. -If at all possible use the available APIs and webhooks to get this data. The method described here is reserved for users who have access to the `ghe-dbconsole`. +If at all possible use the available APIs and webhooks to get this data. The method described here is reserved for users who have access to the `ghe-dbconsole` and have the intention of reading/getting information about Advanced Security rollout. The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` OR run `ghe-dbconsole -y` and a `mysql` prompt will come up for you to run some queries. From c04a6d25f2504b42568fc4bdf191892f40667b00 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 12:17:35 -0800 Subject: [PATCH 033/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 43 ++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 4c4c9c9..65d2796 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -6,7 +6,7 @@ If at all possible use the available APIs and webhooks to get this data. The met The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` OR run `ghe-dbconsole -y` and a `mysql` prompt will come up for you to run some queries. -#### Latest code scanning records +#### Latest code scanning records across enterprise ```sql select r.id, @@ -25,6 +25,33 @@ where code_scanning_alerts as c); ``` +#### Code scanning records across enterprise +```sql +select * from code_scanning_alerts; +``` +``` ++----+---------------------+---------------------+---------------------+--------------+---------------+--------------+ +| id | check_annotation_id | created_at | updated_at | alert_number | repository_id | check_run_id | ++----+---------------------+---------------------+---------------------+--------------+---------------+--------------+ +| 1 | 1 | 2021-12-06 19:07:52 | 2021-12-06 19:07:52 | 12 | 3 | 1 | +| 2 | 2 | 2021-12-06 19:23:01 | 2021-12-06 19:23:01 | 12 | 3 | 2 | ++----+---------------------+---------------------+---------------------+--------------+---------------+--------------+ +``` + +#### Lookup code scanning check suites +```sql +select * from code_scanning_check_suites; +``` + +``` ++----+----------------+---------------+----------------------------+----------------------------+-----------------+------------------------------------------+------------------+------------------------------------------+ +| id | check_suite_id | repository_id | created_at | updated_at | base_ref | base_sha | pull_request_ref | pull_request_sha | ++----+----------------+---------------+----------------------------+----------------------------+-----------------+------------------------------------------+------------------+------------------------------------------+ +| 1 | 1 | 3 | 2021-12-06 19:07:49.725371 | 2021-12-06 19:07:49.725371 | refs/heads/main | ce00909d5df0dbc22e8106de517d2e4aec7f5304 | refs/pull/1 | 8efd25c32a9a27abd498bfd19c5b33775c1cd26a | +| 2 | 2 | 3 | 2021-12-06 19:22:59.875356 | 2021-12-06 19:22:59.875356 | refs/heads/main | ce00909d5df0dbc22e8106de517d2e4aec7f5304 | refs/pull/1 | cad3e08473f4427e0e68a3e0f01428fb4d25460d | ++----+----------------+---------------+----------------------------+----------------------------+-----------------+------------------------------------------+------------------+------------------------------------------+``` +``` + #### Security overview configs ```sql select @@ -41,9 +68,9 @@ from #### List contributors of GHAS enabled repositories ```sql select + distinct u.login as gh_handle, r.owner_login as org_name, - r.name as repo_name, - u.login as gh_handle + r.name as repo_name from ghas_repository_contributions as grc, repositories as r, users as u @@ -52,6 +79,16 @@ where and r.id = grc.repository_id; ``` +``` ++-----------+------------+-------------------------+ +| gh_handle | org_name | repo_name | ++-----------+------------+-------------------------+ +| ghe-admin | elves | urban-computing-machine | +| ghe-admin | santa-foss | jubilant-octo-pancake | +| ghe-admin | santa-foss | fluffy-potato | ++-----------+------------+-------------------------+ +``` + #### Lookup a particular dependency vulnerability ```sql select From f9184a0a3bdb9f69a4a7a7a4b25341364315d9a2 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 12:31:23 -0800 Subject: [PATCH 034/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 65d2796..e842b8e 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -65,6 +65,19 @@ from repository_security_center_configs as r; ``` +``` ++---------------+-----------------+-------------------------+--------------+----------------------------+------------+ +| repository_id | organization_id | name | ghas_enabled | last_push | risk_level | ++---------------+-----------------+-------------------------+--------------+----------------------------+------------+ +| 1 | 5 | laughing-sniffle | 0 | 2021-12-02 01:06:41.000000 | 5 | +| 2 | 5 | urban-computing-machine | 1 | 2021-12-08 06:04:26.000000 | 30 | +| 3 | 4 | jubilant-octo-pancake | 1 | 2021-12-07 06:33:28.000000 | 10 | +| 4 | 4 | fluffy-potato | 1 | 2021-12-02 17:42:44.000000 | 5 | +| 5 | 6 | chanukah | 0 | 2021-12-08 06:22:50.000000 | 5 | +| 6 | 4 | sturdy-adventure | 1 | 2021-12-08 17:57:13.000000 | 0 | ++---------------+-----------------+-------------------------+--------------+----------------------------+------------+ +``` + #### List contributors of GHAS enabled repositories ```sql select From 9bd3350d1ce804561b14827d8f401fceffe0824d Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 12:37:32 -0800 Subject: [PATCH 035/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index e842b8e..028423f 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -125,7 +125,7 @@ where ``` #### Show dependabot alerts across enterprise - +This query shows the CVEs that were fetched for known Dependabot alerts in a repository. ```sql select rva.id, @@ -152,20 +152,21 @@ where ``` ``` -+----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+-----------------------------------------------------------------------------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ -| id | org_name | repo_name | vulnerable_manifest_path | vulnerable_requirements | cve_id | ghsa_id | external_reference | cve_status | created_at | updated_at | last_detected_at | dismisser_id | dismiss_reason | dismissed_at | -+----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+-----------------------------------------------------------------------------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ -| 1 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-33203 | GHSA-68w8-qjq3-2gfm | https://nvd.nist.gov/vuln/detail/CVE-2021-33203 | published | 2021-12-02 01:17:36 | 2021-12-02 01:17:36 | NULL | NULL | NULL | NULL | -| 2 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-33571 | GHSA-p99v-5w3c-jqq9 | https://nvd.nist.gov/vuln/detail/CVE-2021-33571 | published | 2021-12-02 01:17:37 | 2021-12-02 01:17:37 | NULL | NULL | NULL | NULL | -| 3 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-31542 | GHSA-rxjp-mfm9-w4wr | https://nvd.nist.gov/vuln/detail/CVE-2021-31542 | published | 2021-12-02 01:17:48 | 2021-12-02 01:17:48 | NULL | NULL | NULL | NULL | -| 4 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-32052 | GHSA-qm57-vhq3-3fwf | https://nvd.nist.gov/vuln/detail/CVE-2021-32052 | published | 2021-12-02 01:17:49 | 2021-12-02 01:17:49 | NULL | NULL | NULL | NULL | -| 5 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-28658 | GHSA-xgxc-v2qg-chmh | https://nvd.nist.gov/vuln/detail/CVE-2021-28658 | published | 2021-12-02 01:18:53 | 2021-12-02 01:18:53 | NULL | NULL | NULL | NULL | -| 6 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-3281 | GHSA-fvgf-6h6h-3322 | https://nvd.nist.gov/vuln/detail/CVE-2021-3281 | published | 2021-12-02 01:19:08 | 2021-12-02 01:19:08 | NULL | NULL | NULL | NULL | -| 7 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2020-24584 | GHSA-fr28-569j-53c4 | https://nvd.nist.gov/vuln/detail/CVE-2020-24584 | published | 2021-12-02 01:19:09 | 2021-12-02 01:19:09 | NULL | NULL | NULL | NULL | -| 8 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2020-24583 | GHSA-m6gj-h9gm-gw44 | https://nvd.nist.gov/vuln/detail/CVE-2020-24583 | published | 2021-12-02 01:19:09 | 2021-12-02 01:19:09 | NULL | NULL | NULL | NULL | -| 9 | elves | urban-computing-machine | requirements.txt | = 1.11 | CVE-2020-15105 | GHSA-vhr6-pvjm-9qwf | https://github.com/Bouke/django-two-factor-auth/security/advisories/GHSA-vhr6-pvjm-9qwf | published | 2021-12-02 01:20:10 | 2021-12-02 01:20:10 | NULL | NULL | NULL | NULL | -| 12 | elves | urban-computing-machine | requirements.txt | = 0.2.0 | CVE-2011-4103 | GHSA-pvhp-v9qp-xf5r | https://nvd.nist.gov/vuln/detail/CVE-2011-4103 | published | 2021-12-02 23:17:53 | 2021-12-02 23:17:53 | NULL | NULL | NULL | NULL | -+----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+-----------------------------------------------------------------------------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ ++----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ +| id | org_name | repo_name | vulnerable_manifest_path | vulnerable_requirements | cve_id | ghsa_id | cve_status | created_at | updated_at | last_detected_at | dismisser_id | dismiss_reason | dismissed_at | ++----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ +| 1 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-33203 | GHSA-68w8-qjq3-2gfm | published | 2021-12-02 01:17:36 | 2021-12-02 01:17:36 | NULL | NULL | NULL | NULL | +| 2 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-33571 | GHSA-p99v-5w3c-jqq9 | published | 2021-12-02 01:17:37 | 2021-12-02 01:17:37 | NULL | NULL | NULL | NULL | +| 3 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-31542 | GHSA-rxjp-mfm9-w4wr | published | 2021-12-02 01:17:48 | 2021-12-02 01:17:48 | NULL | NULL | NULL | NULL | +| 4 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-32052 | GHSA-qm57-vhq3-3fwf | published | 2021-12-02 01:17:49 | 2021-12-02 01:17:49 | NULL | NULL | NULL | NULL | +| 5 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-28658 | GHSA-xgxc-v2qg-chmh | published | 2021-12-02 01:18:53 | 2021-12-02 01:18:53 | NULL | NULL | NULL | NULL | +| 6 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-3281 | GHSA-fvgf-6h6h-3322 | published | 2021-12-02 01:19:08 | 2021-12-02 01:19:08 | NULL | NULL | NULL | NULL | +| 7 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2020-24584 | GHSA-fr28-569j-53c4 | published | 2021-12-02 01:19:09 | 2021-12-02 01:19:09 | NULL | NULL | NULL | NULL | +| 8 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2020-24583 | GHSA-m6gj-h9gm-gw44 | published | 2021-12-02 01:19:09 | 2021-12-02 01:19:09 | NULL | NULL | NULL | NULL | +| 9 | elves | urban-computing-machine | requirements.txt | = 1.11 | CVE-2020-15105 | GHSA-vhr6-pvjm-9qwf | published | 2021-12-02 01:20:10 | 2021-12-02 01:20:10 | NULL | NULL | NULL | NULL | +| 12 | elves | urban-computing-machine | requirements.txt | = 0.2.0 | CVE-2011-4103 | GHSA-pvhp-v9qp-xf5r | published | 2021-12-02 23:17:53 | 2021-12-02 23:17:53 | NULL | NULL | NULL | NULL | +| 13 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-44420 | GHSA-v6rh-hp5x-86rv | published | 2021-12-09 19:17:47 | 2021-12-09 19:17:47 | NULL | NULL | NULL | NULL | ++----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ ``` #### List secret scanning alerts across the enterprise From cb33713f13f55a28a8571ada610948ef0c8524da Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 13:02:36 -0800 Subject: [PATCH 036/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 028423f..4f13185 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -6,6 +6,10 @@ If at all possible use the available APIs and webhooks to get this data. The met The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` OR run `ghe-dbconsole -y` and a `mysql` prompt will come up for you to run some queries. +If you want to port forward to a SQL client or to a third-party dashboard, here's some things you need: +- The password when running `ghe-config secrets.mysql` +- Then port forward via `ssh -L 127.0.0.1:3307:127.0.0.1:3307 -p122 ssh://user@ghes-url + #### Latest code scanning records across enterprise ```sql select From c11093bddb1414ec7e21d01867221c1c5f256c2f Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 13:26:24 -0800 Subject: [PATCH 037/128] Update ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md index 4f13185..902be12 100644 --- a/reporting/ghes-sql-queries.md +++ b/reporting/ghes-sql-queries.md @@ -8,7 +8,7 @@ The syntax to run this on the appliance is `echo ';' | ghe-dbconsole If you want to port forward to a SQL client or to a third-party dashboard, here's some things you need: - The password when running `ghe-config secrets.mysql` -- Then port forward via `ssh -L 127.0.0.1:3307:127.0.0.1:3307 -p122 ssh://user@ghes-url +- Then port forward via `ssh -L 127.0.0.1:3307:127.0.0.1:3307 -p122 ssh://user@ghes-url` #### Latest code scanning records across enterprise ```sql From ae22c222b3747e5943225e18951a6ef869b0b55b Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 13:53:53 -0800 Subject: [PATCH 038/128] Delete ghes-sql-queries.md --- reporting/ghes-sql-queries.md | 200 ---------------------------------- 1 file changed, 200 deletions(-) delete mode 100644 reporting/ghes-sql-queries.md diff --git a/reporting/ghes-sql-queries.md b/reporting/ghes-sql-queries.md deleted file mode 100644 index 902be12..0000000 --- a/reporting/ghes-sql-queries.md +++ /dev/null @@ -1,200 +0,0 @@ -### SQL queries for Advanced Security metrics -Normally you would use the APIs to get this data, use the Security Overview page to review GHAS rollout and/or use the webhooks to sync GHAS related information to Slack or some centralised security platform. -If you're on GHES, you can get into the `ghes-console` and run some SQL queries to get pretty much the same data. - -If at all possible use the available APIs and webhooks to get this data. The method described here is reserved for users who have access to the `ghe-dbconsole` and have the intention of reading/getting information about Advanced Security rollout. - -The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` OR run `ghe-dbconsole -y` and a `mysql` prompt will come up for you to run some queries. - -If you want to port forward to a SQL client or to a third-party dashboard, here's some things you need: -- The password when running `ghe-config secrets.mysql` -- Then port forward via `ssh -L 127.0.0.1:3307:127.0.0.1:3307 -p122 ssh://user@ghes-url` - -#### Latest code scanning records across enterprise -```sql -select - r.id, - r.name, - c.alert_number as 'number_of_code_scanning_alerts', - c.check_run_id, c.created_at as 'code_scanning_analysis_latest_date' -from - repositories as r, - code_scanning_alerts as c -where - r.id = c.repository_id - and c.check_run_id=( - select - max(c.check_run_id) - from - code_scanning_alerts as c); -``` - -#### Code scanning records across enterprise -```sql -select * from code_scanning_alerts; -``` -``` -+----+---------------------+---------------------+---------------------+--------------+---------------+--------------+ -| id | check_annotation_id | created_at | updated_at | alert_number | repository_id | check_run_id | -+----+---------------------+---------------------+---------------------+--------------+---------------+--------------+ -| 1 | 1 | 2021-12-06 19:07:52 | 2021-12-06 19:07:52 | 12 | 3 | 1 | -| 2 | 2 | 2021-12-06 19:23:01 | 2021-12-06 19:23:01 | 12 | 3 | 2 | -+----+---------------------+---------------------+---------------------+--------------+---------------+--------------+ -``` - -#### Lookup code scanning check suites -```sql -select * from code_scanning_check_suites; -``` - -``` -+----+----------------+---------------+----------------------------+----------------------------+-----------------+------------------------------------------+------------------+------------------------------------------+ -| id | check_suite_id | repository_id | created_at | updated_at | base_ref | base_sha | pull_request_ref | pull_request_sha | -+----+----------------+---------------+----------------------------+----------------------------+-----------------+------------------------------------------+------------------+------------------------------------------+ -| 1 | 1 | 3 | 2021-12-06 19:07:49.725371 | 2021-12-06 19:07:49.725371 | refs/heads/main | ce00909d5df0dbc22e8106de517d2e4aec7f5304 | refs/pull/1 | 8efd25c32a9a27abd498bfd19c5b33775c1cd26a | -| 2 | 2 | 3 | 2021-12-06 19:22:59.875356 | 2021-12-06 19:22:59.875356 | refs/heads/main | ce00909d5df0dbc22e8106de517d2e4aec7f5304 | refs/pull/1 | cad3e08473f4427e0e68a3e0f01428fb4d25460d | -+----+----------------+---------------+----------------------------+----------------------------+-----------------+------------------------------------------+------------------+------------------------------------------+``` -``` - -#### Security overview configs -```sql -select - r.repository_id, - r.organization_id, - r.name, - r.ghas_enabled, - r.last_push, - r.risk_level -from - repository_security_center_configs as r; -``` - -``` -+---------------+-----------------+-------------------------+--------------+----------------------------+------------+ -| repository_id | organization_id | name | ghas_enabled | last_push | risk_level | -+---------------+-----------------+-------------------------+--------------+----------------------------+------------+ -| 1 | 5 | laughing-sniffle | 0 | 2021-12-02 01:06:41.000000 | 5 | -| 2 | 5 | urban-computing-machine | 1 | 2021-12-08 06:04:26.000000 | 30 | -| 3 | 4 | jubilant-octo-pancake | 1 | 2021-12-07 06:33:28.000000 | 10 | -| 4 | 4 | fluffy-potato | 1 | 2021-12-02 17:42:44.000000 | 5 | -| 5 | 6 | chanukah | 0 | 2021-12-08 06:22:50.000000 | 5 | -| 6 | 4 | sturdy-adventure | 1 | 2021-12-08 17:57:13.000000 | 0 | -+---------------+-----------------+-------------------------+--------------+----------------------------+------------+ -``` - -#### List contributors of GHAS enabled repositories -```sql -select - distinct u.login as gh_handle, - r.owner_login as org_name, - r.name as repo_name -from - ghas_repository_contributions as grc, - repositories as r, users as u -where - u.id = grc.user_id - and r.id = grc.repository_id; -``` - -``` -+-----------+------------+-------------------------+ -| gh_handle | org_name | repo_name | -+-----------+------------+-------------------------+ -| ghe-admin | elves | urban-computing-machine | -| ghe-admin | santa-foss | jubilant-octo-pancake | -| ghe-admin | santa-foss | fluffy-potato | -+-----------+------------+-------------------------+ -``` - -#### Lookup a particular dependency vulnerability -```sql -select - id, - status, - ghsa_id, - cve_id, - created_at -from - vulnerabilities -where - id = 4765; -``` - -``` -+------+-----------+---------------------+----------------+---------------------+ -| id | status | ghsa_id | cve_id | created_at | -+------+-----------+---------------------+----------------+---------------------+ -| 4765 | published | GHSA-68w8-qjq3-2gfm | CVE-2021-33203 | 2021-06-10 14:57:28 | -+------+-----------+---------------------+----------------+---------------------+ -``` - -#### Show dependabot alerts across enterprise -This query shows the CVEs that were fetched for known Dependabot alerts in a repository. -```sql -select - rva.id, - r.owner_login as org_name, - r.name as repo_name, - rva.vulnerable_manifest_path, - rva.vulnerable_requirements, - v.cve_id, - v.ghsa_id, - v.status as cve_status, - rva.created_at, - rva.updated_at, - rva.last_detected_at, - rva.dismisser_id, - rva.dismiss_reason, - rva.dismissed_at -from - repository_vulnerability_alerts as rva, - repositories as r, - vulnerabilities as v -where - r.id = rva.repository_id - and v.id = rva.vulnerability_id; -``` - -``` -+----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ -| id | org_name | repo_name | vulnerable_manifest_path | vulnerable_requirements | cve_id | ghsa_id | cve_status | created_at | updated_at | last_detected_at | dismisser_id | dismiss_reason | dismissed_at | -+----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ -| 1 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-33203 | GHSA-68w8-qjq3-2gfm | published | 2021-12-02 01:17:36 | 2021-12-02 01:17:36 | NULL | NULL | NULL | NULL | -| 2 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-33571 | GHSA-p99v-5w3c-jqq9 | published | 2021-12-02 01:17:37 | 2021-12-02 01:17:37 | NULL | NULL | NULL | NULL | -| 3 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-31542 | GHSA-rxjp-mfm9-w4wr | published | 2021-12-02 01:17:48 | 2021-12-02 01:17:48 | NULL | NULL | NULL | NULL | -| 4 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-32052 | GHSA-qm57-vhq3-3fwf | published | 2021-12-02 01:17:49 | 2021-12-02 01:17:49 | NULL | NULL | NULL | NULL | -| 5 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-28658 | GHSA-xgxc-v2qg-chmh | published | 2021-12-02 01:18:53 | 2021-12-02 01:18:53 | NULL | NULL | NULL | NULL | -| 6 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-3281 | GHSA-fvgf-6h6h-3322 | published | 2021-12-02 01:19:08 | 2021-12-02 01:19:08 | NULL | NULL | NULL | NULL | -| 7 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2020-24584 | GHSA-fr28-569j-53c4 | published | 2021-12-02 01:19:09 | 2021-12-02 01:19:09 | NULL | NULL | NULL | NULL | -| 8 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2020-24583 | GHSA-m6gj-h9gm-gw44 | published | 2021-12-02 01:19:09 | 2021-12-02 01:19:09 | NULL | NULL | NULL | NULL | -| 9 | elves | urban-computing-machine | requirements.txt | = 1.11 | CVE-2020-15105 | GHSA-vhr6-pvjm-9qwf | published | 2021-12-02 01:20:10 | 2021-12-02 01:20:10 | NULL | NULL | NULL | NULL | -| 12 | elves | urban-computing-machine | requirements.txt | = 0.2.0 | CVE-2011-4103 | GHSA-pvhp-v9qp-xf5r | published | 2021-12-02 23:17:53 | 2021-12-02 23:17:53 | NULL | NULL | NULL | NULL | -| 13 | elves | urban-computing-machine | requirements.txt | = 2.2.13 | CVE-2021-44420 | GHSA-v6rh-hp5x-86rv | published | 2021-12-09 19:17:47 | 2021-12-09 19:17:47 | NULL | NULL | NULL | NULL | -+----+----------+-------------------------+--------------------------+-------------------------+----------------+---------------------+------------+---------------------+---------------------+------------------+--------------+----------------+--------------+ -``` - -#### List secret scanning alerts across the enterprise -```sql -select - r.owner_login as org_name, - r.name as repo_name, - tsr.* -from - token_scan_results as tsr, - repositories as r -where - r.id = tsr.repository_id; -``` - -``` -+----------+-------------------------+----+---------------+---------------------+---------------------+------------+------------------------------------------------------------------+------------+-------------+-------------+--------+------------+ -| org_name | repo_name | id | repository_id | created_at | updated_at | token_type | token_signature | resolution | resolver_id | resolved_at | number | scan_scope | -+----------+-------------------------+----+---------------+---------------------+---------------------+------------+------------------------------------------------------------------+------------+-------------+-------------+--------+------------+ -| elves | urban-computing-machine | 1 | 2 | 2021-12-02 01:10:09 | 2021-12-02 01:10:09 | cp_1 | blah | NULL | NULL | NULL | 1 | 2 | -+----------+-------------------------+----+---------------+---------------------+---------------------+------------+------------------------------------------------------------------+------------+-------------+-------------+--------+------------+ -``` - - -#### Helper tables -- github_enterprise.repositories -- github_enterprise.users From 51db9696c02817e89458fa190aa0ed6facfef340 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 13:55:21 -0800 Subject: [PATCH 039/128] Create ghes-mysql-connect.md --- reporting/ghes-mysql-connect.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 reporting/ghes-mysql-connect.md diff --git a/reporting/ghes-mysql-connect.md b/reporting/ghes-mysql-connect.md new file mode 100644 index 0000000..04cc68e --- /dev/null +++ b/reporting/ghes-mysql-connect.md @@ -0,0 +1,11 @@ +### SQL queries for Advanced Security metrics +Normally you would use the APIs to get this data, use the Security Overview page to review GHAS rollout and/or use the webhooks to sync GHAS related information to Slack or some centralised security platform. +If you're on GHES, you can get into the `ghes-console` and run some SQL queries to get pretty much the same data. + +If at all possible use the available APIs and webhooks to get this data. The method described here is reserved for users who have access to the `ghe-dbconsole` and have the intention of reading/getting information about Advanced Security rollout. + +The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` OR run `ghe-dbconsole -y` and a `mysql` prompt will come up for you to run some queries. + +If you want to port forward to a SQL client or to a third-party dashboard, here's some things you need: +- The password when running `ghe-config secrets.mysql` +- Then port forward via `ssh -L 127.0.0.1:3307:127.0.0.1:3307 -p122 ssh://user@ghes-url` From 9f887f93a53977cc7981d209462290e85b29ad5f Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 9 Dec 2021 14:06:11 -0800 Subject: [PATCH 040/128] Update ghes-mysql-connect.md --- reporting/ghes-mysql-connect.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/reporting/ghes-mysql-connect.md b/reporting/ghes-mysql-connect.md index 04cc68e..308adb6 100644 --- a/reporting/ghes-mysql-connect.md +++ b/reporting/ghes-mysql-connect.md @@ -5,7 +5,3 @@ If you're on GHES, you can get into the `ghes-console` and run some SQL queries If at all possible use the available APIs and webhooks to get this data. The method described here is reserved for users who have access to the `ghe-dbconsole` and have the intention of reading/getting information about Advanced Security rollout. The syntax to run this on the appliance is `echo ';' | ghe-dbconsole -y` OR run `ghe-dbconsole -y` and a `mysql` prompt will come up for you to run some queries. - -If you want to port forward to a SQL client or to a third-party dashboard, here's some things you need: -- The password when running `ghe-config secrets.mysql` -- Then port forward via `ssh -L 127.0.0.1:3307:127.0.0.1:3307 -p122 ssh://user@ghes-url` From 8a9db25affa4d4e773cf76a39b927ba485873cdd Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 13 Jan 2022 09:57:41 -0800 Subject: [PATCH 041/128] Update troubleshooting.md --- troubleshooting/sarif-upload/troubleshooting.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/troubleshooting/sarif-upload/troubleshooting.md b/troubleshooting/sarif-upload/troubleshooting.md index 4c55a65..e5a0a90 100644 --- a/troubleshooting/sarif-upload/troubleshooting.md +++ b/troubleshooting/sarif-upload/troubleshooting.md @@ -45,3 +45,8 @@ A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff1 ### Test environments - GHES 3.2.1 + CodeQL CLI 2.7.2 + + +### Tools to rewrite SARIF +- `jq` +- https://github.com/microsoft/sarif-sdk/blob/main/docs/multitool-usage.md From 42b27ba8a4bef29eae6964d62336c87dff6c2b1f Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 13 Jan 2022 10:14:50 -0800 Subject: [PATCH 042/128] Update troubleshooting.md --- troubleshooting/sarif-upload/troubleshooting.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/troubleshooting/sarif-upload/troubleshooting.md b/troubleshooting/sarif-upload/troubleshooting.md index e5a0a90..e8d5eae 100644 --- a/troubleshooting/sarif-upload/troubleshooting.md +++ b/troubleshooting/sarif-upload/troubleshooting.md @@ -49,4 +49,5 @@ A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff1 ### Tools to rewrite SARIF - `jq` -- https://github.com/microsoft/sarif-sdk/blob/main/docs/multitool-usage.md +- [Microsoft's SARIF tool](https://github.com/microsoft/sarif-sdk/blob/main/docs/multitool-usage.md) +- [Dr. House's SARIF CLI](https://github.com/hohn/sarif-cli) From 4fb74b346afee0b4121705dea9b75a95eead2e37 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Wed, 9 Feb 2022 21:11:18 -0800 Subject: [PATCH 043/128] Create get-languages.sh --- code-scanning-scripts/get-languages.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 code-scanning-scripts/get-languages.sh diff --git a/code-scanning-scripts/get-languages.sh b/code-scanning-scripts/get-languages.sh new file mode 100644 index 0000000..5190e61 --- /dev/null +++ b/code-scanning-scripts/get-languages.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +curl -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GH_TOKEN" \ +https://api.github.com/repos/advanced-security/ghas-bootcamp-dryrun/languages | \ +jq 'with_entries(select([.key] | inside(["Go", "Java", "JavaScript", "Python", "C++", "C#", "C", "TypeScript"])) | if .key == "C" then .key = "cpp" else . end | if .key == "C#" then .key = "csharp" else . end | if .key == "C++" then .key = "cpp" else . end | if .key == "Go" then .key = "go" else . end | if .key == "Java" then .key = "java" else . end | if .key == "JavaScript" then .key = "javascript" else . end | if .key == "Python" then .key = "python" else . end | if .key == "TypScript" then .key = "typescript" else . end)' | jq "keys" From 4b0350641c5cc95ddbcf4e575acad63e385f5393 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 10 Feb 2022 07:42:26 -0800 Subject: [PATCH 044/128] Update advanced-security-reporting.md --- reporting/advanced-security-reporting.md | 1 + 1 file changed, 1 insertion(+) diff --git a/reporting/advanced-security-reporting.md b/reporting/advanced-security-reporting.md index e08caeb..c6ed04c 100644 --- a/reporting/advanced-security-reporting.md +++ b/reporting/advanced-security-reporting.md @@ -11,3 +11,4 @@ - [ ] dotcom/GHEC: https://github.com/cmboling/get-secret-scanning-alerts-in-org-sample - Other - [ ] https://github.com/ThibaudLopez/GHAS +- Splunk integration From 093fe0672978a9f0efe7fab7743af876f20ac835 Mon Sep 17 00:00:00 2001 From: zerotyr Date: Thu, 17 Feb 2022 09:28:19 -0600 Subject: [PATCH 045/128] fixed broken semmle links --- advanced-security-material.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced-security-material.md b/advanced-security-material.md index f396f0a..18a4554 100644 --- a/advanced-security-material.md +++ b/advanced-security-material.md @@ -13,8 +13,8 @@ - [ ] Javascript: https://www.youtube.com/watch?v=pYzfGaLTqC0 #### CodeQL Resources: -- [ ] QL Tutorials: https://help.semmle.com/QL/learn-ql/beginner/ql-tutorials.html -- [ ] CodeQL for VS Code: https://help.semmle.com/codeql/codeql-for-vscode/procedures/about-codeql-for-vscode.html +- [ ] QL Tutorials: https://codeql.github.com/docs/writing-codeql-queries/ql-tutorials/ +- [ ] CodeQL for VS Code: https://codeql.github.com/docs/codeql-for-visual-studio-code/ - [ ] VS Code starter workspace to use with the CodeQL VS extension: https://github.com/github/vscode-codeql-starter - [ ] CodeQL CTF: https://securitylab.github.com/ctf - [ ] Read about contributing to CodeQL Queries: https://github.com/github/codeql/blob/main/CONTRIBUTING.md @@ -34,7 +34,7 @@ - [ ] Configure code scanning: https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning - [ ] Configuring builds for Compiled Languages: https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-the-codeql-workflow-for-compiled-languages - [ ] Running additional queries: https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#running-additional-queries - - [ ] Built-in Queries: https://help.semmle.com/QL/ql-built-in-queries.html + - [ ] Built-in Queries: https://github.com/github/codeql, https://github.com/github/codeql-go - For example, js query suites: https://github.com/github/codeql/tree/master/javascript/ql/src/codeql-suites - [ ] Troubleshooting code scanning workflow: https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-the-codeql-workflow From ee7cd9646a8692592d3166e2a95cad1ce7007a44 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Thu, 24 Feb 2022 18:50:24 +0100 Subject: [PATCH 046/128] Update reusable_code_scanning-00.yml --- .../reusable_code_scanning-00.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/code-scanning-workflows/reusable_code_scanning-00.yml b/code-scanning-workflows/reusable_code_scanning-00.yml index 61420bb..f2f9c03 100644 --- a/code-scanning-workflows/reusable_code_scanning-00.yml +++ b/code-scanning-workflows/reusable_code_scanning-00.yml @@ -1,25 +1,30 @@ name: "Code Analysis" +# this workflow can be stored in a centralized repo and called externally +# jobs: +# code_analysis: +# uses: [REPO]/.github/workflows/code_analysis.yml@main + on: workflow_dispatch: #for testing - workflow_call: #for composition + workflow_call: #for composition jobs: detect-lang: runs-on: ubuntu-latest outputs: linguist_languages: ${{ steps.linguist_languages.outputs.languages }} - codeql_languages: ${{ steps.codeql_languages.outputs.languages }} + codeql_languages: ${{ steps.codeql_languages.outputs.languages }} steps: - id: linguist_languages run: echo "::set-output name=languages::$(gh api repos/${GITHUB_REPOSITORY}/languages)" - id: codeql_languages # builds the list of languages which are both present in the repo and supported by CodeQL - # remove from the dictionary the languages that should not be considered + # remove from the dictionary the languages that should not be considered (e.g .-[null, "go"]) run: | echo "::set-output name=languages::$(gh api repos/${GITHUB_REPOSITORY}/languages -q '[ {"C":"cpp", "C++":"cpp", "C#":"csharp", "Go":"go", "Java":"java", "JavaScript":"javascript", - "TypeScript":"javascript", "Python":"python", "Ruby":"ruby"}[keys[]]] | unique | . -[null]' )" + "TypeScript":"javascript", "Python":"python", "Ruby":"ruby"}[keys[]]] | unique -[null]')" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -37,8 +42,8 @@ jobs: fail-fast: false matrix: language: ${{ fromJSON(needs.detect-lang.outputs.codeql_languages) }} + # eventually exclude languages exclude: - # eventually exclude languages - language: ruby steps: From 3269a74ac82c7e1ea33c0d320cbf4c794c37259a Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 1 Mar 2022 10:24:10 -0800 Subject: [PATCH 047/128] Create user-defined-patterns-considerations.md --- .../user-defined-patterns-considerations.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 secret-scanning/user-defined-patterns-considerations.md diff --git a/secret-scanning/user-defined-patterns-considerations.md b/secret-scanning/user-defined-patterns-considerations.md new file mode 100644 index 0000000..69d850d --- /dev/null +++ b/secret-scanning/user-defined-patterns-considerations.md @@ -0,0 +1,16 @@ +- overall cost of scans on push is very low. it does not depend on LOC, only the size of the files. so they have a very small memory footprint unless you are pushing 100 mbs on each push. +backfills/full history scans are more expensive, and are relative to the size of the repo, how well its maintained and how costly it is to run git commands. The load here is always one time when scanning is first enabled. +- We provide configurability on how many scans of either type can be run at a given time. It can speeded up or slowed down to control the load on the system. The specific config names have changed from 3.0. +Some key things that are different: +- backfill scans are no longer capped at 15 mins, we run them till completion. +- some of the defaults for how many backfill scans have changed (previously we used to run 10 every 10 mins, now we pick up 1 backfill job every 10 seconds as long as there isnt another active one in flight).. +- everytime a user defined pattern is created, a backfill is triggered for all the repo(s) under scope. That can add additional load ad hoc. +- You should expect to see more network traffic, particularly when backfills are run. thats because scans are no longer run on the file servers, they are run in their own jobs. For single VM, tahts mostly cross the loopback address, for cluster setups that across the job server and file server nodes. +- no concerns for a 1000 repos that was discussed for Infosys. +- If for this or any other customers we are talking about similar or higher scale but with a high number of active monorepos, i can see a raesonable load during backfills. For reference, i would consider that if you have a 1000 very large mono repos in a isntance with 10s of thousands of repos, that would require a bit more thought. +- Note that in none of the cases above incremental or scan on push is a matter of concern. +- To Control backfill traffic, customers can consider a more gradual rollout of scanning across repos in their orgs/instance. when enabled at the org level, we make an attempt to do so using some of the defaults i described, but rollout gives more control. +for user defined patterns, a gradual rollout via enablement is not an option when its created at the org or enterprise level. I would highly recommend doing extensive testing with a repo for a new user defined pattern with data before applying it broadly. One option here for new customers is to create these first, and then rollout enablement, so that backfills for these are included as part of the backfills that are done at enablement. +- In terms of resources usage + - with controlled backfills and rollout without using user defined patterns, unless the instances are at the seams and have a couple of Gbs of memory available, you will be :thubs: .. + -if the customer is also planning to use user defined patterns excessively (and i mean tens or hhundreds of patterns), you would want to have upto 10-20% more resources than the min, especially when the backfills are being run or user defined patterns are being created. From 69438fba6fe3b322f090454706311d41113638e6 Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Tue, 1 Mar 2022 16:25:16 -0800 Subject: [PATCH 048/128] Update advanced-security-reporting.md --- reporting/advanced-security-reporting.md | 1 + 1 file changed, 1 insertion(+) diff --git a/reporting/advanced-security-reporting.md b/reporting/advanced-security-reporting.md index c6ed04c..7a45f47 100644 --- a/reporting/advanced-security-reporting.md +++ b/reporting/advanced-security-reporting.md @@ -12,3 +12,4 @@ - Other - [ ] https://github.com/ThibaudLopez/GHAS - Splunk integration + - [ ] https://github.com/splunk/github_app_for_splunk From cc547272f5b2c70cf31dedc350f113399d423f8b Mon Sep 17 00:00:00 2001 From: Chelsea Boling Date: Thu, 3 Mar 2022 22:16:59 -0800 Subject: [PATCH 049/128] Create combine-n-databases.sh --- code-scanning-scripts/combine-n-databases.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 code-scanning-scripts/combine-n-databases.sh diff --git a/code-scanning-scripts/combine-n-databases.sh b/code-scanning-scripts/combine-n-databases.sh new file mode 100644 index 0000000..2e7f73c --- /dev/null +++ b/code-scanning-scripts/combine-n-databases.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# For n number of dirs, initialise each of them. In this example I have two dirs at the root: +codeql database init dbs/db-js1 --language=javascript --source-root dir1 --overwrite +codeql database init dbs/db-js2 --language=javascript --source-root dir2 --overwrite + +# After db skeletons are created, use the trace command to call out the extractor/add build instructions +codeql database trace-command dbs/db-js1 -- /Users/cmboling/Projects/codeql-home/codeql-latest/javascript/tools/autobuild.sh +codeql database trace-command dbs/db-js2 -- /Users/cmboling/Projects/codeql-home/codeql-latest/javascript/tools/autobuild.sh + +# Then import n unfinalized dbs to the target db +codeql database import dbs/db-js1 db-js2 +codeql database finalize --finalize-dataset db-js1 + +# Analyze target db as usual +codeql database analyze db-js1 javascript-code-scanning.qls --format=sarif-latest --output=codeql-javascript-results.sarif From d0db269f02c0f6f2c4f6a830b02fbfe760376e40 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 22 Apr 2022 17:49:41 +0200 Subject: [PATCH 050/128] Create README.md --- reporting/issues_csv/README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 reporting/issues_csv/README.md diff --git a/reporting/issues_csv/README.md b/reporting/issues_csv/README.md new file mode 100644 index 0000000..560311c --- /dev/null +++ b/reporting/issues_csv/README.md @@ -0,0 +1,2 @@ +A `powershell` script that fetches Code Scanning, Secret Scanning and Dependabot issues for an organization and outputs them to a CSV file using `jq`. +Includes the repository topics that might be used for filtering and grouping the issues. From 0df0a120b2f23c0e7309cb434d70c701dd0405bd Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 22 Apr 2022 17:51:35 +0200 Subject: [PATCH 051/128] Add files via upload --- reporting/issues_csv/code_scanning.jq | 6 ++++++ reporting/issues_csv/dependabot.jq | 16 +++++++++++++++ reporting/issues_csv/reporting.ps1 | 26 +++++++++++++++++++++++++ reporting/issues_csv/secret_scanning.jq | 6 ++++++ 4 files changed, 54 insertions(+) create mode 100644 reporting/issues_csv/code_scanning.jq create mode 100644 reporting/issues_csv/dependabot.jq create mode 100644 reporting/issues_csv/reporting.ps1 create mode 100644 reporting/issues_csv/secret_scanning.jq diff --git a/reporting/issues_csv/code_scanning.jq b/reporting/issues_csv/code_scanning.jq new file mode 100644 index 0000000..5f65042 --- /dev/null +++ b/reporting/issues_csv/code_scanning.jq @@ -0,0 +1,6 @@ +### the csv headers +["repo","severity","created","fixed","dismissed","dismissed reason","state","url","topics"], +(.[]| +### the json path +[.repository.name,.rule.severity,.created_at,.fixed_at,.dismissed_at,.dismissed_reason,.state,.html_url,($topics[][.repository.name]|join(" "))] +) | @csv diff --git a/reporting/issues_csv/dependabot.jq b/reporting/issues_csv/dependabot.jq new file mode 100644 index 0000000..45d7c1b --- /dev/null +++ b/reporting/issues_csv/dependabot.jq @@ -0,0 +1,16 @@ +### the csv headers +["repo","package","severity","CVSS","created","fixed","dismissed","dismissed reason","state","url","topics"], +(.[].data.repository.vulnerabilityAlerts.edges[0].node | select(.!=null)| +### the json path +[.repository.name, + .securityVulnerability.package.name, + .securityVulnerability.severity, + .securityVulnerability.advisory.cvss.score, + .createdAt, + .fixedAt, + .dismissedAt, + .dismissReason, + .state, + ("https://github.com/beazley/"+.repository.name+"/security/dependabot/"+(.number|tostring)), + ($topics[][.repository.name]|join(" "))] +) | @csv diff --git a/reporting/issues_csv/reporting.ps1 b/reporting/issues_csv/reporting.ps1 new file mode 100644 index 0000000..78db83e --- /dev/null +++ b/reporting/issues_csv/reporting.ps1 @@ -0,0 +1,26 @@ +#!/usr/bin/env pwsh +$ORG = "mbaluda-org" + +### FETCH TOPICS ### +$topics = gh api --cache 5m orgs/$ORG/repos -q 'map(select(.name)|{(.name):(.topics)})|add' | jq -s 'add' +$topics | Out-File topics_map.json -encoding utf8 + +### CODE SCANNING ALERTS ### +gh api orgs/$ORG/code-scanning/alerts --method GET --paginate | jq -rf code_scanning.jq --slurpfile topics topics_map.json > code_scanning.csv + +### SECRET SCANNING ALERTS ### +gh api orgs/$ORG/secret-scanning/alerts --method GET --paginate | jq -rf secret_scanning.jq --slurpfile topics topics_map.json > secret_scanning.csv + +### DEPENDABOT SCANNING ALERTS ### +$repos = $topics | jq 'keys[]' +$(foreach ($repo in $repos) { + gh api graphql -F group=$ORG -F repo=$repo -f query=' + query ($endCursor: String, $group: String!, $repo: String!) { + repository(owner: $group, name: $repo) { + vulnerabilityAlerts(first: 100, after: $endCursor) { + edges { node { createdAt fixedAt dismissedAt dismissReason state securityVulnerability { package { name } severity advisory { cvss { score } } } repository { name } number } } + pageInfo { hasNextPage endCursor } + } + } + }' --paginate + }) | jq -srf dependabot.jq --slurpfile topics topics_map.json > dependabot.csv diff --git a/reporting/issues_csv/secret_scanning.jq b/reporting/issues_csv/secret_scanning.jq new file mode 100644 index 0000000..fd6565f --- /dev/null +++ b/reporting/issues_csv/secret_scanning.jq @@ -0,0 +1,6 @@ +### the csv headers +["repo","type","created","fixed","resolution","state","url","topics"], +(.[] | select(.rule.severity!="severity")| +### the json path +[.repository.name,.secret_type,.created_at,.resolved_at,.resolution,.state,.html_url,($topics[][.repository.name]|join(" "))] +) | @csv From 33477078d4fe09a806a87a8d395a3e4fe4ccbe25 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 22 Apr 2022 17:57:53 +0200 Subject: [PATCH 052/128] Update README.md --- reporting/issues_csv/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reporting/issues_csv/README.md b/reporting/issues_csv/README.md index 560311c..f6cd215 100644 --- a/reporting/issues_csv/README.md +++ b/reporting/issues_csv/README.md @@ -1,2 +1,2 @@ -A `powershell` script that fetches Code Scanning, Secret Scanning and Dependabot issues for an organization and outputs them to a CSV file using `jq`. -Includes the repository topics that might be used for filtering and grouping the issues. +A `powershell` script that fetches Code Scanning, Secret Scanning and Dependabot alerts for an organization and outputs them to a CSV file using `jq`. +Includes the repository topics that might be used for filtering and grouping the alerts. From 19a78da13f8b242fed8e6e9190a44ba4ab272286 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Thu, 6 Oct 2022 20:32:29 -0400 Subject: [PATCH 053/128] Initial Docs --- codeql/compiled-languages-csharp.md | 86 +++++++++++++++++++++++++++++ codeql/compiled-languages.md | 54 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 codeql/compiled-languages-csharp.md create mode 100644 codeql/compiled-languages.md diff --git a/codeql/compiled-languages-csharp.md b/codeql/compiled-languages-csharp.md new file mode 100644 index 0000000..ac626b8 --- /dev/null +++ b/codeql/compiled-languages-csharp.md @@ -0,0 +1,86 @@ +Scanning a C# application with CodeQL + +# Speed up C# Analysis + +- Start here: [CodeQL Docs - The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow#the-build-takes-too-long) + + +## Optimization - Removing Unit Tests +CodeQL will extract and analyze any code that is passed through the compiler. Consider excluding any code you do not wish to include in a security scan to speed up and remove noise from this process. + +With .NET we can employ a few mechanisms to remove test/demo code from CodeQL scans (e.g. you would want to run your unit test in another workflow ): + * A solution filter to only build required projects + * An explicit solution file that excludes projects + * have a nice ref on this one from an OSS project Identity Server + * have a build.sh script wrapper around their solution targets + * distinct IdentityServer.CodeQL.sln solution excluding unit tests + * CodeQL yaml passes in a flag to build script to use the CodeQL solution + * Build in release mode - exclude test projects from that build configuration + + + +## Optimizations - CodeQL Queries +* Tweak your current codeql yml workflow in a few ways: + * remove security-extended queries, the default query pack with smaller set of queries will complete faster + * As of v2.10.5 - Query Suite Counts + * code-scanning (default) - 49 queries + * security-extended - 66 queries + * security-and-quality - 171 queries + +* Micro Optimizations +Consider these as a potential quick hit to resolve a specific problem + + * Review workflow log timings to identify a any query that is taking a long time you can consider excluding it In via a CodeQL-config file + * add a query-filter to exclude a specific query from analysis + + * tweak the way CodeQL allocates memory to possibly make the workflow (for example, just below the runs-on field): see sample +env: + CODEQL_ACTION_EXTRA_OPTIONS: '{"database": {"run-queries": ["--off-heap-ram=0"]}}' + + * CodeQL will (by default) pull in source code from your dependencies using CIL extraction to assist in mapping out your data flows. While this can drastically improve the precision of the results, this can also lead to a large increase in database size. You might consider disabling this feature for a quick scan but running a cron based scan with the option enabled. + ```yml + env: + CODEQL_EXTRACTOR_CSHARP_OPTION_CIL: false + ``` + This requires CodeQL CLI v2.11.0+, for the short term (October 2022) - add this to ensure you run the latest version: + ```yml + uses: github/codeql-action/init@v2 + with: + tools: latest + ``` + + + * Running low on disk using the default Actions runner? "You are running out of disk space. The runner will stop working when the machine runs out of disk space." + + Try a few of these workarounds for a potential quick fix: + + * Clean up large directories you are potentially not using on the windows runner , add this to your “CodeQL” workflow: + ```yml + - name: Clean up some disks + run: | + rd C:\Android\android-sdk + docker system prune -af + ``` + + * Specify the temp directory to store the CodeQL database - I have seen this resolve this specific problem with a Windows env (runs-on: windows-2019) + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + db-location: ‘C:\windows\temp\codeql-database’ + + * See also: Vertical Scaling section + + +## Vertical Scaling - Throw hardware at the software problem. + +Large applications can be compute/mem.ory/disk bound as the base Actions runners are small instances (2core/8GBram/14GB SSD). See the recommended hardware requirements for CodeQL based on Codebase size. + * Setup a self hosted CI action runner in your infrastructure that has some more powerful specs that can handle your large application. + * See Adding a self-hosted runner to an organization. + * Enroll for the larger runners public beta + This allows for up to a 64 core machine with 256GB RAM + +## Horizontal Scaling - Continue to decompose your solution. + +Making an investment in optimizing your build process can drastically speed up your developer experience, CI pipelines and start the process of "decomposing the monolith". Continued investment in large or legacy applications is important to keep your security posture and dependencies up to date. Review the suggestions for using domain-driven design to modernize your monolithic application. If you have already investigated removing projects from your solutions that you do not wish to expose to a security scan (test projects / demo code), then you have already begun this journey. + +A great use case would be to filter separate solutions by front end (ex: Web.sln) and back end code(ex: API.sln) that are separated by process/network boundaries. CodeQL can detect data flows through the code but once it reaches a process boundary the flow is stopped. This creates a natural separation point for both feature development teams and security scans based data flows. This would further enable a decrease in wall-clock scan time by using parallel per-solution scans using Action matrix strategy (such that each gets its own runtime and resources). It will be important to include your common framework code in each solution so that you get a successful compilation while you further analyze other ways to share code. diff --git a/codeql/compiled-languages.md b/codeql/compiled-languages.md new file mode 100644 index 0000000..3b5d159 --- /dev/null +++ b/codeql/compiled-languages.md @@ -0,0 +1,54 @@ +# Scanning a compiled language with CodeQL +* NOTE: This guide will focus on GitHub Actions but the concepts can be applied to the CodeQL CLI on other CI platforms. + +## Language Specific Guidance +* CSharp + +## How CodeQL Tracer Works + +## Autobuilder +The autobuilder action (see [docs](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#about-autobuild-for-codeql) ) + +## Build Customizations +See common build configuration and specific compiler flags: [specifying build commands](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/#specifying-build-commands) + + +## Common Problems + +### 401 due to private package server configuration + +### Out of Memory +ex: + +> 2022-06-01T20:08:13.6909315Z Exit code 137 and error was: + +>A fatal error occurred: RelationManager failed to produce already COMPUTED FlowSummaryImpl#b68d378d::Private::TConsSummaryComponentStack#fff/3@e38197wv + + +These errors typically indicate that your project is too large for CodeQL to analyse with the amount of RAM found on the default GitHub runners. You can tweak the way we allocate memory to possibly make the workflow succeed by adding the following environment variable to your CodeQL job (for example, just below the runs-on field): + + +We can tweak the way CodeQL allocates memory to possibly make the workflow succeed +```yml + env: + CODEQL_ACTION_EXTRA_OPTIONS: '{"database": {"run-queries": ["--off-heap-ram=0"]}}' +``` +alternatively we can further define limits +```yml + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + # Increase Values seen in logs: + #2022-06-01T19:37:19.0200037Z CODEQL_RAM: 119741 + #2022-06-01T19:37:19.0200307Z CODEQL_THREADS: 32 + ram: 64000 + threads: 16 +``` + +## Reviewing Results +- [ ] Log viewing +- [ ] Diagnostics Logs Mode +- [ ] [Exit Codes](https://codeql.github.com/docs/codeql-cli/exit-codes/) + +## Optimizaitons +- CodeQL Docs - [The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow) \ No newline at end of file From 8315199047a9a24a4238f0e37241d5357bb1f550 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 17 Oct 2022 10:28:56 -0400 Subject: [PATCH 054/128] CPP init --- codeql/compiled-languages-cpp.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 codeql/compiled-languages-cpp.md diff --git a/codeql/compiled-languages-cpp.md b/codeql/compiled-languages-cpp.md new file mode 100644 index 0000000..804b028 --- /dev/null +++ b/codeql/compiled-languages-cpp.md @@ -0,0 +1,5 @@ +## FAQ + +### Does CodeQL need the resulting object files from the C++ build? + +CodeQL needs to monitor the actual build. Every time we see the build invoke the C++ compiler, we also "compile" the same source code with our own "compiler" that generates what we need for the actual analysis. We don't use the object files from the regular compiler at all. \ No newline at end of file From c07d458bac239d9208d29bfe03bbee7bc40c8051 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:04:31 -0400 Subject: [PATCH 055/128] language links --- codeql/compiled-languages.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codeql/compiled-languages.md b/codeql/compiled-languages.md index 3b5d159..f59524d 100644 --- a/codeql/compiled-languages.md +++ b/codeql/compiled-languages.md @@ -2,7 +2,8 @@ * NOTE: This guide will focus on GitHub Actions but the concepts can be applied to the CodeQL CLI on other CI platforms. ## Language Specific Guidance -* CSharp +* [CSharp](.\compiled-languages-csharp.md) +* [C++](.\compiled-languages-cpp.md) ## How CodeQL Tracer Works From b002a77add03f8af2224023dfd572c5439b3b9c9 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:05:38 -0400 Subject: [PATCH 056/128] relative links --- codeql/compiled-languages.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codeql/compiled-languages.md b/codeql/compiled-languages.md index f59524d..107dfc8 100644 --- a/codeql/compiled-languages.md +++ b/codeql/compiled-languages.md @@ -2,8 +2,8 @@ * NOTE: This guide will focus on GitHub Actions but the concepts can be applied to the CodeQL CLI on other CI platforms. ## Language Specific Guidance -* [CSharp](.\compiled-languages-csharp.md) -* [C++](.\compiled-languages-cpp.md) +* [CSharp](compiled-languages-csharp.md) +* [C++](compiled-languages-cpp.md) ## How CodeQL Tracer Works From 7cd91ebeb24f30c02758cdd399e9ecff1e67c35c Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:39:46 -0400 Subject: [PATCH 057/128] interpreted autobuilder troubleshooting --- codeql/interpreted-languages.md | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 codeql/interpreted-languages.md diff --git a/codeql/interpreted-languages.md b/codeql/interpreted-languages.md new file mode 100644 index 0000000..1019ae9 --- /dev/null +++ b/codeql/interpreted-languages.md @@ -0,0 +1,62 @@ +# Scanning an interpreted language with CodeQL +* NOTE: This guide will focus on GitHub Actions but the concepts can be applied to the CodeQL CLI on other CI platforms. + +# Troubleshooting + +## [ERROR] Spawned process exited abnormally (code 1; tried to run: [/opt/hostedtoolcache/CodeQL//x64/codeql/javascript/tools/autobuild.sh]) + +This is the higher level error handler for the autobuilder (in this case javascript) + +``` +[ERROR] Spawned process exited abnormally (code 1; tried to run: [/opt/hostedtoolcache/CodeQL/0.0.0-20220401/x64/codeql/javascript/tools/autobuild.sh]) + A fatal error occurred: Exit status 1 from command: [/opt/hostedtoolcache/CodeQL/0.0.0-20220401/x64/codeql/javascript/tools/autobuild.sh] + Error: The process '/opt/hostedtoolcache/CodeQL/0.0.0-20220401/x64/codeql/codeql' failed with exit code 2 + Error: The process '/opt/hostedtoolcache/CodeQL/0.0.0-20220401/x64/codeql/codeql' failed with exit code 2 + at toolrunnerErrorCatcher (/home/runner/work/_actions/github/codeql-action/v2/lib/toolrunner-error-catcher.js:86:19) + at processTicksAndRejections (node:internal/process/task_queues:96:5) + at async Object.extractScannedLanguage (/home/runner/work/_actions/github/codeql-action/v2/lib/codeql.js:519:13) + at async createdDBForScannedLanguages (/home/runner/work/_actions/github/codeql-action/v2/lib/analyze.js:79:13) + at async finalizeDatabaseCreation (/home/runner/work/_actions/github/codeql-action/v2/lib/analyze.js:96:5) + at async runFinalize (/home/runner/work/_actions/github/codeql-action/v2/lib/analyze.js:259:5) + at async run (/home/runner/work/_actions/github/codeql-action/v2/lib/analyze-action.js:78:9) + at async runWrapper (/home/runner/work/_actions/github/codeql-action/v2/lib/analyze-action.js:212:9) +``` + +In your logs, look for an exception with the output `[build-stderr]` + +## [build-stderr] java.lang.OutOfMemoryError: Java heap space +``` +[build-stderr] Exception while extracting /home/runner/work/path/to/file/myfile.js. +[build-stderr] java.lang.OutOfMemoryError: Java heap space +[build-stderr] at java.base/java.util.HashMap.resize(Unknown Source) +[build-stderr] at java.base/java.util.HashMap.putVal(Unknown Source) +[build-stderr] at java.base/java.util.HashMap.put(Unknown Source) +[build-stderr] at java.base/java.util.HashSet.add(Unknown Source) +[build-stderr] at com.semmle.js.extractor.LocationManager.emitLocationsDefault(LocationManager.java:156) +[build-stderr] at com.semmle.js.extractor.LocationManager.emitFileLocation(LocationManager.java:146) +[build-stderr] at com.semmle.js.extractor.LocationManager.emitSnippetLocation(LocationManager.java:141) +[build-stderr] at com.semmle.js.extractor.LocationManager.emitNodeLocation(LocationManager.java:126) +[build-stderr] at com.semmle.js.extractor.LexicalExtractor.extractTokens(LexicalExtractor.java:166) +[build-stderr] at com.semmle.js.extractor.JSExtractor.extract(JSExtractor.java:113) +[build-stderr] at com.semmle.js.extractor.JSExtractor.extract(JSExtractor.java:59) +[build-stderr] at com.semmle.js.extractor.ScriptExtractor.extract(ScriptExtractor.java:85) +[build-stderr] at com.semmle.js.extractor.FileExtractor.extractContents(FileExtractor.java:545) +[build-stderr] at com.semmle.js.extractor.FileExtractor.extract(FileExtractor.java:452) +[build-stderr] at com.semmle.js.extractor.AutoBuild.doExtract(AutoBuild.java:1122) +[build-stderr] at com.semmle.js.extractor.AutoBuild.lambda$extract$8(AutoBuild.java:1106) +[build-stderr] at com.semmle.js.extractor.AutoBuild$$Lambda$29/0x00000008000d4950.run(Unknown Source) +[build-stderr] at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(Unknown Source) +[build-stderr] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) +[build-stderr] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) +[build-stderr] at java.base/java.lang.Thread.run(Unknown Source) +``` + +It is best to use the [paths-ignore](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#specifying-directories-to-scan) feature to exclude the file opencv.js. It would look something like this: + +paths-ignore: + - '**/myfile.js' + +In general, when a repository contains a huge JS file that is output of a compiler/bundler process, it is best to exclude it. As an example: opencv.js is from the OpenCV project, it which case it is the result of translating some C++ code to JS using Emscripten. This source code is unreadable and it wouldn't be helpful to get code scanning alerts in such files anyway. + +References: +- https://github.com/github/codeql/issues/9056#issuecomment-1120793848 \ No newline at end of file From 382d374e1f606a40bbfe4df29f92dbe8fcd9aa21 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:42:37 -0400 Subject: [PATCH 058/128] yml --- codeql/interpreted-languages.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codeql/interpreted-languages.md b/codeql/interpreted-languages.md index 1019ae9..e8932e1 100644 --- a/codeql/interpreted-languages.md +++ b/codeql/interpreted-languages.md @@ -53,8 +53,10 @@ In your logs, look for an exception with the output `[build-stderr]` It is best to use the [paths-ignore](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#specifying-directories-to-scan) feature to exclude the file opencv.js. It would look something like this: +```yml paths-ignore: - '**/myfile.js' +``` In general, when a repository contains a huge JS file that is output of a compiler/bundler process, it is best to exclude it. As an example: opencv.js is from the OpenCV project, it which case it is the result of translating some C++ code to JS using Emscripten. This source code is unreadable and it wouldn't be helpful to get code scanning alerts in such files anyway. From 717c5f7d68625c4ccd025f0a52bbd957dd062492 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Thu, 20 Oct 2022 21:41:08 -0400 Subject: [PATCH 059/128] Links --- codeql/compiled-languages-csharp.md | 65 ++++++++++++++--------------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/codeql/compiled-languages-csharp.md b/codeql/compiled-languages-csharp.md index ac626b8..61a9642 100644 --- a/codeql/compiled-languages-csharp.md +++ b/codeql/compiled-languages-csharp.md @@ -9,35 +9,33 @@ Scanning a C# application with CodeQL CodeQL will extract and analyze any code that is passed through the compiler. Consider excluding any code you do not wish to include in a security scan to speed up and remove noise from this process. With .NET we can employ a few mechanisms to remove test/demo code from CodeQL scans (e.g. you would want to run your unit test in another workflow ): - * A solution filter to only build required projects - * An explicit solution file that excludes projects - * have a nice ref on this one from an OSS project Identity Server - * have a build.sh script wrapper around their solution targets - * distinct IdentityServer.CodeQL.sln solution excluding unit tests - * CodeQL yaml passes in a flag to build script to use the CodeQL solution - * Build in release mode - exclude test projects from that build configuration - - +- A [solution filter](https://docs.microsoft.com/en-us/visualstudio/msbuild/solution-filters?view=vs-2019) to only build required projects +- An explicit [solution file that excludes projects](https://docs.microsoft.com/en-us/visualstudio/ide/how-to-exclude-projects-from-a-build?view=vs-2022) + - example from the Open Source project: [Identity Server](https://github.com/DuendeSoftware/IdentityServer/) + - have a [build.sh script wrapper](https://github.com/DuendeSoftware/IdentityServer/blob/main/build/Program.cs#L47) around their solution targets + - distinct [IdentityServer.CodeQL.sln solution](https://github.com/DuendeSoftware/IdentityServer/blob/main/Duende.IdentityServer.CodeQL.sln) excluding unit tests + - [CodeQL yaml passes in a flag to build script](https://github.com/DuendeSoftware/IdentityServer/blob/44d8d5964edfae20c4be424c0b3a2ed5050c6fe9/.github/workflows/codeql-analysis.yml#L57) to use the CodeQL solution +- Build in release mode - exclude test projects from that [build configuration](https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2015/ide/how-to-create-and-edit-configurations?view=vs-2015&redirectedfrom=MSDN#to-modify-a-solution-wide-build-configuration) ## Optimizations - CodeQL Queries -* Tweak your current codeql yml workflow in a few ways: - * remove security-extended queries, the default query pack with smaller set of queries will complete faster - * As of v2.10.5 - Query Suite Counts - * code-scanning (default) - 49 queries - * security-extended - 66 queries - * security-and-quality - 171 queries - -* Micro Optimizations -Consider these as a potential quick hit to resolve a specific problem +- Tweak your current codeql yml workflow in a few ways: + - remove security-extended queries, the default query pack with smaller set of queries will complete faster + - As of [v2.10.5](https://github.com/github/codeql-action/releases/tag/codeql-bundle-20220908) - Query Suite Counts + - code-scanning (default) - 49 queries + - security-extended - 66 queries + - security-and-quality - 171 queries - * Review workflow log timings to identify a any query that is taking a long time you can consider excluding it In via a CodeQL-config file - * add a query-filter to exclude a specific query from analysis +- Micro Optimizations: Consider these as a potential quick hit to resolve a specific problem + - Review workflow log timings to identify a any query that is taking a long time you can consider excluding it via a [CodeQL-config file](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-a-custom-configuration-file) + - add a [query-filter](https://github.blog/changelog/2022-08-31-code-scanning-customize-your-codeql-analysis-using-query-filters/) to [exclude a specific query from analysis](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#excluding-specific-queries-from-analysis) - * tweak the way CodeQL allocates memory to possibly make the workflow (for example, just below the runs-on field): see sample -env: - CODEQL_ACTION_EXTRA_OPTIONS: '{"database": {"run-queries": ["--off-heap-ram=0"]}}' - - * CodeQL will (by default) pull in source code from your dependencies using CIL extraction to assist in mapping out your data flows. While this can drastically improve the precision of the results, this can also lead to a large increase in database size. You might consider disabling this feature for a quick scan but running a cron based scan with the option enabled. + - tweak the way CodeQL allocates memory to possibly make the workflow succeed in low memory conditions (for example, just below the runs-on field): [see sample](https://github.com/vulna-felickz/FullDotNetWebApp/pull/8/commits/263bbc8816a964d70f6267f6b6717f56b6bf6a1d) + ```yml + env: + CODEQL_ACTION_EXTRA_OPTIONS: '{"database": {"run-queries": ["--off-heap-ram=0"]}}' + ``` + + - CodeQL will (by default) pull in source code from your dependencies using CIL extraction to assist in mapping out your data flows. While this can drastically improve the precision of the results, this can also lead to a large increase in database size. You might consider disabling this feature for a quick scan but running a cron based scan with the option enabled. ```yml env: CODEQL_EXTRACTOR_CSHARP_OPTION_CIL: false @@ -48,13 +46,12 @@ env: with: tools: latest ``` - - - * Running low on disk using the default Actions runner? "You are running out of disk space. The runner will stop working when the machine runs out of disk space." + + - Running low on disk using the default Actions runner? "You are running out of disk space. The runner will stop working when the machine runs out of disk space." Try a few of these workarounds for a potential quick fix: - * Clean up large directories you are potentially not using on the windows runner , add this to your “CodeQL” workflow: + - Clean up large directories of [preinstalled software](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#preinstalled-software) that you are potentially not using on the windows runners, add this to your “CodeQL” workflow: ```yml - name: Clean up some disks run: | @@ -62,21 +59,21 @@ env: docker system prune -af ``` - * Specify the temp directory to store the CodeQL database - I have seen this resolve this specific problem with a Windows env (runs-on: windows-2019) + - Specify the temp directory to store the CodeQL database - I have seen this resolve this specific problem with a Windows env (runs-on: windows-2019) - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: db-location: ‘C:\windows\temp\codeql-database’ - * See also: Vertical Scaling section + - See also: Vertical Scaling section ## Vertical Scaling - Throw hardware at the software problem. Large applications can be compute/mem.ory/disk bound as the base Actions runners are small instances (2core/8GBram/14GB SSD). See the recommended hardware requirements for CodeQL based on Codebase size. - * Setup a self hosted CI action runner in your infrastructure that has some more powerful specs that can handle your large application. - * See Adding a self-hosted runner to an organization. - * Enroll for the larger runners public beta + - Setup a self hosted CI action runner in your infrastructure that has some more powerful specs that can handle your large application. + - See Adding a self-hosted runner to an organization. + - Enroll for the larger runners public beta This allows for up to a 64 core machine with 256GB RAM ## Horizontal Scaling - Continue to decompose your solution. From d4dd6785e36ab12ee1dc89202d2f15c5f9f2204f Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 21 Oct 2022 17:49:31 -0400 Subject: [PATCH 060/128] java private package registries --- codeql/compiled-languages-java.md | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 codeql/compiled-languages-java.md diff --git a/codeql/compiled-languages-java.md b/codeql/compiled-languages-java.md new file mode 100644 index 0000000..6178c87 --- /dev/null +++ b/codeql/compiled-languages-java.md @@ -0,0 +1,51 @@ + +# Private Package Registries + +## The codeql for java is failing when it tries to do mvn command and tries to access a artifactory repo where our pom.xml are stored. + +Assuming the given package registry instance is publicly accessible: + + +Option 1 - Pass credentials via environment variable from Actions secrets and configure Maven settings to utilize those credentials (see sample [here](https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#yaml-example)) + +ex `workflow.yml` step: +```yml + env: + MAVEN_USERNAME: maven_username123 + MAVEN_CENTRAL_TOKEN: ${{ secrets.MAVEN_CENTRAL_TOKEN }} +``` + +ex `settings.xml` +```xml + + maven + ${env.MAVEN_USERNAME} + ${env.MAVEN_CENTRAL_TOKEN} + +``` + +Option 2 - Use the [maven-settings-action](https://github.com/s4u/maven-settings-action) to dynamically create/overrite a `settings.xml` that contains the credentials for your specified package manager. + +```yml +- if: matrix.language == 'java' + name: Configure maven credentials + uses: s4u/maven-settings-action@v2.6.0 + with: + servers: '[{"id": "central", "username": "${{ secrets.MAVEN_USERNAME }}", "password": "${{ secrets.MAVEN_CENTRAL_TOKEN }}"}]' +``` + +# FAQ + +## java.lang.IllegalArgumentException: Unsupported class file major version ## + +Ensure you are compiling your java application using CodeQL tracing on a supported version of the JDK as found here: https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/ + +## Fatal error compiling: error: invalid target release: ## + +Specify your [desired java version via the setup-java action](https://github.com/actions/setup-java#supported-version-syntax) +```yml + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'microsoft' +``` \ No newline at end of file From 0bc1084dcbf45c5cd9d7d5583a15a189fe5e88ac Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 21 Oct 2022 17:53:18 -0400 Subject: [PATCH 061/128] csharp links --- codeql/compiled-languages-csharp.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/codeql/compiled-languages-csharp.md b/codeql/compiled-languages-csharp.md index 61a9642..e8eeff0 100644 --- a/codeql/compiled-languages-csharp.md +++ b/codeql/compiled-languages-csharp.md @@ -70,14 +70,14 @@ With .NET we can employ a few mechanisms to remove test/demo code from CodeQL sc ## Vertical Scaling - Throw hardware at the software problem. -Large applications can be compute/mem.ory/disk bound as the base Actions runners are small instances (2core/8GBram/14GB SSD). See the recommended hardware requirements for CodeQL based on Codebase size. +Large applications can be compute/mem.ory/disk bound as the base Actions runners are small instances (2core/8GBram/14GB SSD). See the [recommended hardware requirements for CodeQL](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/recommended-hardware-resources-for-running-codeql) based on Codebase size. - Setup a self hosted CI action runner in your infrastructure that has some more powerful specs that can handle your large application. - - See Adding a self-hosted runner to an organization. - - Enroll for the larger runners public beta - This allows for up to a 64 core machine with 256GB RAM + - See [Adding a self-hosted runner to an organization.](https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners#adding-a-self-hosted-runner-to-an-organization) + - Enroll for access to the [Actions larger runners ](https://github.blog/2022-09-01-github-actions-introducing-the-new-larger-github-hosted-runners-beta/) + - This allows for up to a 64 core machine with 256GB RAM ## Horizontal Scaling - Continue to decompose your solution. -Making an investment in optimizing your build process can drastically speed up your developer experience, CI pipelines and start the process of "decomposing the monolith". Continued investment in large or legacy applications is important to keep your security posture and dependencies up to date. Review the suggestions for using domain-driven design to modernize your monolithic application. If you have already investigated removing projects from your solutions that you do not wish to expose to a security scan (test projects / demo code), then you have already begun this journey. +Making an investment in optimizing your build process can drastically speed up your developer experience, CI pipelines and start the process of "decomposing the monolith". Continued investment in large or legacy applications is important to keep your security posture and dependencies up to date. Review the suggestions for [using domain-driven design to modernize your monolithic application](https://learn.microsoft.com/en-us/azure/architecture/microservices/migrate-monolith). If you have already investigated removing projects from your solutions that you do not wish to expose to a security scan (test projects / demo code), then you have already begun this journey. -A great use case would be to filter separate solutions by front end (ex: Web.sln) and back end code(ex: API.sln) that are separated by process/network boundaries. CodeQL can detect data flows through the code but once it reaches a process boundary the flow is stopped. This creates a natural separation point for both feature development teams and security scans based data flows. This would further enable a decrease in wall-clock scan time by using parallel per-solution scans using Action matrix strategy (such that each gets its own runtime and resources). It will be important to include your common framework code in each solution so that you get a successful compilation while you further analyze other ways to share code. +A great use case would be to filter separate solutions by front end (ex: Web.sln) and back end code(ex: API.sln) that are separated by process/network boundaries. CodeQL can detect data flows through the code but once it reaches a process boundary the flow is stopped. This creates a natural separation point for both feature development teams and security scans based data flows. This would further enable a decrease in wall-clock scan time by using parallel per-solution scans using an [Actions matrix strategy](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs) (such that each gets its own runtime and resources). It will be important to include your common framework code in each solution so that you get a successful compilation while you further analyze other ways to share code. From 0b446abaf3665d66f88f0f21ba3e581a48d755db Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 21 Oct 2022 18:15:18 -0400 Subject: [PATCH 062/128] fix MD --- codeql/compiled-languages-java.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeql/compiled-languages-java.md b/codeql/compiled-languages-java.md index 6178c87..0e03a48 100644 --- a/codeql/compiled-languages-java.md +++ b/codeql/compiled-languages-java.md @@ -40,7 +40,7 @@ Option 2 - Use the [maven-settings-action](https://github.com/s4u/maven-settings Ensure you are compiling your java application using CodeQL tracing on a supported version of the JDK as found here: https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/ -## Fatal error compiling: error: invalid target release: ## +## Fatal error compiling: error: invalid target release: \## Specify your [desired java version via the setup-java action](https://github.com/actions/setup-java#supported-version-syntax) ```yml From e4926bd7414b331d9a206c6b2ffd05ce00eb92bc Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:42:31 -0400 Subject: [PATCH 063/128] directory move --- .../codeql-builds}/compiled-languages-cpp.md | 0 .../codeql-builds}/compiled-languages-csharp.md | 0 .../codeql-builds}/compiled-languages-java.md | 0 {codeql => troubleshooting/codeql-builds}/compiled-languages.md | 0 .../codeql-builds}/interpreted-languages.md | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {codeql => troubleshooting/codeql-builds}/compiled-languages-cpp.md (100%) rename {codeql => troubleshooting/codeql-builds}/compiled-languages-csharp.md (100%) rename {codeql => troubleshooting/codeql-builds}/compiled-languages-java.md (100%) rename {codeql => troubleshooting/codeql-builds}/compiled-languages.md (100%) rename {codeql => troubleshooting/codeql-builds}/interpreted-languages.md (100%) diff --git a/codeql/compiled-languages-cpp.md b/troubleshooting/codeql-builds/compiled-languages-cpp.md similarity index 100% rename from codeql/compiled-languages-cpp.md rename to troubleshooting/codeql-builds/compiled-languages-cpp.md diff --git a/codeql/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md similarity index 100% rename from codeql/compiled-languages-csharp.md rename to troubleshooting/codeql-builds/compiled-languages-csharp.md diff --git a/codeql/compiled-languages-java.md b/troubleshooting/codeql-builds/compiled-languages-java.md similarity index 100% rename from codeql/compiled-languages-java.md rename to troubleshooting/codeql-builds/compiled-languages-java.md diff --git a/codeql/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md similarity index 100% rename from codeql/compiled-languages.md rename to troubleshooting/codeql-builds/compiled-languages.md diff --git a/codeql/interpreted-languages.md b/troubleshooting/codeql-builds/interpreted-languages.md similarity index 100% rename from codeql/interpreted-languages.md rename to troubleshooting/codeql-builds/interpreted-languages.md From 96ab5a57ee92459ecf2fe1d6c0b0421ff55d6484 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:44:22 -0400 Subject: [PATCH 064/128] Add java --- troubleshooting/codeql-builds/compiled-languages.md | 1 + 1 file changed, 1 insertion(+) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index 107dfc8..488dad5 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -4,6 +4,7 @@ ## Language Specific Guidance * [CSharp](compiled-languages-csharp.md) * [C++](compiled-languages-cpp.md) +* [C++](compiled-languages-java.md) ## How CodeQL Tracer Works From c399fa4c96db481cc028a70a9c2e238492719e98 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:44:57 -0400 Subject: [PATCH 065/128] Java --- troubleshooting/codeql-builds/compiled-languages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index 488dad5..a90a602 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -4,7 +4,7 @@ ## Language Specific Guidance * [CSharp](compiled-languages-csharp.md) * [C++](compiled-languages-cpp.md) -* [C++](compiled-languages-java.md) +* [Java](compiled-languages-java.md) ## How CodeQL Tracer Works From 52e0923022c0c1950ab505b36c01dbaad18bd309 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 26 Oct 2022 18:49:43 -0400 Subject: [PATCH 066/128] python --- .../interpreted-languages-python.md | 37 +++++++++++++++++++ .../codeql-builds/interpreted-languages.md | 4 ++ 2 files changed, 41 insertions(+) create mode 100644 troubleshooting/codeql-builds/interpreted-languages-python.md diff --git a/troubleshooting/codeql-builds/interpreted-languages-python.md b/troubleshooting/codeql-builds/interpreted-languages-python.md new file mode 100644 index 0000000..09f7a74 --- /dev/null +++ b/troubleshooting/codeql-builds/interpreted-languages-python.md @@ -0,0 +1,37 @@ +# Autobuild failures + +## ERROR: In --require-hashes mode, all requirements must have their versions pinned with ==. + +Error/Warning in the workflow logs like: + +```yml +ERROR: In --require-hashes mode, all requirements must have their versions pinned with ==. These do not: + importlib-metadata from https://files.pythonhosted.org/packages/b5/64/ef29a63cf08f047bb7fb22ab0f1f774b87eed0bb46d067a5a524798a4af8/importlib_metadata-5.0.0-py3-none-any.whl (from alembic==1.8.1->-r requirements.txt (line ###)) +package installation with `pip install -r requirements.txt` failed, see error above +##[endgroup] +##[warning]An error occurred while trying to automatically install Python dependencies: Error: The process '/usr/bin/python3' failed with exit code 1 +Please make sure any necessary dependencies are installed before calling the codeql-action/analyze step, and add a 'setup-python-dependencies: false' argument to this step to disable our automatic dependency installation and avoid this warning. +``` +![image](https://user-images.githubusercontent.com/1760475/198150549-61326671-e7cc-4cbc-b640-4858fe294f93.png) + + +This is likely due to a python/pypi/pip version mismatch. Ensure you configure your proper version required to build via [actions/setup-python](https://github.com/actions/setup-python/blob/main/docs/advanced-usage.md#using-the-python-version-input) +- see [stackoverflow](https://stackoverflow.com/a/72980455/343347) + +To resolve, specify your required versions before the codeql-action/init step: +```yml +#Set python version +- uses: actions/setup-python@v4 + with: + python-version: '3.9' + cache: 'pip' # caching pip dependencies +#Downgrade to specific pip version +-run: python -m pip install pip==22.0.4 +``` + +Alternatively, you can disable the auto-install dependency functionality. You will need to configure the build requirements/commands from your existing CI. Specify that codeql should disable the python automatic package restoration and run the CI tooling / commands directly before the analysis step as shown [here](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#analyzing-python-dependencies). +```yml + # Override the default behavior so that the action doesn't attempt + # to auto-install Python dependencies + setup-python-dependencies: false +``` \ No newline at end of file diff --git a/troubleshooting/codeql-builds/interpreted-languages.md b/troubleshooting/codeql-builds/interpreted-languages.md index e8932e1..3e21855 100644 --- a/troubleshooting/codeql-builds/interpreted-languages.md +++ b/troubleshooting/codeql-builds/interpreted-languages.md @@ -1,6 +1,10 @@ # Scanning an interpreted language with CodeQL * NOTE: This guide will focus on GitHub Actions but the concepts can be applied to the CodeQL CLI on other CI platforms. +## Language Specific Guidance +* [Python](interpreted-languages-python) + + # Troubleshooting ## [ERROR] Spawned process exited abnormally (code 1; tried to run: [/opt/hostedtoolcache/CodeQL//x64/codeql/javascript/tools/autobuild.sh]) From e43fb45ecb7468223fc2f3494967028d23cbe742 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 28 Oct 2022 17:15:50 -0400 Subject: [PATCH 067/128] Docs enhancements --- .../compiled-languages-csharp.md | 111 ++++++++++++++++-- .../codeql-builds/compiled-languages.md | 10 +- 2 files changed, 105 insertions(+), 16 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index e8eeff0..9c3a300 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -1,10 +1,101 @@ Scanning a C# application with CodeQL -# Speed up C# Analysis +# Build Failures + +## [error]We were unable to automatically build your code. Please replace the call to the autobuild action with your custom build steps. + +
+Expand for sample workflow failure output + +``` + Exit code 1 + Attempting to locate build script + Error: Could not auto-detect a suitable build method + Error: We were unable to automatically build your code. Please replace the call to the autobuild action with your custom build steps. Failure invoking /opt/hostedtoolcache/CodeQL/0.0.0-20221010/x64/codeql/csharp/tools/autobuild.sh with arguments . + + Exit code 1 and error was: + + Error: Could not auto-detect a suitable build method + + CommandInvocationError: Failure invoking /opt/hostedtoolcache/CodeQL/0.0.0-20221010/x64/codeql/csharp/tools/autobuild.sh with arguments . + + Exit code 1 and error was: + + Error: Could not auto-detect a suitable build method + + at runTool (/home/runner/work/_actions/github/codeql-action/v2/lib/codeql.js:867:15) + at processTicksAndRejections (node:internal/process/task_queues:96:5) + at async Object.runAutobuild (/home/runner/work/_actions/github/codeql-action/v2/lib/codeql.js:559:13) + at async runAutobuild (/home/runner/work/_actions/github/codeql-action/v2/lib/autobuild.js:97:5) + at async run (/home/runner/work/_actions/github/codeql-action/v2/lib/autobuild-action.js:71:17) + at async runWrapper (/home/runner/work/_actions/github/codeql-action/v2/lib/autobuild-action.js:88:9) +``` +
+ + +This error indicates there is a scenario where our C# AutoBuilder is unable to build your code. No sweat, check out some of the resources below to get you started: + +Ensure your required build tooling is installed your [runner](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) + - Windows 2019 runner + - [Visual Studio 2019 Enterprise pre-installed](https://github.com/actions/runner-images/blob/main/images/win/Windows2019-Readme.md#visual-studio-enterprise-2019) + - [.NET Framework Developer Pack](https://github.com/actions/runner-images/blob/main/images/win/Windows2019-Readme.md#net-framework) + - [.NET Core SDK](https://github.com/actions/runner-images/blob/main/images/win/Windows2019-Readme.md#net-core-sdk) + - Windows 2022 runner + - [Visual Studio 2022 Enterprise pre-installed](https://github.com/actions/runner-images/blob/main/images/win/Windows2022-Readme.md#visual-studio-enterprise-2022) + - [.NET Framework Developer Pack](https://github.com/actions/runner-images/blob/main/images/win/Windows2022-Readme.md#net-framework) + - [.NET Core SDK](https://github.com/actions/runner-images/blob/main/images/win/Windows2022-Readme.md#net-core-sdk) + + +### DotNet (.NET standard / core / ) +Using `dotnet` is best documented at: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net + +### .NET Framework Manual Build Steps on Windows Runners +NOTE: if you require windows OS to build, ensure you are using a windows runner. + +Example using `windows-latest`: +- Note: The `-latest` runner images are the latest stable images that GitHub provides, and might not be the most recent version of the operating system available from the operating system vendor. +```yml +jobs: + analyze: + name: Analyze + runs-on: windows-latest +``` + +Next, consider specifying your own build steps from an existing CI workflow: -- Start here: [CodeQL Docs - The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow#the-build-takes-too-long) +```yml + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + #- name: Autobuild + # uses: github/codeql-action/autobuild@v2 + # Discover where the MSBuild tool is and automatically add it to the PATH environment variable + - name: Setup MSBuild + uses: microsoft/setup-msbuild@v1 + + # Download/installs a given version of NuGet.exe. Using this action will add nuget to your $PATH + - name: Setup NuGet + uses: NuGet/setup-nuget@v1 + + # CI build with best practices from: https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/#specifying-build-commands + - name: .NET Build Steps + run: | + nuget restore .\FullDotNetWebApp.sln -DisableParallelProcessing + msbuild .\FullDotNetWebApp.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform="Any CPU" /p:Configuration="Debug" /p:MvcBuildViews=true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 +``` + + + + + +# Speed up C# Analysis + +Start here: [CodeQL Docs - The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow#the-build-takes-too-long). + ## Optimization - Removing Unit Tests CodeQL will extract and analyze any code that is passed through the compiler. Consider excluding any code you do not wish to include in a security scan to speed up and remove noise from this process. @@ -40,12 +131,6 @@ With .NET we can employ a few mechanisms to remove test/demo code from CodeQL sc env: CODEQL_EXTRACTOR_CSHARP_OPTION_CIL: false ``` - This requires CodeQL CLI v2.11.0+, for the short term (October 2022) - add this to ensure you run the latest version: - ```yml - uses: github/codeql-action/init@v2 - with: - tools: latest - ``` - Running low on disk using the default Actions runner? "You are running out of disk space. The runner will stop working when the machine runs out of disk space." @@ -70,11 +155,11 @@ With .NET we can employ a few mechanisms to remove test/demo code from CodeQL sc ## Vertical Scaling - Throw hardware at the software problem. -Large applications can be compute/mem.ory/disk bound as the base Actions runners are small instances (2core/8GBram/14GB SSD). See the [recommended hardware requirements for CodeQL](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/recommended-hardware-resources-for-running-codeql) based on Codebase size. - - Setup a self hosted CI action runner in your infrastructure that has some more powerful specs that can handle your large application. - - See [Adding a self-hosted runner to an organization.](https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners#adding-a-self-hosted-runner-to-an-organization) - - Enroll for access to the [Actions larger runners ](https://github.blog/2022-09-01-github-actions-introducing-the-new-larger-github-hosted-runners-beta/) - - This allows for up to a 64 core machine with 256GB RAM +Large applications can be compute/memory/disk bound as the base Actions runners are small instances (2core/8GBram/14GB SSD). See the [recommended hardware requirements for CodeQL](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/recommended-hardware-resources-for-running-codeql) based on Codebase size. +- Setup a [self-hosted CI action runner](https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners#adding-a-self-hosted-runner-to-an-organization) in your infrastructure that has some more powerful specs that can handle your large application. +- [Actions larger runners ](https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners) + - This allows for up to a 64 core machine with 256GB RAM and 2040 GB of SSD storage + ## Horizontal Scaling - Continue to decompose your solution. diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index a90a602..bf3c501 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -48,9 +48,13 @@ alternatively we can further define limits ``` ## Reviewing Results -- [ ] Log viewing -- [ ] Diagnostics Logs Mode -- [ ] [Exit Codes](https://codeql.github.com/docs/codeql-cli/exit-codes/) + +Helpful Articles to understand how to review, troubleshoot, and debug logs: + +- [Viewing Code Scanning Logs](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/viewing-code-scanning-logs) +- [Workflow verbose logging in debug mode](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow#creating-codeql-debugging-artifacts) +- [Adding artifacts on every CodeQL Run](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow#creating-codeql-debugging-artifacts-using-a-workflow-flag) +- [Exit Codes](https://codeql.github.com/docs/codeql-cli/exit-codes/) ## Optimizaitons - CodeQL Docs - [The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow) \ No newline at end of file From 70b1a5c1a216b69c2d5e67e42830e887100a6c23 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 28 Oct 2022 17:21:30 -0400 Subject: [PATCH 068/128] Links for autobuild + runtime customization --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 9c3a300..11d64e2 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -33,7 +33,7 @@ Scanning a C# application with CodeQL -This error indicates there is a scenario where our C# AutoBuilder is unable to build your code. No sweat, check out some of the resources below to get you started: +This error indicates there is a scenario where our [C# AutoBuilder](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#c) is unable to build your code. No sweat, check out some of the resources below to get you started: Ensure your required build tooling is installed your [runner](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) - Windows 2019 runner @@ -46,6 +46,9 @@ Ensure your required build tooling is installed your [runner](https://docs.githu - [.NET Core SDK](https://github.com/actions/runner-images/blob/main/images/win/Windows2022-Readme.md#net-core-sdk) +If any custom tooling is required, consider pulling into your action via [custom script](https://docs.github.com/en/actions/using-github-hosted-runners/customizing-github-hosted-runners) + + ### DotNet (.NET standard / core / ) Using `dotnet` is best documented at: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net From f3dce72ef2759eab54bbc03303625f741dd9fd8a Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 28 Oct 2022 17:32:05 -0400 Subject: [PATCH 069/128] clean up sections --- .../codeql-builds/compiled-languages-cpp.md | 4 +- .../compiled-languages-csharp.md | 45 +++++++++---------- .../codeql-builds/compiled-languages-java.md | 2 +- .../interpreted-languages-python.md | 2 +- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-cpp.md b/troubleshooting/codeql-builds/compiled-languages-cpp.md index 804b028..c7560d6 100644 --- a/troubleshooting/codeql-builds/compiled-languages-cpp.md +++ b/troubleshooting/codeql-builds/compiled-languages-cpp.md @@ -1,5 +1,5 @@ -## FAQ +# FAQ -### Does CodeQL need the resulting object files from the C++ build? +## Does CodeQL need the resulting object files from the C++ build? CodeQL needs to monitor the actual build. Every time we see the build invoke the C++ compiler, we also "compile" the same source code with our own "compiler" that generates what we need for the actual analysis. We don't use the object files from the regular compiler at all. \ No newline at end of file diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 11d64e2..8d05214 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -72,12 +72,11 @@ Next, consider specifying your own build steps from an existing CI workflow: #- name: Autobuild # uses: github/codeql-action/autobuild@v2 - - # Discover where the MSBuild tool is and automatically add it to the PATH environment variable + # Discover where the MSBuild tool is and automatically add it to the PATH environment variable - name: Setup MSBuild uses: microsoft/setup-msbuild@v1 - # Download/installs a given version of NuGet.exe. Using this action will add nuget to your $PATH + # Download/installs a given version of NuGet.exe. Using this action will add nuget to your $PATH - name: Setup NuGet uses: NuGet/setup-nuget@v1 @@ -91,8 +90,28 @@ Next, consider specifying your own build steps from an existing CI workflow: uses: github/codeql-action/analyze@v2 ``` +## "You are running out of disk space. The runner will stop working when the machine runs out of disk space." + +Running low on disk using the default Actions runner? Try a few of these workarounds for a potential quick fix: + + Clean up large directories of [preinstalled software](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#preinstalled-software) that you are potentially not using on the windows runners, add this to your “CodeQL” workflow: +```yml +- name: Clean up some disks + run: | + rd C:\Android\android-sdk + docker system prune -af +``` + Specify the temp directory to store the CodeQL database - I have seen this resolve this specific problem with a Windows env (runs-on: windows-2019) + ```yml +- name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + db-location: ‘C:\windows\temp\codeql-database’ +``` + +- See also: [Vertical Scaling](#vertical-scaling---throw-hardware-at-the-software-problem) # Speed up C# Analysis @@ -135,26 +154,6 @@ With .NET we can employ a few mechanisms to remove test/demo code from CodeQL sc CODEQL_EXTRACTOR_CSHARP_OPTION_CIL: false ``` - - Running low on disk using the default Actions runner? "You are running out of disk space. The runner will stop working when the machine runs out of disk space." - - Try a few of these workarounds for a potential quick fix: - - - Clean up large directories of [preinstalled software](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#preinstalled-software) that you are potentially not using on the windows runners, add this to your “CodeQL” workflow: - ```yml - - name: Clean up some disks - run: | - rd C:\Android\android-sdk - docker system prune -af - ``` - - - Specify the temp directory to store the CodeQL database - I have seen this resolve this specific problem with a Windows env (runs-on: windows-2019) - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - db-location: ‘C:\windows\temp\codeql-database’ - - - See also: Vertical Scaling section - ## Vertical Scaling - Throw hardware at the software problem. diff --git a/troubleshooting/codeql-builds/compiled-languages-java.md b/troubleshooting/codeql-builds/compiled-languages-java.md index 0e03a48..440b589 100644 --- a/troubleshooting/codeql-builds/compiled-languages-java.md +++ b/troubleshooting/codeql-builds/compiled-languages-java.md @@ -34,7 +34,7 @@ Option 2 - Use the [maven-settings-action](https://github.com/s4u/maven-settings servers: '[{"id": "central", "username": "${{ secrets.MAVEN_USERNAME }}", "password": "${{ secrets.MAVEN_CENTRAL_TOKEN }}"}]' ``` -# FAQ +# Build Failures ## java.lang.IllegalArgumentException: Unsupported class file major version ## diff --git a/troubleshooting/codeql-builds/interpreted-languages-python.md b/troubleshooting/codeql-builds/interpreted-languages-python.md index 09f7a74..6f9b30e 100644 --- a/troubleshooting/codeql-builds/interpreted-languages-python.md +++ b/troubleshooting/codeql-builds/interpreted-languages-python.md @@ -1,4 +1,4 @@ -# Autobuild failures +# Build Failures ## ERROR: In --require-hashes mode, all requirements must have their versions pinned with ==. From 1834f3bd6c2e42c5c98226ef9fd7a5bd93d443ce Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 28 Oct 2022 17:52:34 -0400 Subject: [PATCH 070/128] Reporting updates --- reporting/advanced-security-reporting.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/reporting/advanced-security-reporting.md b/reporting/advanced-security-reporting.md index 7a45f47..660359f 100644 --- a/reporting/advanced-security-reporting.md +++ b/reporting/advanced-security-reporting.md @@ -3,13 +3,19 @@ - Dependabot - [ ] https://github.com/mr-sherman/get-dependency-alerts-in-org - [ ] https://github.com/tonycch/get-dependabot-alerts-sample + - [ ] https://github.com/andyfeller/gh-dependency-report + - [ ] https://github.com/thedave42/generate-dependencies-csv-action - Code scanning - [ ] https://github.com/jhutchings1/get-code-scanning-alerts-in-org-sample - [ ] https://github.com/issc29/generate-vuln-report + - [ ] https://github.com/marketplace/actions/get_code_scanning_result + - [ ] https://github.com/marketplace/actions/github-advanced-security-api-to-csv + - [ ] https://github.com/peter-murray/github-security-report-action - Secret scanning - [ ] GHES 3.1+: https://github.com/cmboling/get-secret-scanning-alerts-in-org-sample/tree/ghes/base-url-included - [ ] dotcom/GHEC: https://github.com/cmboling/get-secret-scanning-alerts-in-org-sample - Other - [ ] https://github.com/ThibaudLopez/GHAS -- Splunk integration - - [ ] https://github.com/splunk/github_app_for_splunk +- SIEM integrations + - [ ] https://github.blog/2022-10-13-introducing-github-advanced-security-siem-integrations-for-security-professionals/ + - [ ] https://resources.github.com/security/integrating-github-advanced-security-with-third-party-platforms/ From 6ed0e2758e50c72a26d59b4d2f5625ebad9bdb5d Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 28 Oct 2022 17:52:51 -0400 Subject: [PATCH 071/128] clean up --- .../codeql-builds/compiled-languages-csharp.md | 4 +++- .../codeql-builds/compiled-languages.md | 14 +++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 8d05214..fd4a706 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -50,7 +50,7 @@ If any custom tooling is required, consider pulling into your action via [custom ### DotNet (.NET standard / core / ) -Using `dotnet` is best documented at: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net +Using `dotnet` is best documented at: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net. The [actions/setup-dotnet](https://github.com/actions/setup-dotnet) action can assist in configuring proper build tools. ### .NET Framework Manual Build Steps on Windows Runners NOTE: if you require windows OS to build, ensure you are using a windows runner. @@ -65,6 +65,8 @@ jobs: ``` Next, consider specifying your own build steps from an existing CI workflow: +- The [microsoft/setup-msbuild](https://github.com/microsoft/setup-msbuild) and [Nuget/setup-nuget](https://github.com/nuget/setup-nuget) actions are popular tools to assist in this configuration + ```yml # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index bf3c501..a18c13d 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -6,19 +6,27 @@ * [C++](compiled-languages-cpp.md) * [Java](compiled-languages-java.md) -## How CodeQL Tracer Works - ## Autobuilder The autobuilder action (see [docs](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#about-autobuild-for-codeql) ) ## Build Customizations See common build configuration and specific compiler flags: [specifying build commands](https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/#specifying-build-commands) - ## Common Problems +### Autobuilder [error]We were unable to automatically build your code. Please replace the call to the autobuild action with your custom build steps. + +See [language specific guidance](#language-specific-guidance) for common resolutions to add custom build steps + + ### 401 due to private package server configuration +Ensure network access from GitHub runners to your private registry is open + - For IP Whitelisting, consider using [Larger Runners with Static IP](https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners#networking-for-larger-runners) + - See Also: [Connecting Actions to a private network](https://docs.github.com/en/actions/using-github-hosted-runners/connecting-to-a-private-network) + +See [language specific guidance](#language-specific-guidance) for authentication options to popular package mangers + ### Out of Memory ex: From cb5ed807ee31f817c9caf0021b8c2f49f977c19d Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 28 Oct 2022 17:57:56 -0400 Subject: [PATCH 072/128] fix python link --- troubleshooting/codeql-builds/interpreted-languages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/interpreted-languages.md b/troubleshooting/codeql-builds/interpreted-languages.md index 3e21855..2ee96e0 100644 --- a/troubleshooting/codeql-builds/interpreted-languages.md +++ b/troubleshooting/codeql-builds/interpreted-languages.md @@ -2,7 +2,7 @@ * NOTE: This guide will focus on GitHub Actions but the concepts can be applied to the CodeQL CLI on other CI platforms. ## Language Specific Guidance -* [Python](interpreted-languages-python) +* [Python](interpreted-languages-python.md) # Troubleshooting From f23d8afb8e06e99c25c2fcd73133f2316f78500c Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 11 Jan 2023 10:40:58 -0500 Subject: [PATCH 073/128] CodeQL - CSharp - NuGet Cache --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index fd4a706..8c7d4b5 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -120,6 +120,9 @@ Running low on disk using the default Actions runner? Try a few of these workaro Start here: [CodeQL Docs - The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow#the-build-takes-too-long). +## Optimization - Caching Dependencies + Depending on the number of dependencies, it may be faster to restore packages for your project using the Actions dependency cache. Projects with many large dependencies should see a performance increase as it cuts down the time required for downloading. Projects with fewer dependencies may not see a significant performance increase and may even see a slight decrease due to how NuGet installs cached dependencies. The performance varies from project to project. See [this article](https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net#caching-dependencies) for configuring the NuGet dependency cache. + ## Optimization - Removing Unit Tests CodeQL will extract and analyze any code that is passed through the compiler. Consider excluding any code you do not wish to include in a security scan to speed up and remove noise from this process. From 2a6fe64dc9be565e5fd51014d724258c45bd681f Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:28:36 -0500 Subject: [PATCH 074/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 8c7d4b5..56b03c3 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -49,9 +49,12 @@ Ensure your required build tooling is installed your [runner](https://docs.githu If any custom tooling is required, consider pulling into your action via [custom script](https://docs.github.com/en/actions/using-github-hosted-runners/customizing-github-hosted-runners) -### DotNet (.NET standard / core / ) +### DotNet (.NET standard / core ) Using `dotnet` is best documented at: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net. The [actions/setup-dotnet](https://github.com/actions/setup-dotnet) action can assist in configuring proper build tools. +#### NuGet Error NU1301 +This can indicate your custom package server is not configured which may fail the `dotnet restore` command. For private package servers you can follow the follwing guidance to add authentication to package sources: [Setting up authentication for nuget feeds](https://github.com/actions/setup-dotnet#setting-up-authentication-for-nuget-feeds) + ### .NET Framework Manual Build Steps on Windows Runners NOTE: if you require windows OS to build, ensure you are using a windows runner. From bcbbea8105f87ba993768b35ce4316c95a6cf7be Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:41:31 -0500 Subject: [PATCH 075/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 56b03c3..225b17e 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -53,7 +53,7 @@ If any custom tooling is required, consider pulling into your action via [custom Using `dotnet` is best documented at: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net. The [actions/setup-dotnet](https://github.com/actions/setup-dotnet) action can assist in configuring proper build tools. #### NuGet Error NU1301 -This can indicate your custom package server is not configured which may fail the `dotnet restore` command. For private package servers you can follow the follwing guidance to add authentication to package sources: [Setting up authentication for nuget feeds](https://github.com/actions/setup-dotnet#setting-up-authentication-for-nuget-feeds) +This can indicate your custom package server is not configured which may fail the `dotnet restore` command. For private package servers, the follwing guidance shows how to add package sources: [Setting up authentication for nuget feeds](https://github.com/actions/setup-dotnet#setting-up-authentication-for-nuget-feeds) ### .NET Framework Manual Build Steps on Windows Runners NOTE: if you require windows OS to build, ensure you are using a windows runner. From 7dbc1b5364628430a6451942f7b98d3ae5d44e17 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:42:17 -0500 Subject: [PATCH 076/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 225b17e..9c7bb8b 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -87,7 +87,7 @@ Next, consider specifying your own build steps from an existing CI workflow: # CI build with best practices from: https://codeql.github.com/docs/codeql-cli/creating-codeql-databases/#specifying-build-commands - name: .NET Build Steps - run: | + run: | nuget restore .\FullDotNetWebApp.sln -DisableParallelProcessing msbuild .\FullDotNetWebApp.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform="Any CPU" /p:Configuration="Debug" /p:MvcBuildViews=true From 835a906eb6ea361d7ac2440f7893ab0cfcb6db2f Mon Sep 17 00:00:00 2001 From: Pekka Date: Sat, 28 Jan 2023 22:36:34 +0200 Subject: [PATCH 077/128] Fix typo in compiled-languages.md Fix typo in troubleshooting/codeql-builds/compiled-languages.md --- troubleshooting/codeql-builds/compiled-languages.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index a18c13d..6dbafc2 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -64,5 +64,5 @@ Helpful Articles to understand how to review, troubleshoot, and debug logs: - [Adding artifacts on every CodeQL Run](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow#creating-codeql-debugging-artifacts-using-a-workflow-flag) - [Exit Codes](https://codeql.github.com/docs/codeql-cli/exit-codes/) -## Optimizaitons -- CodeQL Docs - [The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow) \ No newline at end of file +## Optimizations +- CodeQL Docs - [The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow) From 9ba480c3b2309a0356f906c2a77547011ba5df86 Mon Sep 17 00:00:00 2001 From: Adrien Pessu <7055334+adrienpessu@users.noreply.github.com> Date: Tue, 7 Feb 2023 11:16:48 +0000 Subject: [PATCH 078/128] add API export when there is an SLL certificate not trusted --- troubleshooting/sarif-upload/troubleshooting.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/troubleshooting/sarif-upload/troubleshooting.md b/troubleshooting/sarif-upload/troubleshooting.md index e8d5eae..cad4b95 100644 --- a/troubleshooting/sarif-upload/troubleshooting.md +++ b/troubleshooting/sarif-upload/troubleshooting.md @@ -43,6 +43,19 @@ codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/fluffy-potato/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 403 Forbidden:::{"message":"Advanced Security must be enabled for this repository to use code scanning.","documentation_url":"https://docs.github.com/enterprise/3.2/rest/reference/code-scanning#upload-a-sarif-file"} ``` +:gift: posting SARIF when the SSL certificate is not trusted: +```dotnetcli +curl \ + -X POST \ + -k \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer "\ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/OWNER/REPO/code-scanning/sarifs \ + -d '{"commit_sha":"","ref":"refs/heads/master","sarif":""}' +``` +More information on the API can be found [here](https://docs.github.com/en/rest/code-scanning?apiVersion=2022-11-28#upload-an-analysis-as-sarif-data) + ### Test environments - GHES 3.2.1 + CodeQL CLI 2.7.2 From e3f09402100ec40baa7b5bf64b516e2c07439580 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 8 Feb 2023 14:18:07 -0500 Subject: [PATCH 079/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 9c7bb8b..ca8789f 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -126,10 +126,10 @@ Start here: [CodeQL Docs - The build takes too long](https://docs.github.com/en ## Optimization - Caching Dependencies Depending on the number of dependencies, it may be faster to restore packages for your project using the Actions dependency cache. Projects with many large dependencies should see a performance increase as it cuts down the time required for downloading. Projects with fewer dependencies may not see a significant performance increase and may even see a slight decrease due to how NuGet installs cached dependencies. The performance varies from project to project. See [this article](https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net#caching-dependencies) for configuring the NuGet dependency cache. -## Optimization - Removing Unit Tests -CodeQL will extract and analyze any code that is passed through the compiler. Consider excluding any code you do not wish to include in a security scan to speed up and remove noise from this process. +## Optimization - Removing Code From Scans +CodeQL will extract and analyze any code that is passed through the compiler. Consider excluding any code you do not wish to include in a security scan to speed up and remove noise from this process. This is commonly employed for unit tests, demo code, or code that would not benefit from being scanned (ex: DacPacs). -With .NET we can employ a few mechanisms to remove test/demo code from CodeQL scans (e.g. you would want to run your unit test in another workflow ): +With .NET we can employ a few mechanisms to remove code from CodeQL scans (e.g. you would want to run your unit test in another workflow ): - A [solution filter](https://docs.microsoft.com/en-us/visualstudio/msbuild/solution-filters?view=vs-2019) to only build required projects - An explicit [solution file that excludes projects](https://docs.microsoft.com/en-us/visualstudio/ide/how-to-exclude-projects-from-a-build?view=vs-2022) - example from the Open Source project: [Identity Server](https://github.com/DuendeSoftware/IdentityServer/) From d8d920efc225c9f619018aef8aba1b7743efa2d3 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 13 Feb 2023 17:55:56 -0500 Subject: [PATCH 080/128] Update compiled-languages-java.md --- troubleshooting/codeql-builds/compiled-languages-java.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-java.md b/troubleshooting/codeql-builds/compiled-languages-java.md index 440b589..965b146 100644 --- a/troubleshooting/codeql-builds/compiled-languages-java.md +++ b/troubleshooting/codeql-builds/compiled-languages-java.md @@ -42,10 +42,15 @@ Ensure you are compiling your java application using CodeQL tracing on a support ## Fatal error compiling: error: invalid target release: \## -Specify your [desired java version via the setup-java action](https://github.com/actions/setup-java#supported-version-syntax) +Alternative error: +``` +> error: invalid source release: +``` + +Resolution here is to specify your [desired java version via the setup-java action](https://github.com/actions/setup-java#supported-version-syntax) ```yml uses: actions/setup-java@v3 with: java-version: 17 distribution: 'microsoft' -``` \ No newline at end of file +``` From e42a961c52738f125ccc32c0cd083e547ee20df1 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Sun, 19 Feb 2023 16:49:24 -0500 Subject: [PATCH 081/128] Update troubleshooting.md --- .../sarif-upload/troubleshooting.md | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/troubleshooting/sarif-upload/troubleshooting.md b/troubleshooting/sarif-upload/troubleshooting.md index e8d5eae..173ce3a 100644 --- a/troubleshooting/sarif-upload/troubleshooting.md +++ b/troubleshooting/sarif-upload/troubleshooting.md @@ -1,4 +1,5 @@ -### SARIF Upload Errors +## SARIF Upload Errors +* Test environment - GHES 3.2.1 + CodeQL CLI 2.7.2 :gift: wrong ref: ``` @@ -43,11 +44,36 @@ codeql github upload-results --github-url=https://cmboling-0bd0debab4ff16db0.ghe A fatal error occurred: Error uploading SARIF to 'https://cmboling-0bd0debab4ff16db0.ghe-test.ninja/api/v3/repos/santa-foss/fluffy-potato/code-scanning/sarifs' from '/Users/cmboling/Desktop/jubilant-octo-pancake/results.sarif'. REASON: HTTP/1.1 403 Forbidden:::{"message":"Advanced Security must be enabled for this repository to use code scanning.","documentation_url":"https://docs.github.com/enterprise/3.2/rest/reference/code-scanning#upload-a-sarif-file"} ``` -### Test environments -- GHES 3.2.1 + CodeQL CLI 2.7.2 +## SARIF Parsing Errors +### Code Scanning could not process the submitted SARIF file: rejecting SARIF, as there are more runs than allowed (123 > 15) +The GitHub api for accepting SARIF uploads has a limiter to prevent that number from being greater than specified (>15) for each upload. -### Tools to rewrite SARIF +See limits for various thresholds on the [REST API documentation](https://docs.github.com/en/rest/code-scanning?apiVersion=2022-11-28#upload-an-analysis-as-sarif-data) +* Runs per file +* Results per run +* Rules per run +* Tool extensions per run +* Thread Flow Locations per result +* Location per result +* Tags per rule + +### A fatal error occurred: SARIF file is too large. The GitHub code scanning API accepts a max file size of 2000MB. This file is xxxxMB. File: "xyz.sarif" +- aleternatively - `failed decompressing file from the path: "upload /xyz.sarif.gz": maximum SARIF size exceeded` + +First, review recommendedations per language to reduce the amount of code being scanned (e.g. removing test or demo code from the scan in an attempt to remove unwanted detections from SARIF). A detailed analysis of the SARIF file may indicate a massive number of a single rule, in this case excluding a specific rule from the analysis would be the best solution. Alternatively, use a tool like [filter-sarif action](https://github.com/advanced-security/filter-sarif) to rewrite the SARIF file to exclude specific detections via an exclusion pattern. + +If there are many deep code paths highlighted in the SARIF, use `--max-path=0` (or 1) passed into the analyze step or `database analyze` cli command to get rid of the dataflow paths and reduce the SARIF size that way (NOTE this will impact all rules). + +```yml + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + env: + CODEQL_ACTION_EXTRA_OPTIONS: '{"database":{"interpret-results":["--max-paths", 1]}}' +``` + +## Tools to rewrite SARIF - `jq` - [Microsoft's SARIF tool](https://github.com/microsoft/sarif-sdk/blob/main/docs/multitool-usage.md) - [Dr. House's SARIF CLI](https://github.com/hohn/sarif-cli) +- [advanced-security/filter-sarif action](https://github.com/advanced-security/filter-sarif) From 944cb5e081b5fff654d8c0bc2a53ae12b77eaa0b Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Sun, 19 Feb 2023 16:50:15 -0500 Subject: [PATCH 082/128] Update troubleshooting.md --- troubleshooting/sarif-upload/troubleshooting.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/troubleshooting/sarif-upload/troubleshooting.md b/troubleshooting/sarif-upload/troubleshooting.md index 173ce3a..7712188 100644 --- a/troubleshooting/sarif-upload/troubleshooting.md +++ b/troubleshooting/sarif-upload/troubleshooting.md @@ -66,10 +66,10 @@ First, review recommendedations per language to reduce the amount of code being If there are many deep code paths highlighted in the SARIF, use `--max-path=0` (or 1) passed into the analyze step or `database analyze` cli command to get rid of the dataflow paths and reduce the SARIF size that way (NOTE this will impact all rules). ```yml - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - env: - CODEQL_ACTION_EXTRA_OPTIONS: '{"database":{"interpret-results":["--max-paths", 1]}}' +- name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + env: + CODEQL_ACTION_EXTRA_OPTIONS: '{"database":{"interpret-results":["--max-paths", 1]}}' ``` ## Tools to rewrite SARIF From b970ecff5d84ff333821c875695c0aa066b72851 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Sun, 19 Feb 2023 16:55:37 -0500 Subject: [PATCH 083/128] Update troubleshooting.md --- troubleshooting/sarif-upload/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/sarif-upload/troubleshooting.md b/troubleshooting/sarif-upload/troubleshooting.md index 7712188..fc2333f 100644 --- a/troubleshooting/sarif-upload/troubleshooting.md +++ b/troubleshooting/sarif-upload/troubleshooting.md @@ -61,7 +61,7 @@ See limits for various thresholds on the [REST API documentation](https://docs.g ### A fatal error occurred: SARIF file is too large. The GitHub code scanning API accepts a max file size of 2000MB. This file is xxxxMB. File: "xyz.sarif" - aleternatively - `failed decompressing file from the path: "upload /xyz.sarif.gz": maximum SARIF size exceeded` -First, review recommendedations per language to reduce the amount of code being scanned (e.g. removing test or demo code from the scan in an attempt to remove unwanted detections from SARIF). A detailed analysis of the SARIF file may indicate a massive number of a single rule, in this case excluding a specific rule from the analysis would be the best solution. Alternatively, use a tool like [filter-sarif action](https://github.com/advanced-security/filter-sarif) to rewrite the SARIF file to exclude specific detections via an exclusion pattern. +First, review recommendedations per language to reduce the amount of code being scanned (e.g. removing test or demo code from the scan in an attempt to remove unwanted detections from SARIF). A detailed analysis of the SARIF file may indicate a massive number of a single rule, in this case [excluding a specific rule from the analysis](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/customizing-code-scanning#excluding-specific-queries-from-analysis) would be the best solution. Alternatively, use a tool like [filter-sarif action](https://github.com/advanced-security/filter-sarif) to rewrite the SARIF file to exclude specific detections via an exclusion pattern. If there are many deep code paths highlighted in the SARIF, use `--max-path=0` (or 1) passed into the analyze step or `database analyze` cli command to get rid of the dataflow paths and reduce the SARIF size that way (NOTE this will impact all rules). From be43abe4be061e72891bbf096d3c7f3a2987ef24 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:55:53 -0500 Subject: [PATCH 084/128] Update compiled-languages.md --- troubleshooting/codeql-builds/compiled-languages.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index a18c13d..388c925 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -48,9 +48,9 @@ alternatively we can further define limits - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: - # Increase Values seen in logs: + # Increase Values seen in logs: #2022-06-01T19:37:19.0200037Z CODEQL_RAM: 119741 - #2022-06-01T19:37:19.0200307Z CODEQL_THREADS: 32 + #2022-06-01T19:37:19.0200307Z CODEQL_THREADS: 32 ram: 64000 threads: 16 ``` @@ -65,4 +65,4 @@ Helpful Articles to understand how to review, troubleshoot, and debug logs: - [Exit Codes](https://codeql.github.com/docs/codeql-cli/exit-codes/) ## Optimizaitons -- CodeQL Docs - [The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow) \ No newline at end of file +- CodeQL Docs - [The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow) From 5e8e41f429699c69cdb8be8d64b9bfb33e68f4e7 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:56:13 -0500 Subject: [PATCH 085/128] Update compiled-languages.md --- troubleshooting/codeql-builds/compiled-languages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index 388c925..46ee50c 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -48,7 +48,7 @@ alternatively we can further define limits - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: - # Increase Values seen in logs: + # Increase Values seen in logs: #2022-06-01T19:37:19.0200037Z CODEQL_RAM: 119741 #2022-06-01T19:37:19.0200307Z CODEQL_THREADS: 32 ram: 64000 From ad249276d9e2a0a746130544256c03fbedec375a Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:57:12 -0500 Subject: [PATCH 086/128] Update compiled-languages.md --- troubleshooting/codeql-builds/compiled-languages.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index 46ee50c..dc5885a 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -48,9 +48,9 @@ alternatively we can further define limits - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: - # Increase Values seen in logs: + # Increase Values seen in logs: #2022-06-01T19:37:19.0200037Z CODEQL_RAM: 119741 - #2022-06-01T19:37:19.0200307Z CODEQL_THREADS: 32 + #2022-06-01T19:37:19.0200307Z CODEQL_THREADS: 32 ram: 64000 threads: 16 ``` From d2d873730473e741d8af85e7df7268465ab09f7c Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:34:36 -0500 Subject: [PATCH 087/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index ca8789f..4e97406 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -55,7 +55,12 @@ Using `dotnet` is best documented at: https://docs.github.com/en/actions/automat #### NuGet Error NU1301 This can indicate your custom package server is not configured which may fail the `dotnet restore` command. For private package servers, the follwing guidance shows how to add package sources: [Setting up authentication for nuget feeds](https://github.com/actions/setup-dotnet#setting-up-authentication-for-nuget-feeds) -### .NET Framework Manual Build Steps on Windows Runners +### .NET Framework + +#### NuGet Authentication +Utilize the [nuget/setup-nuget](https://github.com/nuget/setup-nuget#basic) action to pass package key/source to nuget exe. + +#### Manual Build Steps on Windows Runners NOTE: if you require windows OS to build, ensure you are using a windows runner. Example using `windows-latest`: From 9c3809212263f140b22d9f16a913a4f0eef49e53 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 13 Mar 2023 10:56:59 -0400 Subject: [PATCH 088/128] Update compiled-languages-java.md --- troubleshooting/codeql-builds/compiled-languages-java.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-java.md b/troubleshooting/codeql-builds/compiled-languages-java.md index 965b146..88269ce 100644 --- a/troubleshooting/codeql-builds/compiled-languages-java.md +++ b/troubleshooting/codeql-builds/compiled-languages-java.md @@ -49,8 +49,8 @@ Alternative error: Resolution here is to specify your [desired java version via the setup-java action](https://github.com/actions/setup-java#supported-version-syntax) ```yml - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: 'microsoft' +- uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'microsoft' ``` From 41fa1775fc264ae73a079d58cf53670665146258 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Thu, 30 Mar 2023 18:42:24 -0400 Subject: [PATCH 089/128] Update compiled-languages-csharp.md --- .../codeql-builds/compiled-languages-csharp.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 4e97406..44e15fb 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -123,6 +123,18 @@ Running low on disk using the default Actions runner? Try a few of these workaro - See also: [Vertical Scaling](#vertical-scaling---throw-hardware-at-the-software-problem) +## MvcBuildViews target failures + +This can manifest through a variety of errors +- `error ASPPARSE` +- `[error]C:\Windows\Microsoft.NET\Framework\v4.0.30319\Config\web.config(113,0): Error ASPCONFIG: Could not load type` +- `Error ASPCONFIG: It is an error to use a section registered as allowDefinition='MachineToApplication' beyond application level.` + +The CodeQL compiler tracer used for `csharp` will auto inject the /p:MvcBuildViews=true flag. This pre-compilation of Views gives us the ability to extract the generated code from those files, leading to (potentially) better error reporting and location information if a query does flag an issue. The lack of view information passing through CodeQL to the compiler will lead to an incomplete database, where important dataflow sources/sinks/taint-steps are not included in the analysis. + +The recommendation here is to ensure that passing /p:MvcBuildViews=true to your CI build will compile even outside of CodeQL. Having a developer reivew this on their local machine is the best scenario. The MVC full framework steps are listed [here](https://learn.microsoft.com/en-us/archive/blogs/jimlamb/turn-on-compile-time-view-checking-for-asp-net-mvc-projects-in-tfs-build-2010). There are a few different reasons why this can cause your project to fail compilation. If you have bin/obj files checked into source then this could be a likely culprit: https://gunnarpeipman.com/aspnet-mvc-allowdefinition-machinetoapplication/. You will find [various permutations of this recommendation](https://stackoverflow.com/questions/12778088/allowdefinition-machinetoapplication-error-setting-mvcbuildviewstrue-mvcbui) out there! + +For `Error ASPCONFIG: It is an error to use a section registered as allowDefinition='MachineToApplication' beyond application level.`, change the locations of the obj and publish folder to not be located under the project folder of the website. # Speed up C# Analysis From 0d723df93e1e5df31f5dc8ffb242f4fbe85ad615 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 31 Mar 2023 09:59:44 -0400 Subject: [PATCH 090/128] Update compiled-languages-java.md --- troubleshooting/codeql-builds/compiled-languages-java.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-java.md b/troubleshooting/codeql-builds/compiled-languages-java.md index 88269ce..407a405 100644 --- a/troubleshooting/codeql-builds/compiled-languages-java.md +++ b/troubleshooting/codeql-builds/compiled-languages-java.md @@ -1,10 +1,10 @@ # Private Package Registries -## The codeql for java is failing when it tries to do mvn command and tries to access a artifactory repo where our pom.xml are stored. - -Assuming the given package registry instance is publicly accessible: +## The autobuild for java is failing when running Maven build command and a private package registry is needed - `status: 401 Unauthorized ` +- ex: artifactory where our pom.xml dependencies are stored +Assuming the given package registry instance is publicly accessible and needs credentials: Option 1 - Pass credentials via environment variable from Actions secrets and configure Maven settings to utilize those credentials (see sample [here](https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#yaml-example)) From bdb7c6dc58ba74b2eed3c66a22d4374723af08bd Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 31 Mar 2023 10:02:25 -0400 Subject: [PATCH 091/128] Update compiled-languages-java.md --- troubleshooting/codeql-builds/compiled-languages-java.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/troubleshooting/codeql-builds/compiled-languages-java.md b/troubleshooting/codeql-builds/compiled-languages-java.md index 407a405..de06928 100644 --- a/troubleshooting/codeql-builds/compiled-languages-java.md +++ b/troubleshooting/codeql-builds/compiled-languages-java.md @@ -34,6 +34,8 @@ Option 2 - Use the [maven-settings-action](https://github.com/s4u/maven-settings servers: '[{"id": "central", "username": "${{ secrets.MAVEN_USERNAME }}", "password": "${{ secrets.MAVEN_CENTRAL_TOKEN }}"}]' ``` +See also: [401 due to private package server configuration](compiled-languages.md#401-due-to-private-package-server-configuration) + # Build Failures ## java.lang.IllegalArgumentException: Unsupported class file major version ## From 56a9a24a4aabd08b486e7e1d320f7249cb4f815f Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 31 Mar 2023 10:05:34 -0400 Subject: [PATCH 092/128] Update compiled-languages.md --- troubleshooting/codeql-builds/compiled-languages.md | 1 + 1 file changed, 1 insertion(+) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index dc5885a..ccd66a2 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -24,6 +24,7 @@ See [language specific guidance](#language-specific-guidance) for common resolut Ensure network access from GitHub runners to your private registry is open - For IP Whitelisting, consider using [Larger Runners with Static IP](https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners#networking-for-larger-runners) - See Also: [Connecting Actions to a private network](https://docs.github.com/en/actions/using-github-hosted-runners/connecting-to-a-private-network) + - Alertnatively, consider a self-hosted actions runner that will execute within your existing private network. See ["Hosting your own runners"](https://docs.github.com/en/actions/hosting-your-own-runners) See [language specific guidance](#language-specific-guidance) for authentication options to popular package mangers From ae7e0a58919a53c759f9f5f289eaf6ceb0b30c77 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 31 Mar 2023 10:05:53 -0400 Subject: [PATCH 093/128] Update compiled-languages.md --- troubleshooting/codeql-builds/compiled-languages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index ccd66a2..932acfb 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -24,7 +24,7 @@ See [language specific guidance](#language-specific-guidance) for common resolut Ensure network access from GitHub runners to your private registry is open - For IP Whitelisting, consider using [Larger Runners with Static IP](https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners#networking-for-larger-runners) - See Also: [Connecting Actions to a private network](https://docs.github.com/en/actions/using-github-hosted-runners/connecting-to-a-private-network) - - Alertnatively, consider a self-hosted actions runner that will execute within your existing private network. See ["Hosting your own runners"](https://docs.github.com/en/actions/hosting-your-own-runners) + - Alternatively, consider a self-hosted actions runner that will execute within your existing private network. See ["Hosting your own runners"](https://docs.github.com/en/actions/hosting-your-own-runners) See [language specific guidance](#language-specific-guidance) for authentication options to popular package mangers From 096ad60065bb4c3068a5f8f1ff99bf8f7a7f1348 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 31 Mar 2023 10:15:38 -0400 Subject: [PATCH 094/128] Update advanced-security-reporting.md --- reporting/advanced-security-reporting.md | 1 + 1 file changed, 1 insertion(+) diff --git a/reporting/advanced-security-reporting.md b/reporting/advanced-security-reporting.md index 660359f..2921234 100644 --- a/reporting/advanced-security-reporting.md +++ b/reporting/advanced-security-reporting.md @@ -18,4 +18,5 @@ - [ ] https://github.com/ThibaudLopez/GHAS - SIEM integrations - [ ] https://github.blog/2022-10-13-introducing-github-advanced-security-siem-integrations-for-security-professionals/ + - [ ] https://github.blog/2023-03-10-introducing-github-vulnerability-management-integrations-for-security-professionals/ - [ ] https://resources.github.com/security/integrating-github-advanced-security-with-third-party-platforms/ From 4c2082d34f1baa258e3e30a4a526b97a3bd7b444 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 31 Mar 2023 18:01:27 -0400 Subject: [PATCH 095/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 44e15fb..a4c70f8 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -136,6 +136,8 @@ The recommendation here is to ensure that passing /p:MvcBuildViews=true to your For `Error ASPCONFIG: It is an error to use a section registered as allowDefinition='MachineToApplication' beyond application level.`, change the locations of the obj and publish folder to not be located under the project folder of the website. +For `Error ASPCONFIG: Could not load type 'X.Y.Z'`, ensure that you do not have excluded `.cshtml`, `.ashx`, `.ashx.cs`, `.aspx` or `.aspx.cs` files on disk in existing `Views` folders or the Root folder of your project! You can show hidden files in your solution view to hunt these down and remove from these folders. MvcBuildViews does not observe the file include from the csproj when compiling the application. You may have to hunt these down one by one, so adding `true` to your local .csproj may help you get this done on your local machine with Visual Studio. The `Error List` view in Visual Studio will have a column that shows you the actual File name you need to delete. + # Speed up C# Analysis Start here: [CodeQL Docs - The build takes too long](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/troubleshooting-the-codeql-workflow#the-build-takes-too-long). From 720329c341fb0b9501438f29fd5b594204a12415 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 3 Apr 2023 13:16:55 -0400 Subject: [PATCH 096/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 1 + 1 file changed, 1 insertion(+) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index a4c70f8..50b2460 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -129,6 +129,7 @@ This can manifest through a variety of errors - `error ASPPARSE` - `[error]C:\Windows\Microsoft.NET\Framework\v4.0.30319\Config\web.config(113,0): Error ASPCONFIG: Could not load type` - `Error ASPCONFIG: It is an error to use a section registered as allowDefinition='MachineToApplication' beyond application level.` +- `(AfterBuildCompiler target) -> D:\a\Orchard\Orchard\src\Orchard.Web\Modules\Orchard.Glimpse\web.config(38): error ASPCONFIG: Could not load file or assembly 'System.Web.Mvc, Version=5.2.3, Culture=neutral, PublicKeyToken=31bf3856ad364e35' or one of its dependencies. The located assembly's manifest definition does not match the assembly reference. (Exception from HRESULT: 0x80131040)` The CodeQL compiler tracer used for `csharp` will auto inject the /p:MvcBuildViews=true flag. This pre-compilation of Views gives us the ability to extract the generated code from those files, leading to (potentially) better error reporting and location information if a query does flag an issue. The lack of view information passing through CodeQL to the compiler will lead to an incomplete database, where important dataflow sources/sinks/taint-steps are not included in the analysis. From 5cfd0156a640c446185a78e11cd5abcf2f26719b Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 3 Apr 2023 13:18:01 -0400 Subject: [PATCH 097/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 50b2460..678d703 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -133,9 +133,9 @@ This can manifest through a variety of errors The CodeQL compiler tracer used for `csharp` will auto inject the /p:MvcBuildViews=true flag. This pre-compilation of Views gives us the ability to extract the generated code from those files, leading to (potentially) better error reporting and location information if a query does flag an issue. The lack of view information passing through CodeQL to the compiler will lead to an incomplete database, where important dataflow sources/sinks/taint-steps are not included in the analysis. -The recommendation here is to ensure that passing /p:MvcBuildViews=true to your CI build will compile even outside of CodeQL. Having a developer reivew this on their local machine is the best scenario. The MVC full framework steps are listed [here](https://learn.microsoft.com/en-us/archive/blogs/jimlamb/turn-on-compile-time-view-checking-for-asp-net-mvc-projects-in-tfs-build-2010). There are a few different reasons why this can cause your project to fail compilation. If you have bin/obj files checked into source then this could be a likely culprit: https://gunnarpeipman.com/aspnet-mvc-allowdefinition-machinetoapplication/. You will find [various permutations of this recommendation](https://stackoverflow.com/questions/12778088/allowdefinition-machinetoapplication-error-setting-mvcbuildviewstrue-mvcbui) out there! +The recommendation here is to ensure that passing /p:MvcBuildViews=true to your CI build will compile even outside of CodeQL. Having a developer reivew this on their local machine is the best scenario. The MVC full framework steps are listed [here](https://learn.microsoft.com/en-us/archive/blogs/jimlamb/turn-on-compile-time-view-checking-for-asp-net-mvc-projects-in-tfs-build-2010). There are a few different reasons why this can cause your project to fail compilation. -For `Error ASPCONFIG: It is an error to use a section registered as allowDefinition='MachineToApplication' beyond application level.`, change the locations of the obj and publish folder to not be located under the project folder of the website. +For `Error ASPCONFIG: It is an error to use a section registered as allowDefinition='MachineToApplication' beyond application level.`, change the locations of the obj and publish folder to not be located under the project folder of the website. If you have bin/obj files checked into source then this could be a likely culprit: https://gunnarpeipman.com/aspnet-mvc-allowdefinition-machinetoapplication/. You will find [various permutations of this recommendation](https://stackoverflow.com/questions/12778088/allowdefinition-machinetoapplication-error-setting-mvcbuildviewstrue-mvcbui) out there! For `Error ASPCONFIG: Could not load type 'X.Y.Z'`, ensure that you do not have excluded `.cshtml`, `.ashx`, `.ashx.cs`, `.aspx` or `.aspx.cs` files on disk in existing `Views` folders or the Root folder of your project! You can show hidden files in your solution view to hunt these down and remove from these folders. MvcBuildViews does not observe the file include from the csproj when compiling the application. You may have to hunt these down one by one, so adding `true` to your local .csproj may help you get this done on your local machine with Visual Studio. The `Error List` view in Visual Studio will have a column that shows you the actual File name you need to delete. From 3e720ef0376b56d3857a1639c8b9b96ee462d302 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 3 Apr 2023 13:20:16 -0400 Subject: [PATCH 098/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 678d703..ef375e5 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -133,7 +133,7 @@ This can manifest through a variety of errors The CodeQL compiler tracer used for `csharp` will auto inject the /p:MvcBuildViews=true flag. This pre-compilation of Views gives us the ability to extract the generated code from those files, leading to (potentially) better error reporting and location information if a query does flag an issue. The lack of view information passing through CodeQL to the compiler will lead to an incomplete database, where important dataflow sources/sinks/taint-steps are not included in the analysis. -The recommendation here is to ensure that passing /p:MvcBuildViews=true to your CI build will compile even outside of CodeQL. Having a developer reivew this on their local machine is the best scenario. The MVC full framework steps are listed [here](https://learn.microsoft.com/en-us/archive/blogs/jimlamb/turn-on-compile-time-view-checking-for-asp-net-mvc-projects-in-tfs-build-2010). There are a few different reasons why this can cause your project to fail compilation. +The recommendation here is to ensure that passing /p:MvcBuildViews=true to your CI build will compile even outside of CodeQL. Having a developer reivew this on their local machine is the best scenario. This can be on done on the specific web project by adding `true` to the local .csproj ( you will often find this defaulted to false). The MVC full framework steps are listed [here](https://learn.microsoft.com/en-us/archive/blogs/jimlamb/turn-on-compile-time-view-checking-for-asp-net-mvc-projects-in-tfs-build-2010). There are a few different reasons why this can cause your project to fail compilation. For `Error ASPCONFIG: It is an error to use a section registered as allowDefinition='MachineToApplication' beyond application level.`, change the locations of the obj and publish folder to not be located under the project folder of the website. If you have bin/obj files checked into source then this could be a likely culprit: https://gunnarpeipman.com/aspnet-mvc-allowdefinition-machinetoapplication/. You will find [various permutations of this recommendation](https://stackoverflow.com/questions/12778088/allowdefinition-machinetoapplication-error-setting-mvcbuildviewstrue-mvcbui) out there! From e0bb3f57a8af7656cfec70dfa1ae523559bf17b2 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Mon, 1 May 2023 15:42:24 -0400 Subject: [PATCH 099/128] add actions/setup-java to generate settings.xml --- .../codeql-builds/compiled-languages-java.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-java.md b/troubleshooting/codeql-builds/compiled-languages-java.md index de06928..6f141ac 100644 --- a/troubleshooting/codeql-builds/compiled-languages-java.md +++ b/troubleshooting/codeql-builds/compiled-languages-java.md @@ -24,7 +24,20 @@ ex `settings.xml` ``` -Option 2 - Use the [maven-settings-action](https://github.com/s4u/maven-settings-action) to dynamically create/overrite a `settings.xml` that contains the credentials for your specified package manager. +Option 2 - Use the GitHub https://github.com/actions/setup-java#maven-options action to generate maven's settings.xml on the fly and pass the values to Apache Maven GPG Plugin as well as Apache Maven Toolchains. + +```yml + - name: Set up Apache Maven Central + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + server-id: maven # Value of the distributionManagement/repository/id field of the pom.xml + server-username: MAVEN_USERNAME # env variable for username in deploy + server-password: MAVEN_CENTRAL_TOKEN # env variable for token in deploy + ``` + +Option 3 - Use the [maven-settings-action](https://github.com/s4u/maven-settings-action) to dynamically create/overrite a `settings.xml` that contains the credentials for your specified package manager. ```yml - if: matrix.language == 'java' From 1ad7098ea608a10778cd71645f1585ac8c0cc5ef Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Thu, 11 May 2023 14:09:43 -0400 Subject: [PATCH 100/128] Add C# Troubleshooting - Nuget Packages Auth --- .../codeql-builds/compiled-languages-csharp.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index ef375e5..a0efdc6 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -55,6 +55,16 @@ Using `dotnet` is best documented at: https://docs.github.com/en/actions/automat #### NuGet Error NU1301 This can indicate your custom package server is not configured which may fail the `dotnet restore` command. For private package servers, the follwing guidance shows how to add package sources: [Setting up authentication for nuget feeds](https://github.com/actions/setup-dotnet#setting-up-authentication-for-nuget-feeds) +#### NuGet.targets(132,5): warning : Your request could not be authenticated by the GitHub Packages service. Please ensure your access token is valid and has the appropriate scopes configured. + +Consider adding auth for your GitHub Packages hosted NuGet feed using the nuget CLI tooling. Add this before the `autobuild` / custom build steps in your workflow. + +```yml + - name: add nuget auth + run: dotnet nuget add source https://nuget.pkg.github.com//index.json -n "GitHub" -u USERNAME -p "${{ secrets.GH_PACKAGES_READ_ONLY }}" --store-password-in-clear-text + ``` + + ### .NET Framework #### NuGet Authentication From 7a5a2bb41f236ef2ae84a86736402bae3677f460 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Fri, 19 May 2023 10:56:42 -0400 Subject: [PATCH 101/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index a0efdc6..87ec72a 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -70,6 +70,12 @@ Consider adding auth for your GitHub Packages hosted NuGet feed using the nuget #### NuGet Authentication Utilize the [nuget/setup-nuget](https://github.com/nuget/setup-nuget#basic) action to pass package key/source to nuget exe. +```yml +- uses: nuget/setup-nuget@v1 + with: + nuget-api-key: ${{ secrets.NuGetAPIKey }} +``` + #### Manual Build Steps on Windows Runners NOTE: if you require windows OS to build, ensure you are using a windows runner. From 99744bac30e7ed0280e9012837ff936bf780e1c2 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Tue, 30 May 2023 18:24:54 -0400 Subject: [PATCH 102/128] Update compiled-languages-csharp.md --- .../codeql-builds/compiled-languages-csharp.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 87ec72a..c877748 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -57,14 +57,22 @@ This can indicate your custom package server is not configured which may fail th #### NuGet.targets(132,5): warning : Your request could not be authenticated by the GitHub Packages service. Please ensure your access token is valid and has the appropriate scopes configured. -Consider adding auth for your GitHub Packages hosted NuGet feed using the nuget CLI tooling. Add this before the `autobuild` / custom build steps in your workflow. +The actions/setup-dotnet action supports [setting up authentication for nuget feeds](https://github.com/actions/setup-dotnet#setting-up-authentication-for-nuget-feeds). Add this before the `autobuild` / custom build steps in your workflow: +```yml +- uses: actions/setup-dotnet@v3 + with: + source-url: https://nuget.pkg.github.com//index.json + env: + NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} +``` + +Alternatively, consider adding auth for your GitHub Packages hosted NuGet feed using the nuget CLI tooling. ```yml - name: add nuget auth run: dotnet nuget add source https://nuget.pkg.github.com//index.json -n "GitHub" -u USERNAME -p "${{ secrets.GH_PACKAGES_READ_ONLY }}" --store-password-in-clear-text ``` - ### .NET Framework #### NuGet Authentication From c65ae7dc740d1f5ca5a431471ca1479594048977 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Tue, 30 May 2023 18:25:12 -0400 Subject: [PATCH 103/128] Update compiled-languages-csharp.md --- troubleshooting/codeql-builds/compiled-languages-csharp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index c877748..36e3fa0 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -57,7 +57,7 @@ This can indicate your custom package server is not configured which may fail th #### NuGet.targets(132,5): warning : Your request could not be authenticated by the GitHub Packages service. Please ensure your access token is valid and has the appropriate scopes configured. -The actions/setup-dotnet action supports [setting up authentication for nuget feeds](https://github.com/actions/setup-dotnet#setting-up-authentication-for-nuget-feeds). Add this before the `autobuild` / custom build steps in your workflow: +The `actions/setup-dotnet` action supports [setting up authentication for nuget feeds](https://github.com/actions/setup-dotnet#setting-up-authentication-for-nuget-feeds). Add this before the `autobuild` / custom build steps in your workflow: ```yml - uses: actions/setup-dotnet@v3 with: From 4202457e12c271c02cd49cfef8c123785b44ba60 Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Thu, 15 Jun 2023 15:07:27 -0700 Subject: [PATCH 104/128] Update README.md --- README.md | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e759fc..2363192 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,45 @@ # Advanced Security Material -A place for resources to help you understand and use GitHub Advanced Security +A place for resources to help you understand and use GitHub Advanced Security (GHAS). Browse the directories in this repository for resources and documentation. To help you get started with GHAS, we've provided some introductory documentation in this file. + +## Get started +The following list of links are great resources to get you started on learning how to use, deploy, and manage GitHub Advanced Security in your environment. + +New to GitHub Advanced Security? Start with [GitHub security features](https://docs.github.com/en/enterprise-cloud@latest/code-security/getting-started/github-security-features) :+1: + +## Code Scanning +- [About GitHub Code Scanning](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning) +- [Configuring Code Scanning](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning) +- [Integrating other tools with GHAS](https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning) + +## CodeQL +- [Meet CodeQL](https://codeql.github.com/) +- [CodeQL Documentation](https://codeql.github.com/docs/) +- [CWE Query Mapping Documentation](https://codeql.github.com/codeql-query-help/codeql-cwe-coverage) +- [Running additional queries](https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#running-additional-queries) +- [CodeQL CLI Docs](https://codeql.github.com/docs/codeql-cli/getting-started-with-the-codeql-cli) +- [Running CodeQL in your CI System](https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/running-codeql-code-scanning-in-your-ci-system) + +## Secret Scanning +- [About Secret Scanning](https://docs.github.com/en/code-security/secret-scanning/about-secret-scanning) +- [Supported secret patterns](https://docs.github.com/en/code-security/secret-scanning/secret-scanning-patterns#supported-secrets-for-partner-patterns) +- [Defining custom secret patterns](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning) + +## Supply Chain Security (Dependabot) +- [About](https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/about-supply-chain-security) +- [Dependency Graph](https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/about-the-dependency-graph) +- [Dependabot Alerts](https://docs.github.com/en/enterprise-cloud@latest/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) +- [Dependabot Security Updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates) +- [GitHub Advisory Database](https://github.com/advisories) + +## Security Overview +- [About Security Overview](https://docs.github.com/en/code-security/security-overview/about-the-security-overview) +- [Managing alerts in your repository](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository) + +## Other Resources +- [SARIF Tutorials](https://github.com/microsoft/sarif-tutorials) +- [GitHub Advanced Security Learning Path](https://docs.microsoft.com/en-us/users/githubtraining/collections/rqymc6yw8q5rey) +- [Scaling GHAS in Your Organization](https://resources.github.com/downloads/Whitepaper-Scaling-GHAS-in-an-Enterprise.pdf) +- [The Complete Guide to Developer-first Security](https://resources.github.com/downloads/GitHubAdvanced%20SecurityEbook.pdf) +- [GitHub Checkout - Code Scanning (video)](https://www.youtube.com/watch?v=z0wvGf3O69E) +- [GitHub Checkout - Secret Scanning (video)](https://www.youtube.com/watch?v=aoL7pDrXt74) +- [GitHub Checkout - Viewing and Managing your Dependencies (video)](https://www.youtube.com/watch?v=gNd_TGdZ1xc) From 28190e9e50d0c4d17c5e98a6297791ead132a58a Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Thu, 15 Jun 2023 15:07:50 -0700 Subject: [PATCH 105/128] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2363192..ce61d4a 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ The following list of links are great resources to get you started on learning h New to GitHub Advanced Security? Start with [GitHub security features](https://docs.github.com/en/enterprise-cloud@latest/code-security/getting-started/github-security-features) :+1: -## Code Scanning +### Code Scanning - [About GitHub Code Scanning](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning) - [Configuring Code Scanning](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning) - [Integrating other tools with GHAS](https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning) -## CodeQL +### CodeQL - [Meet CodeQL](https://codeql.github.com/) - [CodeQL Documentation](https://codeql.github.com/docs/) - [CWE Query Mapping Documentation](https://codeql.github.com/codeql-query-help/codeql-cwe-coverage) @@ -19,23 +19,23 @@ New to GitHub Advanced Security? Start with [GitHub security features](https:// - [CodeQL CLI Docs](https://codeql.github.com/docs/codeql-cli/getting-started-with-the-codeql-cli) - [Running CodeQL in your CI System](https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/running-codeql-code-scanning-in-your-ci-system) -## Secret Scanning +### Secret Scanning - [About Secret Scanning](https://docs.github.com/en/code-security/secret-scanning/about-secret-scanning) - [Supported secret patterns](https://docs.github.com/en/code-security/secret-scanning/secret-scanning-patterns#supported-secrets-for-partner-patterns) - [Defining custom secret patterns](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning) -## Supply Chain Security (Dependabot) +### Supply Chain Security (Dependabot) - [About](https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/about-supply-chain-security) - [Dependency Graph](https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/about-the-dependency-graph) - [Dependabot Alerts](https://docs.github.com/en/enterprise-cloud@latest/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) - [Dependabot Security Updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates) - [GitHub Advisory Database](https://github.com/advisories) -## Security Overview +### Security Overview - [About Security Overview](https://docs.github.com/en/code-security/security-overview/about-the-security-overview) - [Managing alerts in your repository](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository) -## Other Resources +### Other Resources - [SARIF Tutorials](https://github.com/microsoft/sarif-tutorials) - [GitHub Advanced Security Learning Path](https://docs.microsoft.com/en-us/users/githubtraining/collections/rqymc6yw8q5rey) - [Scaling GHAS in Your Organization](https://resources.github.com/downloads/Whitepaper-Scaling-GHAS-in-an-Enterprise.pdf) From 258444c29b239819455f0efdb6315d7cc1d86b55 Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Thu, 15 Jun 2023 15:08:20 -0700 Subject: [PATCH 106/128] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce61d4a..3828786 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Advanced Security Material A place for resources to help you understand and use GitHub Advanced Security (GHAS). Browse the directories in this repository for resources and documentation. To help you get started with GHAS, we've provided some introductory documentation in this file. -## Get started +## Get started with GitHub Advanced Security The following list of links are great resources to get you started on learning how to use, deploy, and manage GitHub Advanced Security in your environment. New to GitHub Advanced Security? Start with [GitHub security features](https://docs.github.com/en/enterprise-cloud@latest/code-security/getting-started/github-security-features) :+1: From d59f698154b55c5bf4a1118b93837cf164ab6a12 Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Thu, 15 Jun 2023 16:44:22 -0700 Subject: [PATCH 107/128] Create GHAS-on-GHES-feature-matrix.md --- GHAS-on-GHES-feature-matrix.md | 140 +++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 GHAS-on-GHES-feature-matrix.md diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md new file mode 100644 index 0000000..fb0afbc --- /dev/null +++ b/GHAS-on-GHES-feature-matrix.md @@ -0,0 +1,140 @@ +# GitHub Advanced Security (GHAS) Feature Matrix + +This document helps answer the question "is this GHAS feature available in my version of GitHub Enterprise Server?". + +The following tables include notable feature releases for GitHub Advanced Security. Each row represents a feature. The columns in the row indicate the level of support for each **supported** Enterprise Server release. Are your repositories hosted on github.com? All of these features are already available for you :+1:. + +Each section of this document represents a different capability of the GitHub security features. Each row in the tables represent a different feature of GHAS. The columns indicate if that feature is available in each version of GitHub Enterprise Server. Cells with ☑️ indicate beta support. ✅ indicates full support. + +#### Contents +- [Secret scanning](#secret-scanning) +- [Code scanning](#code-scanning) +- [Supply-chain security](#supply-chain-security) +- [Security Overview](#security-overview) +- [Administration](#administration) + +## Release notes +|Version |3.4 |3.5 |3.6 |3.7 |3.8 |3.9| +|---------|-----|-----|-----|-----|-----|-----| +|Release date| Mar. 15 2022| May 31 2022 |Aug. 16 2022 |Nov. 8 2022 |Mar. 7 2023 |Jun. 8 2023 (rc1) | +|| [Release notes](https://docs.github.com/en/enterprise-server@3.4/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.5/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.6/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.7/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.8/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.9/admin/release-notes)| + +## Secret scanning +Secret scanning identifies plain text credentials inside your code repository. Learn more about secret scanning +* [Secret scanning documentation](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/about-secret-scanning) +* [Secret scanning API documentation](https://docs.github.com/en/enterprise-cloud@latest/rest/secret-scanning?apiVersion=2022-11-28) + +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| +|Partner pattern count|155|169|173|173|183|200| +|[User defined (custom) patterns](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)|✅|✅|✅|✅|✅|✅| +|[Enterprise level API for secret scanning](https://docs.github.com/en/enterprise-cloud@latest/rest/secret-scanning?apiVersion=2022-11-28#list-secret-scanning-alerts-for-an-enterprise)|✅|✅|✅|✅|✅|✅| +|[Secret scanning push protection](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/protecting-pushes-with-secret-scanning)||✅|✅|✅|✅|✅| +|[Dry runs for secret scanning push protection (repo level)](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||✅|✅|✅|✅|✅| +|[Secret scanning support for archived repos](https://github.blog/changelog/2022-02-16-secret-scanning-now-supports-archived-repositories/)||✅|✅|✅|✅|✅| +|[Custom pattern events in the audit log](https://github.blog/changelog/2022-04-06-secret-scanning-custom-pattern-events-now-in-the-audit-log/)||✅|✅|✅|✅|✅| +|[Push protection events in the audit log](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-security-settings-for-your-organization/audit-log-events-for-your-organization#secret_scanning_push_protection-category-actions)|||✅|✅|✅|✅| +|[Push protection in the web editor](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/protecting-pushes-with-secret-scanning#using-secret-scanning-as-a-push-protection-from-the-web-ui)|||✅|✅|✅|✅| +|[Enable secret scanning at the enterprise level](https://github.blog/changelog/2022-10-06-enable-secret-scanning-for-an-enterprise-with-one-click/)||||✅|✅|✅| +|[Dry runs for secret scanning custom patterns (org level)](https://github.blog/changelog/2022-02-11-secret-scanning-dry-runs-for-repository-level-custom-pattern/)||||✅|✅|✅| +|[Email notification for push protection bypass](https://github.blog/changelog/2022-07-27-secret-scanning-admins-now-receive-emails-when-contributors-bypass-a-push-protection-block/)||||✅|✅|✅| +|[Custom links in push protection notification](https://github.blog/changelog/2022-08-24-secret-scanning-admins-can-now-provide-a-link-to-display-when-a-push-is-blocked/)||||✅|✅|✅| +|[View secret scanning enablement status at the org-level via API](https://github.blog/changelog/2021-08-24-secret-scanning-org-level-rest-api/)||||✅|✅|✅| +|[Enable secret scanning at the enterprise level using the REST API](https://github.blog/changelog/2022-12-13-enable-secret-scanning-with-the-enterprise-level-rest-api/)|||||✅|✅| +|[Add comment when dismissing a secret scanning alert in UI or API](https://github.blog/changelog/2022-09-29-secret-scanning-alerts-now-have-a-timeline-and-users-can-add-a-comment-when-resolving/)|||||✅|✅| +|[Custom pattern creation at the enterprise level](https://docs.github.com/en/enterprise-server@3.9/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||||||✅| + + +## Code scanning +Code scanning is a feature that you use to analyze the code in a GitHub repository to find security vulnerabilities and coding errors. Any problems identified by the analysis are shown in GitHub. +* [Code scanning documentation](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning) +* [Code scanning API documentation](https://docs.github.com/en/rest/code-scanning?apiVersion=2022-11-28) + +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| +|CodeQL "toolcache" Installed Version|2.7.6|2.8.5|2.9.4|2.10.5|2.11.6| +|[Language support: Python, Javascript, Java, Go, C/C++, C#, Typescript](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|✅|✅|✅|✅|✅|✅| +|[Ruby Support](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|☑️|☑️|☑️|☑️|✅|✅| +|[Apple M1 support for CodeQL](https://github.blog/changelog/2021-11-10-codeql-now-supports-apple-silicon-m1/)|☑️|☑️|☑️|☑️|✅|✅| +|[Org-wide code scanning alerts via the REST API](https://docs.github.com/en/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-organization)||✅|✅|✅|✅|✅| +|[Add comments when dismissing alerts](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository#dismissing--alerts)|||✅|✅|✅|✅| +|[Code scanning alert comments in the pull request conversation tab](https://github.blog/changelog/2022-06-02-users-can-view-and-comment-on-code-scanning-alerts-on-the-conversation-tab-in-a-pull-request/)||||✅|✅|✅| +|[Users can publish CodeQL packs to the container registry](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/publishing-and-using-codeql-packs)||||✅|✅|✅| +|[CodeQL query filters to exclude individual queries](https://github.blog/changelog/2022-08-31-code-scanning-customize-your-codeql-analysis-using-query-filters/)||||✅|✅|✅| +|[Enterprise-wide code scanning alerts via the REST API](https://docs.github.com/en/enterprise-cloud@latest/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-enterprise)||||✅|✅|✅| +|[Filter API results by severity](https://github.blog/changelog/2022-11-25-filter-code-scanning-api-results-by-alert-severity/)|||||✅|✅| +|[Kotlin language support](https://github.blog/changelog/2022-11-28-codeql-code-scanning-launches-kotlin-analysis-support-beta/)|||||☑️|☑️| +|[Default CodeQL setup](https://docs.github.com/en/enterprise-server@3.9/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-for-a-repository#configuring-code-scanning-automatically)||||||✅| +|[Default CodeQL setup via API](https://docs.github.com/en/enterprise-server@3.9/rest/code-scanning#update-a-code-scanning-default-setup-configuration)||||||✅| +|["Enable all" functionality at the org level (API and UI)](https://docs.github.com/en/enterprise-server@3.9/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-at-scale)||||||✅| +|[Tool status page](https://docs.github.com/en/enterprise-server@3.9/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-the-tool-status-page)||||||✅| +|[View org-level enablement status via the API ](https://docs.github.com/en/enterprise-server@3.9/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories)||||||✅| + + + + +## Supply-chain security + +#### Dependabot Alerts +Dependabot alerts tell you that your code depends on a package that is insecure. +* [Dependabot alerts documentation](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) +* [Dependabot alerts API](https://docs.github.com/en/enterprise-cloud@latest/rest/dependabot/alerts?apiVersion=2022-11-28) + +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| +|[Dependabot Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts)|✅|✅|✅|✅|✅|✅| +|[Go modules support](https://docs.github.com/en/enterprise-cloud@latest/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| +|[Poetry support](https://docs.github.com/en/enterprise-cloud@latest/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| +|[Cargo support](https://docs.github.com/en/enterprise-cloud@latest/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|||✅|✅|✅|✅| +|[Reopen dismissed alerts](https://github.blog/changelog/2022-03-07-reopen-dismissed-dependabot-alerts/)|||✅|✅|✅|✅| +|[Dependabot alerts show vulnerable function calls](https://github.blog/2022-04-14-dependabot-alerts-now-surface-if-code-is-calling-vulnerability/)|||☑️|☑️|☑️|☑️| +|[Dependabot Alert timeline](https://github.blog/changelog/2022-07-28-dependabot-alerts-timeline-of-events-on-the-alert-details-page/)||||✅|✅|✅| +|[Bulk Editing of Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/viewing-and-updating-dependabot-alerts)||||✅|✅|✅| +|[Add comment when dismissing dependabot alert](https://github.blog/changelog/2022-08-22-dependabot-alerts-optional-dismissal-comment-2/)||||✅|✅|✅| +|[Dev Dependencies label](https://github.blog/2023-05-02-dependabot-relieves-alert-fatigue-from-npm-devdependencies/) ||||✅|✅|✅| +|[View Dependabot enablement status via org-level API](https://github.blog/changelog/2023-02-28-dependabot-alerts-enterprise-enablement-and-status-checking/)||||✅|✅|✅| +|[Receive alerts for vulnerable GitHub Actions](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot)||||✅|✅|✅| +|[Dependabot alert webhooks](https://github.blog/changelog/2022-10-06-new-dependabot-alerts-webhook/)||||✅|✅|✅| +|[Dependabot alerts REST API endpoint for repository org and enterprise](https://docs.github.com/en/rest/dependabot/alerts?apiVersion=2022-11-28)|||||☑️|✅| +|[Export SBOM from dependency graph](https://docs.github.com/en/enterprise-server@3.9/code-security/supply-chain-security/understanding-your-software-supply-chain/exporting-a-software-bill-of-materials-for-your-repository)||||||✅| +|[Dependabot can parse and update Gradle version catalogs in `settings.gradle`](https://docs.github.com/en/enterprise-server@3.9/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates)||||||✅| + + +#### Dependabot Updates +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| +|[Dependabot Updates](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuring-dependabot-version-updates)|☑️|✅|✅|✅|✅|✅| +|Actions authors can automatically update dependencies within workflow files|||||✅|✅| +|Dart and Flutter (using Pub) support for updates|||||✅|✅| +|[Automatically pause pull request activity after 90 days of inactivity](https://docs.github.com/en/enterprise-server@3.9/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates#about-automatic-deactivation-of-dependabot-updates)||||||✅| + +#### Dependency Review +Dependency review helps you understand dependency changes and the security impact of these changes at every pull request. +* [Dependency review docs](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/configuring-dependency-review) +* [Dependency review API docs](https://docs.github.com/en/rest/dependency-graph/dependency-review?apiVersion=2022-11-28) + +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| +|[Dependency Review](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/configuring-dependency-review)|✅|✅|✅|✅|✅|✅| +|[Enforcement Action](https://github.blog/changelog/2022-04-06-github-action-for-dependency-review-enforcement/)|||✅|✅|✅|✅| +|[Dependency Submission API](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/using-the-dependency-submission-api)||||✅|✅|✅| + + +## Security Overview +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 | +|------------------------------------------------------------|-----|-----|-----|-----|-----| +|Security Overview - Docs|✅|✅|✅|✅|✅| +|Organization view|☑️|✅|✅|✅|✅| +|Enterprise view||☑️|☑️|✅|✅| +|Organization-level Code Scanning Alert View||✅|✅|✅|✅| +|Organization-level Dependabot Alert View||✅|✅|✅|✅| +|Enterprse-level view of Dependabot alerts|||✅|✅|✅| +|Enterprse-level view of code scanning alerts||||✅|✅| +|Enterprse-level view of secret scanning alerts||||✅|✅| +|Coverage and Risk Security Overview pages|||||☑️| + +## Administration +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 | +|------------------------------------------------------------|-----|-----|-----|-----|-----| +|Security Managers Role - Docs|✅|✅|✅|✅|✅| +|Manage Security Managers role via the API||||✅|✅ From 2443f9a1c74a6c407c20e93c1e43980b59d28f83 Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Thu, 15 Jun 2023 16:57:14 -0700 Subject: [PATCH 108/128] Update GHAS-on-GHES-feature-matrix.md --- GHAS-on-GHES-feature-matrix.md | 42 +++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index fb0afbc..e335847 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -10,6 +10,9 @@ Each section of this document represents a different capability of the GitHub se - [Secret scanning](#secret-scanning) - [Code scanning](#code-scanning) - [Supply-chain security](#supply-chain-security) + - [Dependabot alerts](#dependabot-alerts) + - [Dependabot security updates](#dependabot-updates) + - [Dependency review](#dependency-review-and-submission-api) - [Security Overview](#security-overview) - [Administration](#administration) @@ -108,7 +111,7 @@ Dependabot alerts tell you that your code depends on a package that is insecure. |Dart and Flutter (using Pub) support for updates|||||✅|✅| |[Automatically pause pull request activity after 90 days of inactivity](https://docs.github.com/en/enterprise-server@3.9/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates#about-automatic-deactivation-of-dependabot-updates)||||||✅| -#### Dependency Review +#### Dependency Review and submission API Dependency review helps you understand dependency changes and the security impact of these changes at every pull request. * [Dependency review docs](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/configuring-dependency-review) * [Dependency review API docs](https://docs.github.com/en/rest/dependency-graph/dependency-review?apiVersion=2022-11-28) @@ -121,20 +124,27 @@ Dependency review helps you understand dependency changes and the security impac ## Security Overview -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 | -|------------------------------------------------------------|-----|-----|-----|-----|-----| -|Security Overview - Docs|✅|✅|✅|✅|✅| -|Organization view|☑️|✅|✅|✅|✅| -|Enterprise view||☑️|☑️|✅|✅| -|Organization-level Code Scanning Alert View||✅|✅|✅|✅| -|Organization-level Dependabot Alert View||✅|✅|✅|✅| -|Enterprse-level view of Dependabot alerts|||✅|✅|✅| -|Enterprse-level view of code scanning alerts||||✅|✅| -|Enterprse-level view of secret scanning alerts||||✅|✅| -|Coverage and Risk Security Overview pages|||||☑️| +Security overview provides high-level summaries of the security status of an organization or enterprise and makes it easy to identify repositories that require intervention. +* [Security Overview documentation](https://docs.github.com/en/enterprise-cloud@latest/code-security/security-overview/about-security-overview) + +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| +|[Security Overview](https://docs.github.com/en/enterprise-cloud@latest/code-security/security-overview/about-security-overview)|✅|✅|✅|✅|✅|✅| +|Organization view|☑️|✅|✅|✅|✅|✅| +|Enterprise view||☑️|☑️|✅|✅|✅| +|Organization-level Code Scanning Alert View||✅|✅|✅|✅|✅| +|Organization-level Dependabot Alert View||✅|✅|✅|✅|✅| +|Enterprse-level view of Dependabot alerts|||✅|✅|✅|✅| +|Enterprse-level view of code scanning alerts||||✅|✅|✅| +|Enterprse-level view of secret scanning alerts||||✅|✅|✅| +|Coverage and Risk Security Overview pages|||||☑️|☑️| +|[Filter alerts by repo topic](https://docs.github.com/en/enterprise-server@3.9/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅| +|[Filter alerts by team](https://docs.github.com/en/enterprise-server@3.9/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅| +|[Enable GHAS features in security overview](https://docs.github.com/en/enterprise-server@3.9/code-security/security-overview/about-security-overview)||||||✅| + ## Administration -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 | -|------------------------------------------------------------|-----|-----|-----|-----|-----| -|Security Managers Role - Docs|✅|✅|✅|✅|✅| -|Manage Security Managers role via the API||||✅|✅ +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| +|[Security Managers Role](https://docs.github.com/en/enterprise-cloud@latest/organizations/managing-peoples-access-to-your-organization-with-roles/managing-security-managers-in-your-organization)|✅|✅|✅|✅|✅|✅| +|[Manage Security Managers role via the API](https://docs.github.com/en/enterprise-cloud@latest/rest/orgs/security-managers?apiVersion=2022-11-28)||||✅|✅|✅| From d59027b8c394c3929421b515e30ed37efedbbd9b Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Thu, 15 Jun 2023 16:57:54 -0700 Subject: [PATCH 109/128] Update GHAS-on-GHES-feature-matrix.md --- GHAS-on-GHES-feature-matrix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index e335847..6163607 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -12,7 +12,7 @@ Each section of this document represents a different capability of the GitHub se - [Supply-chain security](#supply-chain-security) - [Dependabot alerts](#dependabot-alerts) - [Dependabot security updates](#dependabot-updates) - - [Dependency review](#dependency-review-and-submission-api) + - [Dependency review and submission api](#dependency-review-and-submission-api) - [Security Overview](#security-overview) - [Administration](#administration) From 96ed1b48ff2e4eb1bcbed1fb61e299363e50bba6 Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Thu, 15 Jun 2023 16:59:32 -0700 Subject: [PATCH 110/128] Update GHAS-on-GHES-feature-matrix.md --- GHAS-on-GHES-feature-matrix.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index 6163607..67163c3 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -4,7 +4,9 @@ This document helps answer the question "is this GHAS feature available in my ve The following tables include notable feature releases for GitHub Advanced Security. Each row represents a feature. The columns in the row indicate the level of support for each **supported** Enterprise Server release. Are your repositories hosted on github.com? All of these features are already available for you :+1:. -Each section of this document represents a different capability of the GitHub security features. Each row in the tables represent a different feature of GHAS. The columns indicate if that feature is available in each version of GitHub Enterprise Server. Cells with ☑️ indicate beta support. ✅ indicates full support. +Each section of this document represents a different capability of the GitHub security features. Each row in the tables represent a different feature of GHAS. The columns indicate if that feature is available in each version of GitHub Enterprise Server. + +Cells with ☑️ indicate beta support. ✅ indicates full support. #### Contents - [Secret scanning](#secret-scanning) From 92443f346069e25cd86f2b4ccd6f9fb42a201428 Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Thu, 15 Jun 2023 17:01:15 -0700 Subject: [PATCH 111/128] Update GHAS-on-GHES-feature-matrix.md --- GHAS-on-GHES-feature-matrix.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index 67163c3..6375df4 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -4,10 +4,6 @@ This document helps answer the question "is this GHAS feature available in my ve The following tables include notable feature releases for GitHub Advanced Security. Each row represents a feature. The columns in the row indicate the level of support for each **supported** Enterprise Server release. Are your repositories hosted on github.com? All of these features are already available for you :+1:. -Each section of this document represents a different capability of the GitHub security features. Each row in the tables represent a different feature of GHAS. The columns indicate if that feature is available in each version of GitHub Enterprise Server. - -Cells with ☑️ indicate beta support. ✅ indicates full support. - #### Contents - [Secret scanning](#secret-scanning) - [Code scanning](#code-scanning) @@ -18,6 +14,11 @@ Cells with ☑️ indicate beta support. ✅ indicates full support. - [Security Overview](#security-overview) - [Administration](#administration) +#### How do I read this document? +Each section of this document represents a different capability of the GitHub security features. Each row in the tables represent a different feature of GHAS. The columns indicate if that feature is available in each version of GitHub Enterprise Server. + +Cells with ☑️ indicate beta support. ✅ indicates full support. + ## Release notes |Version |3.4 |3.5 |3.6 |3.7 |3.8 |3.9| |---------|-----|-----|-----|-----|-----|-----| From a61f0ddb966280808a2817af8e9239720eb919f3 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 21 Jun 2023 15:39:34 -0400 Subject: [PATCH 112/128] update all links to GHES instead of GHEC@latest --- GHAS-on-GHES-feature-matrix.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index 6375df4..3af8b2e 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -27,20 +27,20 @@ Cells with ☑️ indicate beta support. ✅ indicates full support. ## Secret scanning Secret scanning identifies plain text credentials inside your code repository. Learn more about secret scanning -* [Secret scanning documentation](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/about-secret-scanning) -* [Secret scanning API documentation](https://docs.github.com/en/enterprise-cloud@latest/rest/secret-scanning?apiVersion=2022-11-28) +* [Secret scanning documentation](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/about-secret-scanning) +* [Secret scanning API documentation](https://docs.github.com/en/enterprise-server/rest/secret-scanning?apiVersion=2022-11-28) |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----| |Partner pattern count|155|169|173|173|183|200| -|[User defined (custom) patterns](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)|✅|✅|✅|✅|✅|✅| -|[Enterprise level API for secret scanning](https://docs.github.com/en/enterprise-cloud@latest/rest/secret-scanning?apiVersion=2022-11-28#list-secret-scanning-alerts-for-an-enterprise)|✅|✅|✅|✅|✅|✅| -|[Secret scanning push protection](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/protecting-pushes-with-secret-scanning)||✅|✅|✅|✅|✅| -|[Dry runs for secret scanning push protection (repo level)](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||✅|✅|✅|✅|✅| +|[User defined (custom) patterns](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)|✅|✅|✅|✅|✅|✅| +|[Enterprise level API for secret scanning](https://docs.github.com/en/enterprise-server/rest/secret-scanning?apiVersion=2022-11-28#list-secret-scanning-alerts-for-an-enterprise)|✅|✅|✅|✅|✅|✅| +|[Secret scanning push protection](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/protecting-pushes-with-secret-scanning)||✅|✅|✅|✅|✅| +|[Dry runs for secret scanning push protection (repo level)](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||✅|✅|✅|✅|✅| |[Secret scanning support for archived repos](https://github.blog/changelog/2022-02-16-secret-scanning-now-supports-archived-repositories/)||✅|✅|✅|✅|✅| |[Custom pattern events in the audit log](https://github.blog/changelog/2022-04-06-secret-scanning-custom-pattern-events-now-in-the-audit-log/)||✅|✅|✅|✅|✅| |[Push protection events in the audit log](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-security-settings-for-your-organization/audit-log-events-for-your-organization#secret_scanning_push_protection-category-actions)|||✅|✅|✅|✅| -|[Push protection in the web editor](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/protecting-pushes-with-secret-scanning#using-secret-scanning-as-a-push-protection-from-the-web-ui)|||✅|✅|✅|✅| +|[Push protection in the web editor](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/protecting-pushes-with-secret-scanning#using-secret-scanning-as-a-push-protection-from-the-web-ui)|||✅|✅|✅|✅| |[Enable secret scanning at the enterprise level](https://github.blog/changelog/2022-10-06-enable-secret-scanning-for-an-enterprise-with-one-click/)||||✅|✅|✅| |[Dry runs for secret scanning custom patterns (org level)](https://github.blog/changelog/2022-02-11-secret-scanning-dry-runs-for-repository-level-custom-pattern/)||||✅|✅|✅| |[Email notification for push protection bypass](https://github.blog/changelog/2022-07-27-secret-scanning-admins-now-receive-emails-when-contributors-bypass-a-push-protection-block/)||||✅|✅|✅| @@ -63,11 +63,11 @@ Code scanning is a feature that you use to analyze the code in a GitHub reposito |[Ruby Support](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|☑️|☑️|☑️|☑️|✅|✅| |[Apple M1 support for CodeQL](https://github.blog/changelog/2021-11-10-codeql-now-supports-apple-silicon-m1/)|☑️|☑️|☑️|☑️|✅|✅| |[Org-wide code scanning alerts via the REST API](https://docs.github.com/en/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-organization)||✅|✅|✅|✅|✅| -|[Add comments when dismissing alerts](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository#dismissing--alerts)|||✅|✅|✅|✅| +|[Add comments when dismissing alerts](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository#dismissing--alerts)|||✅|✅|✅|✅| |[Code scanning alert comments in the pull request conversation tab](https://github.blog/changelog/2022-06-02-users-can-view-and-comment-on-code-scanning-alerts-on-the-conversation-tab-in-a-pull-request/)||||✅|✅|✅| |[Users can publish CodeQL packs to the container registry](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/publishing-and-using-codeql-packs)||||✅|✅|✅| |[CodeQL query filters to exclude individual queries](https://github.blog/changelog/2022-08-31-code-scanning-customize-your-codeql-analysis-using-query-filters/)||||✅|✅|✅| -|[Enterprise-wide code scanning alerts via the REST API](https://docs.github.com/en/enterprise-cloud@latest/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-enterprise)||||✅|✅|✅| +|[Enterprise-wide code scanning alerts via the REST API](https://docs.github.com/en/enterprise-server/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-enterprise)||||✅|✅|✅| |[Filter API results by severity](https://github.blog/changelog/2022-11-25-filter-code-scanning-api-results-by-alert-severity/)|||||✅|✅| |[Kotlin language support](https://github.blog/changelog/2022-11-28-codeql-code-scanning-launches-kotlin-analysis-support-beta/)|||||☑️|☑️| |[Default CodeQL setup](https://docs.github.com/en/enterprise-server@3.9/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-for-a-repository#configuring-code-scanning-automatically)||||||✅| @@ -84,14 +84,14 @@ Code scanning is a feature that you use to analyze the code in a GitHub reposito #### Dependabot Alerts Dependabot alerts tell you that your code depends on a package that is insecure. * [Dependabot alerts documentation](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) -* [Dependabot alerts API](https://docs.github.com/en/enterprise-cloud@latest/rest/dependabot/alerts?apiVersion=2022-11-28) +* [Dependabot alerts API](https://docs.github.com/en/enterprise-server/rest/dependabot/alerts?apiVersion=2022-11-28) |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----| |[Dependabot Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts)|✅|✅|✅|✅|✅|✅| -|[Go modules support](https://docs.github.com/en/enterprise-cloud@latest/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| -|[Poetry support](https://docs.github.com/en/enterprise-cloud@latest/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| -|[Cargo support](https://docs.github.com/en/enterprise-cloud@latest/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|||✅|✅|✅|✅| +|[Go modules support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| +|[Poetry support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| +|[Cargo support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|||✅|✅|✅|✅| |[Reopen dismissed alerts](https://github.blog/changelog/2022-03-07-reopen-dismissed-dependabot-alerts/)|||✅|✅|✅|✅| |[Dependabot alerts show vulnerable function calls](https://github.blog/2022-04-14-dependabot-alerts-now-surface-if-code-is-calling-vulnerability/)|||☑️|☑️|☑️|☑️| |[Dependabot Alert timeline](https://github.blog/changelog/2022-07-28-dependabot-alerts-timeline-of-events-on-the-alert-details-page/)||||✅|✅|✅| @@ -128,11 +128,11 @@ Dependency review helps you understand dependency changes and the security impac ## Security Overview Security overview provides high-level summaries of the security status of an organization or enterprise and makes it easy to identify repositories that require intervention. -* [Security Overview documentation](https://docs.github.com/en/enterprise-cloud@latest/code-security/security-overview/about-security-overview) +* [Security Overview documentation](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview) |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|[Security Overview](https://docs.github.com/en/enterprise-cloud@latest/code-security/security-overview/about-security-overview)|✅|✅|✅|✅|✅|✅| +|[Security Overview](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview)|✅|✅|✅|✅|✅|✅| |Organization view|☑️|✅|✅|✅|✅|✅| |Enterprise view||☑️|☑️|✅|✅|✅| |Organization-level Code Scanning Alert View||✅|✅|✅|✅|✅| @@ -149,5 +149,5 @@ Security overview provides high-level summaries of the security status of an org ## Administration |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|[Security Managers Role](https://docs.github.com/en/enterprise-cloud@latest/organizations/managing-peoples-access-to-your-organization-with-roles/managing-security-managers-in-your-organization)|✅|✅|✅|✅|✅|✅| -|[Manage Security Managers role via the API](https://docs.github.com/en/enterprise-cloud@latest/rest/orgs/security-managers?apiVersion=2022-11-28)||||✅|✅|✅| +|[Security Managers Role](https://docs.github.com/en/enterprise-server/organizations/managing-peoples-access-to-your-organization-with-roles/managing-security-managers-in-your-organization)|✅|✅|✅|✅|✅|✅| +|[Manage Security Managers role via the API](https://docs.github.com/en/enterprise-server/rest/orgs/security-managers?apiVersion=2022-11-28)||||✅|✅|✅| From 27444d93aa94ddf4dfa1ea58a83cd5a030e0fb5f Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 21 Jun 2023 17:21:58 -0400 Subject: [PATCH 113/128] Update compiled-languages-csharp.md --- .../compiled-languages-csharp.md | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/troubleshooting/codeql-builds/compiled-languages-csharp.md b/troubleshooting/codeql-builds/compiled-languages-csharp.md index 36e3fa0..d739ce6 100644 --- a/troubleshooting/codeql-builds/compiled-languages-csharp.md +++ b/troubleshooting/codeql-builds/compiled-languages-csharp.md @@ -126,26 +126,28 @@ Next, consider specifying your own build steps from an existing CI workflow: ## "You are running out of disk space. The runner will stop working when the machine runs out of disk space." -Running low on disk using the default Actions runner? Try a few of these workarounds for a potential quick fix: - - Clean up large directories of [preinstalled software](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#preinstalled-software) that you are potentially not using on the windows runners, add this to your “CodeQL” workflow: +Running low on disk using the default Actions runner? -```yml -- name: Clean up some disks - run: | - rd C:\Android\android-sdk - docker system prune -af -``` +GitHub also offers larger runners, which are available in larger disk configurations. For more information, see "[About larger runners.](https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners#machine-specs-for-larger-runners)" +- See also: [Vertical Scaling](#vertical-scaling---throw-hardware-at-the-software-problem) + +Alternatively, try a few of these workarounds for a potential quick fix: - Specify the temp directory to store the CodeQL database - I have seen this resolve this specific problem with a Windows env (runs-on: windows-2019) +Specify the OS Disk's (C:\) temp directory to store the CodeQL database. There is a slower disk speed tradeoff compared to using the Data Disk (D:\) ```yml - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: db-location: ‘C:\windows\temp\codeql-database’ ``` - -- See also: [Vertical Scaling](#vertical-scaling---throw-hardware-at-the-software-problem) + +Clean up large directories of [preinstalled software](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#preinstalled-software) that you are not using on the windows runner OS Disk. Add this to your “CodeQL” workflow: +```yml +- name: Clean up some disks + run: | + rd C:\Android\android-sdk + docker system prune -af +``` ## MvcBuildViews target failures From 0dd071988d5708c99d74f5fc51cba12173946634 Mon Sep 17 00:00:00 2001 From: Natalie Somersall Date: Thu, 6 Jul 2023 14:42:34 +0000 Subject: [PATCH 114/128] add dependency matrix for ghes --- GHAS-on-GHES-feature-matrix.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index 6375df4..a1f221c 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -13,6 +13,7 @@ The following tables include notable feature releases for GitHub Advanced Securi - [Dependency review and submission api](#dependency-review-and-submission-api) - [Security Overview](#security-overview) - [Administration](#administration) +- [Dependencies](#dependencies) #### How do I read this document? Each section of this document represents a different capability of the GitHub security features. Each row in the tables represent a different feature of GHAS. The columns indicate if that feature is available in each version of GitHub Enterprise Server. @@ -151,3 +152,18 @@ Security overview provides high-level summaries of the security status of an org |------------------------------------------------------------|-----|-----|-----|-----|-----|-----| |[Security Managers Role](https://docs.github.com/en/enterprise-cloud@latest/organizations/managing-peoples-access-to-your-organization-with-roles/managing-security-managers-in-your-organization)|✅|✅|✅|✅|✅|✅| |[Manage Security Managers role via the API](https://docs.github.com/en/enterprise-cloud@latest/rest/orgs/security-managers?apiVersion=2022-11-28)||||✅|✅|✅| + + +## Dependencies +| Feature | GHAS license
required? | GitHub Actions
required? | GitHub Connect
required? | Documentation | Notes | +|---|---|---|---|---|---| +| Security Overview

DescriptionKnow what needs attention throughout the entire SDLC
| No * | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/security-overview/about-the-security-overview) | * Features not needing a GHAS license will still show up | +| Dependency Graph

DescriptionParse manifest and lock files in your repository
| No | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/admin/code-security/managing-supply-chain-security-for-your-enterprise/enabling-the-dependency-graph-for-your-enterprise) | Enabling this feature will reload some services on the appliance. | +| Dependabot Alerts

DescriptionKnow which of :point_up: have open CVEs
| No | No | Yes | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) | [GitHub Connect](https://docs.github.com/en/enterprise-server@latest/admin/configuration/configuring-github-connect/enabling-dependabot-for-your-enterprise) dependency and data transmission details | +| Dependabot Security Updates

DescriptionOne-click "enable all" to send PRs updating :point_up:
| No | Yes | Yes | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates) | Requires a runner with Docker and internet connectivity to open PRs ([specs](https://docs.github.com/en/enterprise-server@latest/admin/github-actions/enabling-github-actions-for-github-enterprise-server/managing-self-hosted-runners-for-dependabot-updates))

As of GHES 3.8, will not require internet connectivity _if_ private registry is configured | +| Dependabot Updates

DescriptionAllows Dependabot to process optional updates using `~/.github/dependabot.yml` file
| No | Yes | Yes | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/dependabot/dependabot-version-updates) | Same requirements as :point_up: - this just allows the same "non-security" updates using the same flexible configuration file as GitHub.com | +| Dependency Review

DescriptionInspect dependencies at pull request, blocking merges that add more security vulnerabilities
| Yes | Yes | Yes | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-dependency-review-for-your-appliance) | Does not require the build to be moved into GitHub Actions, but needs a runner to inspect manifests | +| CodeQL

DescriptionHighly accurate static analysis tool, flexible and extensible query language
| Yes | No * | No * | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning-with-codeql) | * CodeQL can be installed in your existing build system ([directions](https://docs.github.com/en/enterprise-server@latest/code-security/code-scanning/using-codeql-code-scanning-with-your-existing-ci-system/installing-codeql-cli-in-your-ci-system)) and/or be used on GitHub Actions with self-hosted runners ([directions](https://docs.github.com/en/enterprise-server@latest/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#running-code-scanning-using-github-actions))

* GitHub Connect is not required, but it makes keeping the CodeQL queries up-to-date easier.

* [codeql-action-sync-tool](https://github.com/github/codeql-action-sync-tool) is the offline updater without Connect. | +| Upload SARIF files from other tools

DescriptionView security results from other tools using SARIF file uploads
| Yes | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/code-scanning/integrating-with-code-scanning/uploading-a-sarif-file-to-github) | Many other tools support the SARIF interchange format. This feature provides a single pane of glass into the entire codebase. | +| Secret scanning

DescriptionLook at the present and all history for secrets, including partner patterns and custom regex
| Yes | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/secret-scanning) | | +| Push protection for secrets

DescriptionBlock commits containing partner patterns and custom regex from GitHub, preventing compromise
| Yes | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/secret-scanning/protecting-pushes-with-secret-scanning) | Bare metal hypervisors may require an additional CPU flag, as outlined [here](https://docs.github.com/en/enterprise-server@latest/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-secret-scanning-for-your-appliance) | From 04d221584ef1e73dfde37df0e7b4221eca9097a3 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:43:18 -0400 Subject: [PATCH 115/128] add link to new Dependabot quickstart guide closes #15 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3828786..fbf7f03 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ New to GitHub Advanced Security? Start with [GitHub security features](https:// - [Dependabot Alerts](https://docs.github.com/en/enterprise-cloud@latest/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) - [Dependabot Security Updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates) - [GitHub Advisory Database](https://github.com/advisories) +- [Dependabot Quicksatart Guide](https://docs.github.com/en/code-security/getting-started/dependabot-quickstart-guide) ### Security Overview - [About Security Overview](https://docs.github.com/en/code-security/security-overview/about-the-security-overview) From 90a9b8658b31d6843aa8c659250922e1a6174c2e Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:44:41 -0400 Subject: [PATCH 116/128] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbf7f03..5b897dc 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ New to GitHub Advanced Security? Start with [GitHub security features](https:// - [Dependabot Alerts](https://docs.github.com/en/enterprise-cloud@latest/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) - [Dependabot Security Updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates) - [GitHub Advisory Database](https://github.com/advisories) -- [Dependabot Quicksatart Guide](https://docs.github.com/en/code-security/getting-started/dependabot-quickstart-guide) +- [Dependabot Quickstart Guide](https://docs.github.com/en/code-security/getting-started/dependabot-quickstart-guide) ### Security Overview - [About Security Overview](https://docs.github.com/en/code-security/security-overview/about-the-security-overview) From 976e2969615a214919532fdcc9eabfe8fae3bd5e Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:01:12 -0400 Subject: [PATCH 117/128] Create compiled-languages-go.md --- .../codeql-builds/compiled-languages-go.md | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 troubleshooting/codeql-builds/compiled-languages-go.md diff --git a/troubleshooting/codeql-builds/compiled-languages-go.md b/troubleshooting/codeql-builds/compiled-languages-go.md new file mode 100644 index 0000000..b27630b --- /dev/null +++ b/troubleshooting/codeql-builds/compiled-languages-go.md @@ -0,0 +1,40 @@ +## GoLang Private Modules + +Autobuild fails with error "Some packages could not be found" + +There are two options when it comes to private repositories: + +- Set-up the Go environment within the Actions workflow (not vendoring then) +- Vendor the dependencies + +Setting up the Go environment can be done by adding a Actions step to update the [Go settings](https://go.dev/ref/mod#private-modules) pointing them to use a [GitHub Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) with the corresponding access to the private repository. +The example below shows how this can be done using a single step beforet the CodeQL Initize step and stores the GitHub PAT in Secrets. + +**Example:** + +```yml +name: CodeQL + +env: + GOLANG_TOKEN: ${{ secrets.GOLANG_GITHUB_TOKEN }} + GOLANG_USER: octocat + +# ... +jobs: + analyze: + name: Analyze + # ... + steps: + - name: Go Configuration + run: git config --global url."https://${GOLANG_USER}:${GOLANG_TOKEN}@github.com".insteadOf "https://github.com" + + # ... Start scanning +``` + +Alternatively, pass the token into the CodeQL init action to allow it to be used for downstream git operations: + +```yml +- uses: github/codeql-action/init@v2 + with: + external-repository-token: ${{ secrets.GOLANG_GITHUB_TOKEN }} +``` From c852b3e4ef020cfa749258d822c7fda87fafaf97 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:02:24 -0400 Subject: [PATCH 118/128] Update compiled-languages.md --- troubleshooting/codeql-builds/compiled-languages.md | 1 + 1 file changed, 1 insertion(+) diff --git a/troubleshooting/codeql-builds/compiled-languages.md b/troubleshooting/codeql-builds/compiled-languages.md index f84d11f..7838bba 100644 --- a/troubleshooting/codeql-builds/compiled-languages.md +++ b/troubleshooting/codeql-builds/compiled-languages.md @@ -5,6 +5,7 @@ * [CSharp](compiled-languages-csharp.md) * [C++](compiled-languages-cpp.md) * [Java](compiled-languages-java.md) +* [Go](compiled-languages-go.md) ## Autobuilder The autobuilder action (see [docs](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages#about-autobuild-for-codeql) ) From c07d2b17ca971a68ba84d7607c1c08ebb55b3540 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:15:11 -0400 Subject: [PATCH 119/128] update links from 3.9 to (latest) --- GHAS-on-GHES-feature-matrix.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index 9f36472..81f97da 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -49,7 +49,7 @@ Secret scanning identifies plain text credentials inside your code repository. |[View secret scanning enablement status at the org-level via API](https://github.blog/changelog/2021-08-24-secret-scanning-org-level-rest-api/)||||✅|✅|✅| |[Enable secret scanning at the enterprise level using the REST API](https://github.blog/changelog/2022-12-13-enable-secret-scanning-with-the-enterprise-level-rest-api/)|||||✅|✅| |[Add comment when dismissing a secret scanning alert in UI or API](https://github.blog/changelog/2022-09-29-secret-scanning-alerts-now-have-a-timeline-and-users-can-add-a-comment-when-resolving/)|||||✅|✅| -|[Custom pattern creation at the enterprise level](https://docs.github.com/en/enterprise-server@3.9/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||||||✅| +|[Custom pattern creation at the enterprise level](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||||||✅| ## Code scanning @@ -71,11 +71,11 @@ Code scanning is a feature that you use to analyze the code in a GitHub reposito |[Enterprise-wide code scanning alerts via the REST API](https://docs.github.com/en/enterprise-server/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-enterprise)||||✅|✅|✅| |[Filter API results by severity](https://github.blog/changelog/2022-11-25-filter-code-scanning-api-results-by-alert-severity/)|||||✅|✅| |[Kotlin language support](https://github.blog/changelog/2022-11-28-codeql-code-scanning-launches-kotlin-analysis-support-beta/)|||||☑️|☑️| -|[Default CodeQL setup](https://docs.github.com/en/enterprise-server@3.9/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-for-a-repository#configuring-code-scanning-automatically)||||||✅| -|[Default CodeQL setup via API](https://docs.github.com/en/enterprise-server@3.9/rest/code-scanning#update-a-code-scanning-default-setup-configuration)||||||✅| -|["Enable all" functionality at the org level (API and UI)](https://docs.github.com/en/enterprise-server@3.9/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-at-scale)||||||✅| -|[Tool status page](https://docs.github.com/en/enterprise-server@3.9/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-the-tool-status-page)||||||✅| -|[View org-level enablement status via the API ](https://docs.github.com/en/enterprise-server@3.9/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories)||||||✅| +|[Default CodeQL setup](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-for-a-repository#configuring-code-scanning-automatically)||||||✅| +|[Default CodeQL setup via API](https://docs.github.com/en/enterprise-server/rest/code-scanning#update-a-code-scanning-default-setup-configuration)||||||✅| +|["Enable all" functionality at the org level (API and UI)](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-at-scale)||||||✅| +|[Tool status page](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-the-tool-status-page)||||||✅| +|[View org-level enablement status via the API ](https://docs.github.com/en/enterprise-server/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories)||||||✅| @@ -103,8 +103,8 @@ Dependabot alerts tell you that your code depends on a package that is insecure. |[Receive alerts for vulnerable GitHub Actions](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot)||||✅|✅|✅| |[Dependabot alert webhooks](https://github.blog/changelog/2022-10-06-new-dependabot-alerts-webhook/)||||✅|✅|✅| |[Dependabot alerts REST API endpoint for repository org and enterprise](https://docs.github.com/en/rest/dependabot/alerts?apiVersion=2022-11-28)|||||☑️|✅| -|[Export SBOM from dependency graph](https://docs.github.com/en/enterprise-server@3.9/code-security/supply-chain-security/understanding-your-software-supply-chain/exporting-a-software-bill-of-materials-for-your-repository)||||||✅| -|[Dependabot can parse and update Gradle version catalogs in `settings.gradle`](https://docs.github.com/en/enterprise-server@3.9/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates)||||||✅| +|[Export SBOM from dependency graph](https://docs.github.com/en/enterprise-server/code-security/supply-chain-security/understanding-your-software-supply-chain/exporting-a-software-bill-of-materials-for-your-repository)||||||✅| +|[Dependabot can parse and update Gradle version catalogs in `settings.gradle`](https://docs.github.com/en/enterprise-server/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates)||||||✅| #### Dependabot Updates @@ -113,7 +113,7 @@ Dependabot alerts tell you that your code depends on a package that is insecure. |[Dependabot Updates](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuring-dependabot-version-updates)|☑️|✅|✅|✅|✅|✅| |Actions authors can automatically update dependencies within workflow files|||||✅|✅| |Dart and Flutter (using Pub) support for updates|||||✅|✅| -|[Automatically pause pull request activity after 90 days of inactivity](https://docs.github.com/en/enterprise-server@3.9/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates#about-automatic-deactivation-of-dependabot-updates)||||||✅| +|[Automatically pause pull request activity after 90 days of inactivity](https://docs.github.com/en/enterprise-server/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates#about-automatic-deactivation-of-dependabot-updates)||||||✅| #### Dependency Review and submission API Dependency review helps you understand dependency changes and the security impact of these changes at every pull request. @@ -142,9 +142,9 @@ Security overview provides high-level summaries of the security status of an org |Enterprse-level view of code scanning alerts||||✅|✅|✅| |Enterprse-level view of secret scanning alerts||||✅|✅|✅| |Coverage and Risk Security Overview pages|||||☑️|☑️| -|[Filter alerts by repo topic](https://docs.github.com/en/enterprise-server@3.9/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅| -|[Filter alerts by team](https://docs.github.com/en/enterprise-server@3.9/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅| -|[Enable GHAS features in security overview](https://docs.github.com/en/enterprise-server@3.9/code-security/security-overview/about-security-overview)||||||✅| +|[Filter alerts by repo topic](https://docs.github.com/en/enterprise-server/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅| +|[Filter alerts by team](https://docs.github.com/en/enterprise-server/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅| +|[Enable GHAS features in security overview](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview)||||||✅| ## Administration From d394c306819aafba1ac7310e19f7f00ef13cd8d4 Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 2 Aug 2023 10:12:48 -0400 Subject: [PATCH 120/128] GHES + Codeql Versions --- GHAS-on-GHES-feature-matrix.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index 81f97da..ea6b08d 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -23,7 +23,8 @@ Cells with ☑️ indicate beta support. ✅ indicates full support. ## Release notes |Version |3.4 |3.5 |3.6 |3.7 |3.8 |3.9| |---------|-----|-----|-----|-----|-----|-----| -|Release date| Mar. 15 2022| May 31 2022 |Aug. 16 2022 |Nov. 8 2022 |Mar. 7 2023 |Jun. 8 2023 (rc1) | +|Release date| 2022-02-15 | 2022-05-10 | 2022-07-26 |2022-10-25 |2023-02-07 | 2023-06-08 | +|Deprecation date | 2023-03-23 | 2023-06-29 | 2023-08-16 | 2023-11-08 | 2024-03-07 | 2024-06-29 | || [Release notes](https://docs.github.com/en/enterprise-server@3.4/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.5/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.6/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.7/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.8/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.9/admin/release-notes)| ## Secret scanning @@ -59,7 +60,7 @@ Code scanning is a feature that you use to analyze the code in a GitHub reposito |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|CodeQL "toolcache" Installed Version|2.7.6|2.8.5|2.9.4|2.10.5|2.11.6| +|[CodeQL "toolcache" Installed Version](https://docs.github.com/en/enterprise-server/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#provisioning-the-actions-for-code-scanning)|2.7.6|2.11.6|2.11.7|2.11.7|2.11.7|2.11.7| |[Language support: Python, Javascript, Java, Go, C/C++, C#, Typescript](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|✅|✅|✅|✅|✅|✅| |[Ruby Support](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|☑️|☑️|☑️|☑️|✅|✅| |[Apple M1 support for CodeQL](https://github.blog/changelog/2021-11-10-codeql-now-supports-apple-silicon-m1/)|☑️|☑️|☑️|☑️|✅|✅| From 9006b76f06bfa803c20992f822cf0dc9ed8ef9db Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 2 Aug 2023 10:23:05 -0400 Subject: [PATCH 121/128] Add link to Releases of GitHub Enterprise Server --- GHAS-on-GHES-feature-matrix.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index ea6b08d..e97a263 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -21,6 +21,8 @@ Each section of this document represents a different capability of the GitHub se Cells with ☑️ indicate beta support. ✅ indicates full support. ## Release notes +* [Releases of GitHub Enterprise Server](https://docs.github.com/en/enterprise-server/admin/all-releases#releases-of-github-enterprise-server) + |Version |3.4 |3.5 |3.6 |3.7 |3.8 |3.9| |---------|-----|-----|-----|-----|-----|-----| |Release date| 2022-02-15 | 2022-05-10 | 2022-07-26 |2022-10-25 |2023-02-07 | 2023-06-08 | From c42883d8759c418e0930f7b9304d08ccef55140e Mon Sep 17 00:00:00 2001 From: Mathew Payne <2772944+GeekMasher@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:21:02 +0100 Subject: [PATCH 122/128] Create owasp-webgoat-codeql.yml --- .../owasp-webgoat-codeql.yml | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 code-scanning-workflows/owasp-webgoat-codeql.yml diff --git a/code-scanning-workflows/owasp-webgoat-codeql.yml b/code-scanning-workflows/owasp-webgoat-codeql.yml new file mode 100644 index 0000000..68dec52 --- /dev/null +++ b/code-scanning-workflows/owasp-webgoat-codeql.yml @@ -0,0 +1,61 @@ +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + workflow_dispatch: + +permissions: + actions: read + contents: read + security-events: write + +env: + CODEQL_EXTRACTOR_JAVA_RUN_ANNOTATION_PROCESSORS: true + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'java', 'javascript' ] + + steps: + - uses: actions/checkout@v2 + + # WebGoat requires Java/JDK 11 + - name: Set up JDK 1.11 + if: matrix.language == 'java' + uses: actions/setup-java@v1 + with: + java-version: 1.11 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # [optional] enabled extended queries + queries: +security-extended # or `+security-and-quality` can be used + # [optional] Field Config - standard packs, extensions, and extra packs + # config-file: advanced-security/codeql-queries/config/codeql.yml@main + + # Use mvn command + - name: Build Code + if: matrix.language == 'java' + run: | + mvn clean install --file pom.xml + + # Run the Analysis + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + + # Submit Maven Dependency Tree to GitHub + - name: Maven Dependency Tree Dependency Submission + if: matrix.language == 'java' + uses: advanced-security/maven-dependency-submission-action@v3.0.2 From 5d8ae8dca6895525bb9d2e8c3488b4db8e5eaac1 Mon Sep 17 00:00:00 2001 From: Mathew Payne <2772944+GeekMasher@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:37:06 +0000 Subject: [PATCH 123/128] Update WebGoat --- .../owasp-webgoat-codeql.yml | 0 .../synthetic-applications/owasp-webgoat.md | 13 +++++++++++++ 2 files changed, 13 insertions(+) rename {code-scanning-workflows => code-scanning-guides/synthetic-applications}/owasp-webgoat-codeql.yml (100%) create mode 100644 code-scanning-guides/synthetic-applications/owasp-webgoat.md diff --git a/code-scanning-workflows/owasp-webgoat-codeql.yml b/code-scanning-guides/synthetic-applications/owasp-webgoat-codeql.yml similarity index 100% rename from code-scanning-workflows/owasp-webgoat-codeql.yml rename to code-scanning-guides/synthetic-applications/owasp-webgoat-codeql.yml diff --git a/code-scanning-guides/synthetic-applications/owasp-webgoat.md b/code-scanning-guides/synthetic-applications/owasp-webgoat.md new file mode 100644 index 0000000..3fda3b0 --- /dev/null +++ b/code-scanning-guides/synthetic-applications/owasp-webgoat.md @@ -0,0 +1,13 @@ +# OWASP WebGoat + +[A full Actions workflow can be found here](./owasp-webgoat-codeql.yml) + +## Common Issues + +Scanning OWASP WebGoat can have some issues right out of the box where CodeQL might find very little or worse not find anything at all. +This is due to the following: + +1. WebGoat uses JDK 11 + - Action uses JDK 8 by default +2. Uses Project Lombok + - Future support will be coming to CodeQL natively From 3ecb38b4d797800737b05518d2a3e1fa4dba5859 Mon Sep 17 00:00:00 2001 From: Mathew Payne <2772944+GeekMasher@users.noreply.github.com> Date: Wed, 16 Aug 2023 15:08:04 +0100 Subject: [PATCH 124/128] Update and rename owasp-webgoat-codeql.yml to owasp-webgoat.yml --- ...p-webgoat-codeql.yml => owasp-webgoat.yml} | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) rename code-scanning-guides/synthetic-applications/{owasp-webgoat-codeql.yml => owasp-webgoat.yml} (69%) diff --git a/code-scanning-guides/synthetic-applications/owasp-webgoat-codeql.yml b/code-scanning-guides/synthetic-applications/owasp-webgoat.yml similarity index 69% rename from code-scanning-guides/synthetic-applications/owasp-webgoat-codeql.yml rename to code-scanning-guides/synthetic-applications/owasp-webgoat.yml index 68dec52..f38ee87 100644 --- a/code-scanning-guides/synthetic-applications/owasp-webgoat-codeql.yml +++ b/code-scanning-guides/synthetic-applications/owasp-webgoat.yml @@ -2,10 +2,10 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [ main ] pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: [ main ] workflow_dispatch: permissions: @@ -14,6 +14,7 @@ permissions: security-events: write env: + # in the future, this flag will not be needed CODEQL_EXTRACTOR_JAVA_RUN_ANNOTATION_PROCESSORS: true jobs: @@ -29,27 +30,26 @@ jobs: steps: - uses: actions/checkout@v2 - # WebGoat requires Java/JDK 11 - - name: Set up JDK 1.11 + # WebGoat requires Java/JDK 17 + - name: Set up JDK 17 if: matrix.language == 'java' - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 1.11 + distribution: 'temurin' + java-version: 17 + architecture: x64 - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # [optional] enabled extended queries - queries: +security-extended # or `+security-and-quality` can be used + # queries: +security-extended,security-and-quality # [optional] Field Config - standard packs, extensions, and extra packs - # config-file: advanced-security/codeql-queries/config/codeql.yml@main + config-file: advanced-security/codeql-queries/config/codeql.yml@main - # Use mvn command - - name: Build Code - if: matrix.language == 'java' - run: | - mvn clean install --file pom.xml + - name: Autobuild + uses: github/codeql-action/autobuild@v2 # Run the Analysis - name: Perform CodeQL Analysis From 65ae0ab52f04a2bf0a8e152c48e1a7699a9e6614 Mon Sep 17 00:00:00 2001 From: Mathew Payne <2772944+GeekMasher@users.noreply.github.com> Date: Wed, 16 Aug 2023 15:10:01 +0100 Subject: [PATCH 125/128] Update owasp-webgoat.md --- .../synthetic-applications/owasp-webgoat.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code-scanning-guides/synthetic-applications/owasp-webgoat.md b/code-scanning-guides/synthetic-applications/owasp-webgoat.md index 3fda3b0..dd92456 100644 --- a/code-scanning-guides/synthetic-applications/owasp-webgoat.md +++ b/code-scanning-guides/synthetic-applications/owasp-webgoat.md @@ -1,13 +1,15 @@ # OWASP WebGoat -[A full Actions workflow can be found here](./owasp-webgoat-codeql.yml) +[A full Actions workflow can be found here](./owasp-webgoat.yml) ## Common Issues Scanning OWASP WebGoat can have some issues right out of the box where CodeQL might find very little or worse not find anything at all. This is due to the following: -1. WebGoat uses JDK 11 +1. WebGoat uses JDK 17 - Action uses JDK 8 by default 2. Uses Project Lombok - Future support will be coming to CodeQL natively +3. Dependencies are not all present in Dependency Graph + - Using [Submission API](https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/using-the-dependency-submission-api#using-pre-made-actions) From 0424cb99932c4f243fcb4fedbac23f9b908dca1a Mon Sep 17 00:00:00 2001 From: Chad Bentz <1760475+felickz@users.noreply.github.com> Date: Wed, 16 Aug 2023 18:29:06 -0400 Subject: [PATCH 126/128] Init GHES 3.10 changes --- GHAS-on-GHES-feature-matrix.md | 54 +++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index e97a263..6f6d40f 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -23,20 +23,20 @@ Cells with ☑️ indicate beta support. ✅ indicates full support. ## Release notes * [Releases of GitHub Enterprise Server](https://docs.github.com/en/enterprise-server/admin/all-releases#releases-of-github-enterprise-server) -|Version |3.4 |3.5 |3.6 |3.7 |3.8 |3.9| -|---------|-----|-----|-----|-----|-----|-----| -|Release date| 2022-02-15 | 2022-05-10 | 2022-07-26 |2022-10-25 |2023-02-07 | 2023-06-08 | -|Deprecation date | 2023-03-23 | 2023-06-29 | 2023-08-16 | 2023-11-08 | 2024-03-07 | 2024-06-29 | -|| [Release notes](https://docs.github.com/en/enterprise-server@3.4/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.5/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.6/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.7/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.8/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.9/admin/release-notes)| +|Version |3.4 |3.5 |3.6 |3.7 |3.8 |3.9| 3.10| +|---------|-----|-----|-----|-----|-----|-----|----| +|Release date| 2022-02-15 | 2022-05-10 | 2022-07-26 |2022-10-25 |2023-02-07 | 2023-06-08 | 2023-08-08 | +|Deprecation date | 2023-03-23 | 2023-06-29 | 2023-08-16 | 2023-11-08 | 2024-03-07 | 2024-06-29 | 2024-08-29 | +|| [Release notes](https://docs.github.com/en/enterprise-server@3.4/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.5/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.6/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.7/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.8/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.9/admin/release-notes)|[Release notes](https://docs.github.com/en/enterprise-server@3.10/admin/release-notes) ## Secret scanning Secret scanning identifies plain text credentials inside your code repository. Learn more about secret scanning * [Secret scanning documentation](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/about-secret-scanning) * [Secret scanning API documentation](https://docs.github.com/en/enterprise-server/rest/secret-scanning?apiVersion=2022-11-28) -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | -|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|Partner pattern count|155|169|173|173|183|200| +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | 3.10 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----|-----| +|Partner pattern count|155|169|173|173|183|200|🚩| |[User defined (custom) patterns](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)|✅|✅|✅|✅|✅|✅| |[Enterprise level API for secret scanning](https://docs.github.com/en/enterprise-server/rest/secret-scanning?apiVersion=2022-11-28#list-secret-scanning-alerts-for-an-enterprise)|✅|✅|✅|✅|✅|✅| |[Secret scanning push protection](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/protecting-pushes-with-secret-scanning)||✅|✅|✅|✅|✅| @@ -60,9 +60,9 @@ Code scanning is a feature that you use to analyze the code in a GitHub reposito * [Code scanning documentation](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning) * [Code scanning API documentation](https://docs.github.com/en/rest/code-scanning?apiVersion=2022-11-28) -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | -|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|[CodeQL "toolcache" Installed Version](https://docs.github.com/en/enterprise-server/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#provisioning-the-actions-for-code-scanning)|2.7.6|2.11.6|2.11.7|2.11.7|2.11.7|2.11.7| +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| +|[CodeQL "toolcache" Installed Version](https://docs.github.com/en/enterprise-server/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#provisioning-the-actions-for-code-scanning)|2.7.6|2.11.6|2.11.7|2.11.7|2.11.7|2.11.7|🚩| |[Language support: Python, Javascript, Java, Go, C/C++, C#, Typescript](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|✅|✅|✅|✅|✅|✅| |[Ruby Support](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|☑️|☑️|☑️|☑️|✅|✅| |[Apple M1 support for CodeQL](https://github.blog/changelog/2021-11-10-codeql-now-supports-apple-silicon-m1/)|☑️|☑️|☑️|☑️|✅|✅| @@ -90,9 +90,9 @@ Dependabot alerts tell you that your code depends on a package that is insecure. * [Dependabot alerts documentation](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) * [Dependabot alerts API](https://docs.github.com/en/enterprise-server/rest/dependabot/alerts?apiVersion=2022-11-28) -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | -|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|[Dependabot Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts)|✅|✅|✅|✅|✅|✅| +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| +|[Dependabot Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts)|✅|✅|✅|✅|✅|✅|🚩| |[Go modules support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| |[Poetry support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| |[Cargo support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|||✅|✅|✅|✅| @@ -111,9 +111,9 @@ Dependabot alerts tell you that your code depends on a package that is insecure. #### Dependabot Updates -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | -|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|[Dependabot Updates](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuring-dependabot-version-updates)|☑️|✅|✅|✅|✅|✅| +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| +|[Dependabot Updates](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuring-dependabot-version-updates)|☑️|✅|✅|✅|✅|✅|🚩| |Actions authors can automatically update dependencies within workflow files|||||✅|✅| |Dart and Flutter (using Pub) support for updates|||||✅|✅| |[Automatically pause pull request activity after 90 days of inactivity](https://docs.github.com/en/enterprise-server/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates#about-automatic-deactivation-of-dependabot-updates)||||||✅| @@ -123,9 +123,9 @@ Dependency review helps you understand dependency changes and the security impac * [Dependency review docs](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/configuring-dependency-review) * [Dependency review API docs](https://docs.github.com/en/rest/dependency-graph/dependency-review?apiVersion=2022-11-28) -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | -|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|[Dependency Review](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/configuring-dependency-review)|✅|✅|✅|✅|✅|✅| +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 | +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| +|[Dependency Review](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/configuring-dependency-review)|✅|✅|✅|✅|✅|✅|🚩| |[Enforcement Action](https://github.blog/changelog/2022-04-06-github-action-for-dependency-review-enforcement/)|||✅|✅|✅|✅| |[Dependency Submission API](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/using-the-dependency-submission-api)||||✅|✅|✅| @@ -134,9 +134,9 @@ Dependency review helps you understand dependency changes and the security impac Security overview provides high-level summaries of the security status of an organization or enterprise and makes it easy to identify repositories that require intervention. * [Security Overview documentation](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview) -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | -|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|[Security Overview](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview)|✅|✅|✅|✅|✅|✅| +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10| +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| +|[Security Overview](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview)|✅|✅|✅|✅|✅|✅|🚩| |Organization view|☑️|✅|✅|✅|✅|✅| |Enterprise view||☑️|☑️|✅|✅|✅| |Organization-level Code Scanning Alert View||✅|✅|✅|✅|✅| @@ -151,9 +151,9 @@ Security overview provides high-level summaries of the security status of an org ## Administration -|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | -|------------------------------------------------------------|-----|-----|-----|-----|-----|-----| -|[Security Managers Role](https://docs.github.com/en/enterprise-server/organizations/managing-peoples-access-to-your-organization-with-roles/managing-security-managers-in-your-organization)|✅|✅|✅|✅|✅|✅| +|Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 +|------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| +|[Security Managers Role](https://docs.github.com/en/enterprise-server/organizations/managing-peoples-access-to-your-organization-with-roles/managing-security-managers-in-your-organization)|✅|✅|✅|✅|✅|✅|🚩| |[Manage Security Managers role via the API](https://docs.github.com/en/enterprise-server/rest/orgs/security-managers?apiVersion=2022-11-28)||||✅|✅|✅| ## Dependencies @@ -165,7 +165,7 @@ Security overview provides high-level summaries of the security status of an org | Dependabot Security Updates

DescriptionOne-click "enable all" to send PRs updating :point_up:
| No | Yes | Yes | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates) | Requires a runner with Docker and internet connectivity to open PRs ([specs](https://docs.github.com/en/enterprise-server@latest/admin/github-actions/enabling-github-actions-for-github-enterprise-server/managing-self-hosted-runners-for-dependabot-updates))

As of GHES 3.8, will not require internet connectivity _if_ private registry is configured | | Dependabot Updates

DescriptionAllows Dependabot to process optional updates using `~/.github/dependabot.yml` file
| No | Yes | Yes | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/dependabot/dependabot-version-updates) | Same requirements as :point_up: - this just allows the same "non-security" updates using the same flexible configuration file as GitHub.com | | Dependency Review

DescriptionInspect dependencies at pull request, blocking merges that add more security vulnerabilities
| Yes | Yes | Yes | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-dependency-review-for-your-appliance) | Does not require the build to be moved into GitHub Actions, but needs a runner to inspect manifests | -| CodeQL

DescriptionHighly accurate static analysis tool, flexible and extensible query language
| Yes | No * | No * | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning-with-codeql) | * CodeQL can be installed in your existing build system ([directions](https://docs.github.com/en/enterprise-server@latest/code-security/code-scanning/using-codeql-code-scanning-with-your-existing-ci-system/installing-codeql-cli-in-your-ci-system)) and/or be used on GitHub Actions with self-hosted runners ([directions](https://docs.github.com/en/enterprise-server@latest/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#running-code-scanning-using-github-actions))

* GitHub Connect is not required, but it makes keeping the CodeQL queries up-to-date easier.

* [codeql-action-sync-tool](https://github.com/github/codeql-action-sync-tool) is the offline updater without Connect. | +| CodeQL

DescriptionHighly accurate static analysis tool, flexible and extensible query language
| Yes | No * | No * | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning-with-codeql) | * CodeQL can be installed in your existing build system ([directions](https://docs.github.com/en/enterprise-server@latest/code-security/code-scanning/using-codeql-code-scanning-with-your-existing-ci-system/installing-codeql-cli-in-your-ci-system)) and/or be used on GitHub Actions with self-hosted runners ([directions](https://docs.github.com/en/enterprise-server@latest/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#running-code-scanning-using-github-actions))

* GitHub Connect is not required, but it makes keeping the CodeQL queries up-to-date easier.

* [codeql-action-sync-tool](https://github.com/github/codeql-action-sync-tool) is the offline updater without Connect.

* Code Scanning default setup requires runners with the `code-scanning` label applied. | | Upload SARIF files from other tools

DescriptionView security results from other tools using SARIF file uploads
| Yes | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/code-scanning/integrating-with-code-scanning/uploading-a-sarif-file-to-github) | Many other tools support the SARIF interchange format. This feature provides a single pane of glass into the entire codebase. | | Secret scanning

DescriptionLook at the present and all history for secrets, including partner patterns and custom regex
| Yes | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/secret-scanning) | | | Push protection for secrets

DescriptionBlock commits containing partner patterns and custom regex from GitHub, preventing compromise
| Yes | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/secret-scanning/protecting-pushes-with-secret-scanning) | Bare metal hypervisors may require an additional CPU flag, as outlined [here](https://docs.github.com/en/enterprise-server@latest/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-secret-scanning-for-your-appliance) | From 9d65433412072530bfdc81acaa5ce6a6c2a972b7 Mon Sep 17 00:00:00 2001 From: Dan Shanahan Date: Tue, 29 Aug 2023 14:45:36 -0700 Subject: [PATCH 127/128] Update GHAS-on-GHES-feature-matrix.md --- GHAS-on-GHES-feature-matrix.md | 158 ++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 74 deletions(-) diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md index 6f6d40f..5b26512 100644 --- a/GHAS-on-GHES-feature-matrix.md +++ b/GHAS-on-GHES-feature-matrix.md @@ -36,24 +36,24 @@ Secret scanning identifies plain text credentials inside your code repository. |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 | 3.10 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----|-----| -|Partner pattern count|155|169|173|173|183|200|🚩| -|[User defined (custom) patterns](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)|✅|✅|✅|✅|✅|✅| -|[Enterprise level API for secret scanning](https://docs.github.com/en/enterprise-server/rest/secret-scanning?apiVersion=2022-11-28#list-secret-scanning-alerts-for-an-enterprise)|✅|✅|✅|✅|✅|✅| -|[Secret scanning push protection](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/protecting-pushes-with-secret-scanning)||✅|✅|✅|✅|✅| -|[Dry runs for secret scanning push protection (repo level)](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||✅|✅|✅|✅|✅| -|[Secret scanning support for archived repos](https://github.blog/changelog/2022-02-16-secret-scanning-now-supports-archived-repositories/)||✅|✅|✅|✅|✅| -|[Custom pattern events in the audit log](https://github.blog/changelog/2022-04-06-secret-scanning-custom-pattern-events-now-in-the-audit-log/)||✅|✅|✅|✅|✅| -|[Push protection events in the audit log](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-security-settings-for-your-organization/audit-log-events-for-your-organization#secret_scanning_push_protection-category-actions)|||✅|✅|✅|✅| -|[Push protection in the web editor](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/protecting-pushes-with-secret-scanning#using-secret-scanning-as-a-push-protection-from-the-web-ui)|||✅|✅|✅|✅| -|[Enable secret scanning at the enterprise level](https://github.blog/changelog/2022-10-06-enable-secret-scanning-for-an-enterprise-with-one-click/)||||✅|✅|✅| -|[Dry runs for secret scanning custom patterns (org level)](https://github.blog/changelog/2022-02-11-secret-scanning-dry-runs-for-repository-level-custom-pattern/)||||✅|✅|✅| -|[Email notification for push protection bypass](https://github.blog/changelog/2022-07-27-secret-scanning-admins-now-receive-emails-when-contributors-bypass-a-push-protection-block/)||||✅|✅|✅| -|[Custom links in push protection notification](https://github.blog/changelog/2022-08-24-secret-scanning-admins-can-now-provide-a-link-to-display-when-a-push-is-blocked/)||||✅|✅|✅| -|[View secret scanning enablement status at the org-level via API](https://github.blog/changelog/2021-08-24-secret-scanning-org-level-rest-api/)||||✅|✅|✅| -|[Enable secret scanning at the enterprise level using the REST API](https://github.blog/changelog/2022-12-13-enable-secret-scanning-with-the-enterprise-level-rest-api/)|||||✅|✅| -|[Add comment when dismissing a secret scanning alert in UI or API](https://github.blog/changelog/2022-09-29-secret-scanning-alerts-now-have-a-timeline-and-users-can-add-a-comment-when-resolving/)|||||✅|✅| -|[Custom pattern creation at the enterprise level](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||||||✅| - +|Partner pattern count|155|169|173|173|183|200|218| +|[User defined (custom) patterns](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)|✅|✅|✅|✅|✅|✅|✅| +|[Enterprise level API for secret scanning](https://docs.github.com/en/enterprise-server/rest/secret-scanning?apiVersion=2022-11-28#list-secret-scanning-alerts-for-an-enterprise)|✅|✅|✅|✅|✅|✅|✅| +|[Secret scanning push protection](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/protecting-pushes-with-secret-scanning)||✅|✅|✅|✅|✅|✅| +|[Dry runs for secret scanning push protection (repo level)](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||✅|✅|✅|✅|✅|✅| +|[Secret scanning support for archived repos](https://github.blog/changelog/2022-02-16-secret-scanning-now-supports-archived-repositories/)||✅|✅|✅|✅|✅|✅| +|[Custom pattern events in the audit log](https://github.blog/changelog/2022-04-06-secret-scanning-custom-pattern-events-now-in-the-audit-log/)||✅|✅|✅|✅|✅|✅| +|[Push protection events in the audit log](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-security-settings-for-your-organization/audit-log-events-for-your-organization#secret_scanning_push_protection-category-actions)|||✅|✅|✅|✅|✅| +|[Push protection in the web editor](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/protecting-pushes-with-secret-scanning#using-secret-scanning-as-a-push-protection-from-the-web-ui)|||✅|✅|✅|✅|✅| +|[Enable secret scanning at the enterprise level](https://github.blog/changelog/2022-10-06-enable-secret-scanning-for-an-enterprise-with-one-click/)||||✅|✅|✅|✅| +|[Dry runs for secret scanning custom patterns (org level)](https://github.blog/changelog/2022-02-11-secret-scanning-dry-runs-for-repository-level-custom-pattern/)||||✅|✅|✅|✅| +|[Email notification for push protection bypass](https://github.blog/changelog/2022-07-27-secret-scanning-admins-now-receive-emails-when-contributors-bypass-a-push-protection-block/)||||✅|✅|✅|✅| +|[Custom links in push protection notification](https://github.blog/changelog/2022-08-24-secret-scanning-admins-can-now-provide-a-link-to-display-when-a-push-is-blocked/)||||✅|✅|✅|✅| +|[View secret scanning enablement status at the org-level via API](https://github.blog/changelog/2021-08-24-secret-scanning-org-level-rest-api/)||||✅|✅|✅|✅| +|[Enable secret scanning at the enterprise level using the REST API](https://github.blog/changelog/2022-12-13-enable-secret-scanning-with-the-enterprise-level-rest-api/)|||||✅|✅|✅| +|[Add comment when dismissing a secret scanning alert in UI or API](https://github.blog/changelog/2022-09-29-secret-scanning-alerts-now-have-a-timeline-and-users-can-add-a-comment-when-resolving/)|||||✅|✅|✅| +|[Custom pattern creation at the enterprise level](https://docs.github.com/en/enterprise-server/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)||||||✅|✅| +|[Custom pattern alert metrics](https://docs.github.com/en/enterprise-server@3.10/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)|||||||✅| ## Code scanning Code scanning is a feature that you use to analyze the code in a GitHub repository to find security vulnerabilities and coding errors. Any problems identified by the analysis are shown in GitHub. @@ -62,23 +62,29 @@ Code scanning is a feature that you use to analyze the code in a GitHub reposito |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| -|[CodeQL "toolcache" Installed Version](https://docs.github.com/en/enterprise-server/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#provisioning-the-actions-for-code-scanning)|2.7.6|2.11.6|2.11.7|2.11.7|2.11.7|2.11.7|🚩| -|[Language support: Python, Javascript, Java, Go, C/C++, C#, Typescript](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|✅|✅|✅|✅|✅|✅| -|[Ruby Support](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|☑️|☑️|☑️|☑️|✅|✅| -|[Apple M1 support for CodeQL](https://github.blog/changelog/2021-11-10-codeql-now-supports-apple-silicon-m1/)|☑️|☑️|☑️|☑️|✅|✅| -|[Org-wide code scanning alerts via the REST API](https://docs.github.com/en/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-organization)||✅|✅|✅|✅|✅| -|[Add comments when dismissing alerts](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository#dismissing--alerts)|||✅|✅|✅|✅| -|[Code scanning alert comments in the pull request conversation tab](https://github.blog/changelog/2022-06-02-users-can-view-and-comment-on-code-scanning-alerts-on-the-conversation-tab-in-a-pull-request/)||||✅|✅|✅| -|[Users can publish CodeQL packs to the container registry](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/publishing-and-using-codeql-packs)||||✅|✅|✅| -|[CodeQL query filters to exclude individual queries](https://github.blog/changelog/2022-08-31-code-scanning-customize-your-codeql-analysis-using-query-filters/)||||✅|✅|✅| -|[Enterprise-wide code scanning alerts via the REST API](https://docs.github.com/en/enterprise-server/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-enterprise)||||✅|✅|✅| -|[Filter API results by severity](https://github.blog/changelog/2022-11-25-filter-code-scanning-api-results-by-alert-severity/)|||||✅|✅| -|[Kotlin language support](https://github.blog/changelog/2022-11-28-codeql-code-scanning-launches-kotlin-analysis-support-beta/)|||||☑️|☑️| -|[Default CodeQL setup](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-for-a-repository#configuring-code-scanning-automatically)||||||✅| -|[Default CodeQL setup via API](https://docs.github.com/en/enterprise-server/rest/code-scanning#update-a-code-scanning-default-setup-configuration)||||||✅| -|["Enable all" functionality at the org level (API and UI)](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-at-scale)||||||✅| -|[Tool status page](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-the-tool-status-page)||||||✅| -|[View org-level enablement status via the API ](https://docs.github.com/en/enterprise-server/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories)||||||✅| +|[CodeQL "toolcache" Installed Version](https://docs.github.com/en/enterprise-server/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#provisioning-the-actions-for-code-scanning)|2.7.6|2.11.6|2.11.7|2.11.7|2.11.7|2.11.7|2.13.5| +|[Language support: Python, Javascript, Java, Go, C/C++, C#, Typescript](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|✅|✅|✅|✅|✅|✅|✅| +|[Ruby Support](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/)|☑️|☑️|☑️|☑️|✅|✅|✅| +|[Apple M1 support for CodeQL](https://github.blog/changelog/2021-11-10-codeql-now-supports-apple-silicon-m1/)|☑️|☑️|☑️|☑️|✅|✅|✅| +|[Org-wide code scanning alerts via the REST API](https://docs.github.com/en/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-organization)||✅|✅|✅|✅|✅|✅| +|[Add comments when dismissing alerts](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository#dismissing--alerts)|||✅|✅|✅|✅|✅| +|[Code scanning alert comments in the pull request conversation tab](https://github.blog/changelog/2022-06-02-users-can-view-and-comment-on-code-scanning-alerts-on-the-conversation-tab-in-a-pull-request/)||||✅|✅|✅|✅| +|[Users can publish CodeQL packs to the container registry](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/publishing-and-using-codeql-packs)||||✅|✅|✅|✅| +|[CodeQL query filters to exclude individual queries](https://github.blog/changelog/2022-08-31-code-scanning-customize-your-codeql-analysis-using-query-filters/)||||✅|✅|✅|✅| +|[Enterprise-wide code scanning alerts via the REST API](https://docs.github.com/en/enterprise-server/rest/code-scanning?apiVersion=2022-11-28#list-code-scanning-alerts-for-an-enterprise)||||✅|✅|✅|✅| +|[Filter API results by severity](https://github.blog/changelog/2022-11-25-filter-code-scanning-api-results-by-alert-severity/)|||||✅|✅|✅| +|[Kotlin language support](https://github.blog/changelog/2022-11-28-codeql-code-scanning-launches-kotlin-analysis-support-beta/)|||||☑️|☑️|☑️| +|[Default CodeQL setup](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-for-a-repository#configuring-code-scanning-automatically)||||||✅|✅| +|[Default CodeQL setup via API](https://docs.github.com/en/enterprise-server/rest/code-scanning#update-a-code-scanning-default-setup-configuration)||||||✅|✅| +|["Enable all" functionality at the org level (API and UI)](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning-at-scale)||||||✅|✅| +|[Tool status page](https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-the-tool-status-page)||||||✅|✅| +|[View org-level enablement status via the API ](https://docs.github.com/en/enterprise-server/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories)||||||✅|✅| +|[CodeQL default setup supports compiled languages](https://docs.github.com/en/enterprise-server@3.10/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages)|||||||✅| +|[Choose which language to enable or disable in CodeQL default setup](https://docs.github.com/en/enterprise-server@3.10/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-default-setup-for-code-scanning)|||||||✅| +|[Filter code scanning alerts by `path` and `language`](https://docs.github.com/en/enterprise-server@3.10/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository)|||||||✅| +|[CodeQL supports C# 11](https://github.com/github/roadmap/issues/598)|||||||✅| +|[CodeQL supports Swift programming language](https://github.blog/changelog/2023-06-01-codeql-code-scanning-now-supports-swift-beta/)|||||||☑️| + @@ -92,31 +98,33 @@ Dependabot alerts tell you that your code depends on a package that is insecure. |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| -|[Dependabot Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts)|✅|✅|✅|✅|✅|✅|🚩| -|[Go modules support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| -|[Poetry support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅| -|[Cargo support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|||✅|✅|✅|✅| -|[Reopen dismissed alerts](https://github.blog/changelog/2022-03-07-reopen-dismissed-dependabot-alerts/)|||✅|✅|✅|✅| -|[Dependabot alerts show vulnerable function calls](https://github.blog/2022-04-14-dependabot-alerts-now-surface-if-code-is-calling-vulnerability/)|||☑️|☑️|☑️|☑️| -|[Dependabot Alert timeline](https://github.blog/changelog/2022-07-28-dependabot-alerts-timeline-of-events-on-the-alert-details-page/)||||✅|✅|✅| -|[Bulk Editing of Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/viewing-and-updating-dependabot-alerts)||||✅|✅|✅| -|[Add comment when dismissing dependabot alert](https://github.blog/changelog/2022-08-22-dependabot-alerts-optional-dismissal-comment-2/)||||✅|✅|✅| -|[Dev Dependencies label](https://github.blog/2023-05-02-dependabot-relieves-alert-fatigue-from-npm-devdependencies/) ||||✅|✅|✅| -|[View Dependabot enablement status via org-level API](https://github.blog/changelog/2023-02-28-dependabot-alerts-enterprise-enablement-and-status-checking/)||||✅|✅|✅| -|[Receive alerts for vulnerable GitHub Actions](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot)||||✅|✅|✅| -|[Dependabot alert webhooks](https://github.blog/changelog/2022-10-06-new-dependabot-alerts-webhook/)||||✅|✅|✅| -|[Dependabot alerts REST API endpoint for repository org and enterprise](https://docs.github.com/en/rest/dependabot/alerts?apiVersion=2022-11-28)|||||☑️|✅| -|[Export SBOM from dependency graph](https://docs.github.com/en/enterprise-server/code-security/supply-chain-security/understanding-your-software-supply-chain/exporting-a-software-bill-of-materials-for-your-repository)||||||✅| -|[Dependabot can parse and update Gradle version catalogs in `settings.gradle`](https://docs.github.com/en/enterprise-server/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates)||||||✅| +|[Dependabot Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/about-dependabot-alerts)|✅|✅|✅|✅|✅|✅|✅| +|[Go modules support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅|✅| +|[Poetry support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|✅|✅|✅|✅|✅|✅|✅| +|[Cargo support](https://docs.github.com/en/enterprise-server/get-started/learning-about-github/github-language-support#core-languages-supported-by-github-features)|||✅|✅|✅|✅|✅| +|[Reopen dismissed alerts](https://github.blog/changelog/2022-03-07-reopen-dismissed-dependabot-alerts/)|||✅|✅|✅|✅|✅| +|[Dependabot alerts show vulnerable function calls](https://github.blog/2022-04-14-dependabot-alerts-now-surface-if-code-is-calling-vulnerability/)|||☑️|☑️|☑️|☑️|☑️| +|[Dependabot Alert timeline](https://github.blog/changelog/2022-07-28-dependabot-alerts-timeline-of-events-on-the-alert-details-page/)||||✅|✅|✅|✅| +|[Bulk Editing of Alerts](https://docs.github.com/en/code-security/dependabot/dependabot-alerts/viewing-and-updating-dependabot-alerts)||||✅|✅|✅|✅| +|[Add comment when dismissing dependabot alert](https://github.blog/changelog/2022-08-22-dependabot-alerts-optional-dismissal-comment-2/)||||✅|✅|✅|✅| +|[Dev Dependencies label](https://github.blog/2023-05-02-dependabot-relieves-alert-fatigue-from-npm-devdependencies/) ||||✅|✅|✅|✅| +|[View Dependabot enablement status via org-level API](https://github.blog/changelog/2023-02-28-dependabot-alerts-enterprise-enablement-and-status-checking/)||||✅|✅|✅|✅| +|[Receive alerts for vulnerable GitHub Actions](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot)||||✅|✅|✅|✅| +|[Dependabot alert webhooks](https://github.blog/changelog/2022-10-06-new-dependabot-alerts-webhook/)||||✅|✅|✅|✅| +|[Dependabot alerts REST API endpoint for repository org and enterprise](https://docs.github.com/en/rest/dependabot/alerts?apiVersion=2022-11-28)|||||☑️|✅|✅| +|[Export SBOM from dependency graph](https://docs.github.com/en/enterprise-server/code-security/supply-chain-security/understanding-your-software-supply-chain/exporting-a-software-bill-of-materials-for-your-repository)||||||✅|✅| +|[Dependabot can parse and update Gradle version catalogs in `settings.gradle`](https://docs.github.com/en/enterprise-server/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates)||||||✅|✅| + #### Dependabot Updates |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| -|[Dependabot Updates](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuring-dependabot-version-updates)|☑️|✅|✅|✅|✅|✅|🚩| -|Actions authors can automatically update dependencies within workflow files|||||✅|✅| -|Dart and Flutter (using Pub) support for updates|||||✅|✅| -|[Automatically pause pull request activity after 90 days of inactivity](https://docs.github.com/en/enterprise-server/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates#about-automatic-deactivation-of-dependabot-updates)||||||✅| +|[Dependabot Updates](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuring-dependabot-version-updates)|☑️|✅|✅|✅|✅|✅|✅| +|Actions authors can automatically update dependencies within workflow files|||||✅|✅|✅| +|Dart and Flutter (using Pub) support for updates|||||✅|✅|✅| +|[Automatically pause pull request activity after 90 days of inactivity](https://docs.github.com/en/enterprise-server/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates#about-automatic-deactivation-of-dependabot-updates)||||||✅|✅| +|[Dependabot updates supports pnpm](https://github.blog/changelog/2023-06-12-dependabot-version-updates-now-supports-pnpm/)|||||||✅| #### Dependency Review and submission API Dependency review helps you understand dependency changes and the security impact of these changes at every pull request. @@ -125,9 +133,9 @@ Dependency review helps you understand dependency changes and the security impac |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 | |------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| -|[Dependency Review](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/configuring-dependency-review)|✅|✅|✅|✅|✅|✅|🚩| -|[Enforcement Action](https://github.blog/changelog/2022-04-06-github-action-for-dependency-review-enforcement/)|||✅|✅|✅|✅| -|[Dependency Submission API](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/using-the-dependency-submission-api)||||✅|✅|✅| +|[Dependency Review](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/configuring-dependency-review)|✅|✅|✅|✅|✅|✅|✅| +|[Enforcement Action](https://github.blog/changelog/2022-04-06-github-action-for-dependency-review-enforcement/)|||✅|✅|✅|✅|✅| +|[Dependency Submission API](https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/using-the-dependency-submission-api)||||✅|✅|✅|✅| ## Security Overview @@ -136,27 +144,29 @@ Security overview provides high-level summaries of the security status of an org |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10| |------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| -|[Security Overview](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview)|✅|✅|✅|✅|✅|✅|🚩| -|Organization view|☑️|✅|✅|✅|✅|✅| -|Enterprise view||☑️|☑️|✅|✅|✅| -|Organization-level Code Scanning Alert View||✅|✅|✅|✅|✅| -|Organization-level Dependabot Alert View||✅|✅|✅|✅|✅| -|Enterprse-level view of Dependabot alerts|||✅|✅|✅|✅| -|Enterprse-level view of code scanning alerts||||✅|✅|✅| -|Enterprse-level view of secret scanning alerts||||✅|✅|✅| -|Coverage and Risk Security Overview pages|||||☑️|☑️| -|[Filter alerts by repo topic](https://docs.github.com/en/enterprise-server/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅| -|[Filter alerts by team](https://docs.github.com/en/enterprise-server/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅| -|[Enable GHAS features in security overview](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview)||||||✅| - +|[Security Overview](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview)|✅|✅|✅|✅|✅|✅|✅| +|Organization view|☑️|✅|✅|✅|✅|✅|✅| +|Enterprise view||☑️|☑️|✅|✅|✅|✅| +|Organization-level Code Scanning Alert View||✅|✅|✅|✅|✅|✅| +|Organization-level Dependabot Alert View||✅|✅|✅|✅|✅|✅| +|Enterprse-level view of Dependabot alerts|||✅|✅|✅|✅|✅| +|Enterprse-level view of code scanning alerts||||✅|✅|✅|✅| +|Enterprse-level view of secret scanning alerts||||✅|✅|✅|✅| +|Coverage and Risk Security Overview pages|||||☑️|☑️|✅| +|[Filter alerts by repo topic](https://docs.github.com/en/enterprise-server/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅|✅| +|[Filter alerts by team](https://docs.github.com/en/enterprise-server/code-security/security-overview/filtering-alerts-in-security-overview)||||||✅|✅| +|[Enable GHAS features in security overview](https://docs.github.com/en/enterprise-server/code-security/security-overview/about-security-overview)||||||✅|✅| +|[Enterprise-level security coverage and risk dashboards](https://docs.github.com/en/enterprise-server@3.10/code-security/security-overview/about-security-overview#about-security-overview-for-enterprises)|||||||✅| ## Administration |Feature |3.4 |3.5 |3.6 |3.7 |3.8 |3.9 |3.10 |------------------------------------------------------------|-----|-----|-----|-----|-----|-----|----| -|[Security Managers Role](https://docs.github.com/en/enterprise-server/organizations/managing-peoples-access-to-your-organization-with-roles/managing-security-managers-in-your-organization)|✅|✅|✅|✅|✅|✅|🚩| -|[Manage Security Managers role via the API](https://docs.github.com/en/enterprise-server/rest/orgs/security-managers?apiVersion=2022-11-28)||||✅|✅|✅| +|[Security Managers Role](https://docs.github.com/en/enterprise-server/organizations/managing-peoples-access-to-your-organization-with-roles/managing-security-managers-in-your-organization)|✅|✅|✅|✅|✅|✅|✅| +|[Manage Security Managers role via the API](https://docs.github.com/en/enterprise-server/rest/orgs/security-managers?apiVersion=2022-11-28)||||✅|✅|✅|✅| + +# Dependencies +This section calls out the dependencies required to enable GitHub Advanced Security on GitHub Enterprise Server. -## Dependencies | Feature | GHAS license
required? | GitHub Actions
required? | GitHub Connect
required? | Documentation | Notes | |---|---|---|---|---|---| | Security Overview

DescriptionKnow what needs attention throughout the entire SDLC
| No * | No | No | [Feature Docs](https://docs.github.com/en/enterprise-server@latest/code-security/security-overview/about-the-security-overview) | * Features not needing a GHAS license will still show up | From bd05c22c67a1a1d3b7195b19f3271fa542f006c8 Mon Sep 17 00:00:00 2001 From: Nikita Kraiouchkine Date: Wed, 30 Aug 2023 16:26:05 +0200 Subject: [PATCH 128/128] Create code-scanning-codeql-cli-example-00.yml --- .../code-scanning-codeql-cli-example-00.yml | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 code-scanning-workflows/code-scanning-codeql-cli-example-00.yml diff --git a/code-scanning-workflows/code-scanning-codeql-cli-example-00.yml b/code-scanning-workflows/code-scanning-codeql-cli-example-00.yml new file mode 100644 index 0000000..8f9bebf --- /dev/null +++ b/code-scanning-workflows/code-scanning-codeql-cli-example-00.yml @@ -0,0 +1,127 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '20 21 * * 3' + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go', 'java', 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Install Java if necessary + - if: matrix.language == 'java' + name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: 'adopt' + java-version: '15' + + # Initialize the CodeQL tools for scanning. + - name: Initialize CodeQL + run: | + gh extensions install github/gh-codeql + gh codeql set-version latest + gh codeql pack download codeql/${{ matrix.language }}-queries + env: + GH_TOKEN: ${{ github.token }} + + # Create a CodeQL database and start tracing for compiled languages + - name: Create CodeQL Database + run: | + gh codeql database init --begin-tracing --language=${{ matrix.language }} --source-root=${{ env.GITHUB_WORKSPACE }} ${{ matrix.language }}-db + env: + GH_TOKEN: ${{ github.token }} + + - if: matrix.language == 'java' + name: Build Java Code + run: | + source ../${{ matrix.language }}-db/temp/tracingEnvironment/start-tracing.sh + mvn clean install + working-directory: ./storage-service + + - if: matrix.language == 'go' + name: Build Go Code + run: | + source ../${{ matrix.language }}-db/temp/tracingEnvironment/start-tracing.sh + go build + working-directory: ./gallery-service + + - name: Traceless Database Build (Python/JS) + if: contains(fromJSON('["javascript", "python"]'), ${{ matrix.language }}) + run: | + gh codeql database trace-command --index-traceless-dbs ${{ matrix.language }}-db + env: + GH_TOKEN: ${{ github.token }} + + # Finalize the database + - name: Finalize database + run: | + gh codeql database finalize ${{ matrix.language }}-db + env: + GH_TOKEN: ${{ github.token }} + + # The --sarif-category must be set for each language's database + - name: Analyze database + run: | + gh codeql database analyze \ + --format="sarif-latest" \ + --sarif-category="codeql-scan:${{ matrix.language }}" \ + --output=${{ matrix.language }}-db.sarif \ + -j=0 \ + --sarif-add-query-help --sarif-add-snippets \ + ${{matrix.language}}-db + env: + GH_TOKEN: ${{ github.token }} + + # Upload the CodeQL scan results + - name: Upload results + run: | + echo ${{ github.token }} | \ + gh codeql github upload-results \ + --sarif=${{ matrix.language }}-db.sarif \ + --repository=$GITHUB_REPOSITORY \ + --ref=$GITHUB_REF \ + --commit=$GITHUB_SHA \ + --github-auth-stdin + env: + GH_TOKEN: ${{ github.token }}