Merge branch 'main' into 19530-use-goreleaser-v2

fleetdm · Dec 3, 2024 · 240e639 · 240e639
2 parents 2de07d3 + a0b03b0
commit 240e639
Show file tree

Hide file tree

Showing 630 changed files with 12,900 additions and 30,749 deletions.
diff --git a/.github/workflows/check-tuf-timestamps.yml b/.github/workflows/check-tuf-timestamps.yml
@@ -3,10 +3,10 @@ name: Check TUF timestamps
 on:
   pull_request:
     paths:
-      - '.github/workflows/check-tuf-timestamps.yml'
+      - ".github/workflows/check-tuf-timestamps.yml"
   workflow_dispatch: # Manual
   schedule:
-    - cron: '0 10,22 * * *'
+    - cron: "0 10,22 * * *"
 
 # This allows a subsequently queued workflow run to interrupt previous runs
 concurrency:
@@ -29,81 +29,151 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-    - name: Harden Runner
-      uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
-      with:
-        egress-policy: audit
-
-    - name: Check remote timestamp.json file
-      id: check_timestamp
-      run: |
-        expires=$(curl -s http://tuf.fleetctl.com/timestamp.json | jq -r '.signed.expires' | cut -c 1-10)
-        today=$(date "+%Y-%m-%d")
-        warning_at=$(date -d "$today + 4 day" "+%Y-%m-%d")
-        expires_sec=$(date -d "$expires" "+%s")
-        warning_at_sec=$(date -d "$warning_at" "+%s")
-
-        if [ "$expires_sec" -le "$warning_at_sec" ]; then
-            echo "timestamp_warn=true" >> ${GITHUB_OUTPUT}
-        else
-            echo "timestamp_warn=false" >> ${GITHUB_OUTPUT}
-        fi
-
-    - name: Check remote root.json file
-      id: check_root
-      run: |
-        expires=$(curl -s http://tuf.fleetctl.com/root.json | jq -r '.signed.expires' | cut -c 1-10)
-        today=$(date "+%Y-%m-%d")
-        warning_at=$(date -d "$today + 30 day" "+%Y-%m-%d")
-        expires_sec=$(date -d "$expires" "+%s")
-        warning_at_sec=$(date -d "$warning_at" "+%s")
-
-        if [ "$expires_sec" -le "$warning_at_sec" ]; then
-            echo "root_warn=true" >> ${GITHUB_OUTPUT}
-        else
-            echo "root_warn=false" >> ${GITHUB_OUTPUT}
-        fi
-
-
-    - name: Slack Timestamp Notification
-      if: ${{ steps.check_timestamp.outputs.timestamp_warn  == 'true' }}
-      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
-      with:
-        payload: |
-          {
-            "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
-            "blocks": [
-              {
-                "type": "section",
-                "text": {
-                  "type": "mrkdwn",
-                  "text": "⚠️ TUF timestamp.json is about to expire or has already expired\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}"
+      - name: Harden Runner
+        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
+        with:
+          egress-policy: audit
+
+      - name: Check remote timestamp.json file
+        id: check_timestamp
+        run: |
+          expires=$(curl -s http://tuf.fleetctl.com/timestamp.json | jq -r '.signed.expires' | cut -c 1-10)
+          today=$(date "+%Y-%m-%d")
+          warning_at=$(date -d "$today + 4 day" "+%Y-%m-%d")
+          expires_sec=$(date -d "$expires" "+%s")
+          warning_at_sec=$(date -d "$warning_at" "+%s")
+
+          if [ "$expires_sec" -le "$warning_at_sec" ]; then
+              echo "timestamp_warn=true" >> ${GITHUB_OUTPUT}
+          else
+              echo "timestamp_warn=false" >> ${GITHUB_OUTPUT}
+          fi
+
+      - name: Check remote snapshot.json file
+        id: check_snapshot
+        run: |
+          expires=$(curl -s http://tuf.fleetctl.com/snapshot.json | jq -r '.signed.expires' | cut -c 1-10)
+          today=$(date "+%Y-%m-%d")
+          warning_at=$(date -d "$today + 30 day" "+%Y-%m-%d")
+          expires_sec=$(date -d "$expires" "+%s")
+          warning_at_sec=$(date -d "$warning_at" "+%s")
+
+          if [ "$expires_sec" -le "$warning_at_sec" ]; then
+              echo "snapshot_warn=true" >> ${GITHUB_OUTPUT}
+          else
+              echo "snapshot_warn=false" >> ${GITHUB_OUTPUT}
+          fi
+
+      - name: Check remote targets.json file
+        id: check_targets
+        run: |
+          expires=$(curl -s http://tuf.fleetctl.com/targets.json | jq -r '.signed.expires' | cut -c 1-10)
+          today=$(date "+%Y-%m-%d")
+          warning_at=$(date -d "$today + 30 day" "+%Y-%m-%d")
+          expires_sec=$(date -d "$expires" "+%s")
+          warning_at_sec=$(date -d "$warning_at" "+%s")
+
+          if [ "$expires_sec" -le "$warning_at_sec" ]; then
+              echo "targets_warn=true" >> ${GITHUB_OUTPUT}
+          else
+              echo "targets_warn=false" >> ${GITHUB_OUTPUT}
+          fi
+
+      - name: Check remote root.json file
+        id: check_root
+        run: |
+          expires=$(curl -s http://tuf.fleetctl.com/root.json | jq -r '.signed.expires' | cut -c 1-10)
+          today=$(date "+%Y-%m-%d")
+          warning_at=$(date -d "$today + 30 day" "+%Y-%m-%d")
+          expires_sec=$(date -d "$expires" "+%s")
+          warning_at_sec=$(date -d "$warning_at" "+%s")
+
+          if [ "$expires_sec" -le "$warning_at_sec" ]; then
+              echo "root_warn=true" >> ${GITHUB_OUTPUT}
+          else
+              echo "root_warn=false" >> ${GITHUB_OUTPUT}
+          fi
+
+      - name: Slack timestamp notification
+        if: ${{ steps.check_timestamp.outputs.timestamp_warn  == 'true' }}
+        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+        with:
+          payload: |
+            {
+              "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "⚠️ TUF timestamp.json is about to expire or has already expired\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+      - name: Slack snapshot notification
+        if: ${{ steps.check_snapshot.outputs.snapshot_warn  == 'true' }}
+        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+        with:
+          payload: |
+            {
+              "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "⚠️ TUF snapshot.json is about to expire or has already expired\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}"
+                  }
                 }
-              }
-            ]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
-        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
-
-    - name: Slack Root Notification
-      if: ${{ steps.check_root.outputs.root_warn  == 'true' }}
-      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
-      with:
-        payload: |
-          {
-            "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
-            "blocks": [
-              {
-                "type": "section",
-                "text": {
-                  "type": "mrkdwn",
-                  "text": "⚠️ TUF root.json is about to expire or has already expired\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}"
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+      - name: Slack targets notification
+        if: ${{ steps.check_targets.outputs.targets_warn  == 'true' }}
+        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+        with:
+          payload: |
+            {
+              "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "⚠️ TUF targets.json is about to expire or has already expired\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}"
+                  }
                 }
-              }
-            ]
-          }
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
-        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
+      - name: Slack root notification
+        if: ${{ steps.check_root.outputs.root_warn  == 'true' }}
+        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+        with:
+          payload: |
+            {
+              "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "⚠️ TUF root.json is about to expire or has already expired\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
diff --git a/.github/workflows/dogfood-gitops.yml b/.github/workflows/dogfood-gitops.yml
@@ -48,13 +48,6 @@ jobs:
           ref: main
           path: fleet-gitops
 
-      - name: Apply env vars to profiles
-        env:
-          MANAGED_CHROME_ENROLLMENT_TOKEN: ${{ secrets.CLOUD_MANAGEMENT_ENROLLMENT_TOKEN }}
-        run: |
-          envsubst < ./it-and-security/lib/configuration-profiles/macos-chrome-enrollment.mobileconfig > ./it-and-security/lib/configuration-profiles/macos-chrome-enrollment.confidential.mobileconfig
-          mv ./it-and-security/lib/configuration-profiles/macos-chrome-enrollment.confidential.mobileconfig ./it-and-security/lib/configuration-profiles/macos-chrome-enrollment.mobileconfig
-
       - name: Apply latest configuration to Fleet
         uses: ./fleet-gitops/.github/gitops-action
         with:
@@ -81,4 +74,5 @@ jobs:
           DOGFOOD_CALENDAR_API_KEY: ${{ secrets.DOGFOOD_CALENDAR_API_KEY }}
           DOGFOOD_COMPLIANCE_EXCLUSIONS_ENROLL_SECRET: ${{ secrets.DOGFOOD_COMPLIANCE_EXCLUSIONS_ENROLL_SECRET }}
           DOGFOOD_COMPANY_OWNED_IPHONES_ENROLL_SECRET: ${{ secrets.DOGFOOD_COMPANY_OWNED_IPHONES_ENROLL_SECRET }}
-          DOGFOOD_COMPANY_OWNED_IPADS_ENROLL_SECRET: ${{ secrets.DOGFOOD_COMPANY_OWNED_IPADS_ENROLL_SECRET }}
+          DOGFOOD_COMPANY_OWNED_IPADS_ENROLL_SECRET: ${{ secrets.DOGFOOD_COMPANY_OWNED_IPADS_ENROLL_SECRET }}
+          MANAGED_CHROME_ENROLLMENT_TOKEN: ${{ secrets.CLOUD_MANAGEMENT_ENROLLMENT_TOKEN }}
diff --git a/.github/workflows/generate-desktop-targets.yml b/.github/workflows/generate-desktop-targets.yml
@@ -19,20 +19,17 @@ defaults:
     shell: bash
 
 env:
-  FLEET_DESKTOP_VERSION: 1.35.0
+  FLEET_DESKTOP_VERSION: 1.36.0
 
 permissions:
   contents: read
 
 jobs:
   desktop-macos:
-    # Set macOS version to '12' (current equivalent to macos-latest) for
+    # Set macOS version to '13' (previously was macos-12, and it was deprecated) for
     # building the binary. This ensures compatibility with macOS version 13 and
     # later, avoiding runtime errors on systems using macOS 13 or newer.
-    #
-    # Note: Update this version to '13' once GitHub marks macOS 13 as stable
-    # or if we revise our minimum supported macOS version.
-    runs-on: macos-12
+    runs-on: macos-13
     steps:
 
       - name: Harden Runner

diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml
@@ -186,9 +186,112 @@ jobs:
         name: ${{ matrix.suite }}-${{ env.MATRIX_MYSQL_ID }}-summary-test-log
         path: /tmp/summary.txt
 
+  # Based on https://github.com/micromdm/nanomdm/blob/main/.github/workflows/on-push-pr.yml#L87
+  test-go-nanomdm:
+    runs-on: 'ubuntu-latest'
+    services:
+      mysql:
+        image: mysql:8.0.36
+        env:
+          MYSQL_RANDOM_ROOT_PASSWORD: yes
+          MYSQL_DATABASE: testdb
+          MYSQL_USER: testuser
+          MYSQL_PASSWORD: testpw
+        ports:
+          - 3800:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+    env:
+      MYSQL_PWD: testpw
+      PORT: 3800
+      RACE_ENABLED: true
+      GO_TEST_TIMEOUT: 20m
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
+        with:
+          egress-policy: audit
+
+      - name: Checkout Code
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Install Go
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: 'go.mod'
+
+      - name: verify mysql
+        run: |
+          while ! mysqladmin ping --host=localhost --port=$PORT --protocol=TCP --silent; do
+            sleep 1
+          done
+
+      - name: mysql schema
+        run: |
+          mysql --version
+          mysql --user=testuser --host=localhost --port=$PORT --protocol=TCP testdb < ./server/mdm/nanomdm/storage/mysql/schema.sql
+
+      - name: set test dsn
+        run: echo "NANOMDM_MYSQL_STORAGE_TEST_DSN=testuser:testpw@tcp(localhost:$PORT)/testdb" >> $GITHUB_ENV
+
+      - name: Run Go tests
+        run: |
+          go test -v -parallel 8 -race=$RACE_ENABLED -timeout=$GO_TEST_TIMEOUT \
+            -coverprofile=coverage.txt -covermode=atomic -coverpkg=github.com/fleetdm/fleet/v4/server/mdm/nanomdm/... \
+          ./server/mdm/nanomdm/storage/mysql 2>&1 | tee /tmp/gotest.log
+
+      - name: Save coverage
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        with:
+          name: nanomdm-coverage
+          path: ./coverage.txt
+          if-no-files-found: error
+
+      - name: Generate summary of errors
+        if: failure()
+        run: |
+          c1grep() { grep "$@" || test $? = 1; }
+          c1grep -oP 'FAIL: .*$' /tmp/gotest.log > /tmp/summary.txt
+          c1grep 'test timed out after' /tmp/gotest.log >> /tmp/summary.txt
+          c1grep 'fatal error:' /tmp/gotest.log >> /tmp/summary.txt
+          c1grep -A 10 'panic: runtime error: ' /tmp/gotest.log >> /tmp/summary.txt
+          c1grep ' FAIL\t' /tmp/gotest.log >> /tmp/summary.txt
+          GO_FAIL_SUMMARY=$(head -n 5 /tmp/summary.txt | sed ':a;N;$!ba;s/\n/\\n/g')
+          echo "GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY"
+          if [[ -z "$GO_FAIL_SUMMARY" ]]; then
+            GO_FAIL_SUMMARY="unknown, please check the build URL"
+          fi
+          GO_FAIL_SUMMARY=$GO_FAIL_SUMMARY envsubst < .github/workflows/config/slack_payload_template.json > ./payload.json
+
+      - name: Slack Notification
+        if: github.event.schedule == '0 4 * * *' && failure()
+        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+        with:
+          payload-file-path: ./payload.json
+        env:
+          JOB_STATUS: ${{ job.status }}
+          EVENT_URL: ${{ github.event.pull_request.html_url || github.event.head.html_url }}
+          RUN_URL: https://github.com/fleetdm/fleet/actions/runs/${{ github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_G_HELP_ENGINEERING_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+      - name: Upload test log
+        if: always()
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        with:
+          name: nanomdm-test-log
+          path: /tmp/gotest.log
+          if-no-files-found: error
+
+      - name: Upload summary test log
+        if: always()
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  # v4.3.6
+        with:
+          name: nanomdm-summary-test-log
+          path: /tmp/summary.txt
+
   # We upload all backend coverage in one step so that we're less like to end up in a situation with a partial coverage report.
   upload-coverage:
-    needs: [test-go]
+    needs: [test-go, test-go-nanomdm]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code