LLVM and SPIRV-LLVM-Translator pulldown (WW04)

.ci/generate-buildkite-pipeline-premerge

@@ -9,7 +9,7 @@
 
 #
 # This file generates a Buildkite pipeline that triggers the various CI jobs for
-# the LLVM project during pre-commit CI (each time a Phabricator diff is uploaded).
+# the LLVM project during pre-commit CI.
 #
 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
 #
@@ -30,8 +30,6 @@ git fetch origin main:main
 : ${WINDOWS_AGENTS:='{"queue": "windows"}'}
 # Filter rules for generic linux tests
 : ${LINUX_AGENTS:='{"queue": "linux"}'}
-# Service agents, for interacting with Phabricator.
-: ${SERVICE_AGENTS:='{"queue": "service"}'}
 # Set by buildkite
 : ${BUILDKITE_COMMIT:=}
 : ${BUILDKITE_BRANCH:=}
@@ -284,28 +282,3 @@ if [[ "${windows_projects}" != "" ]]; then
   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
 EOF
 fi
-
-# If build was triggered from a Phabricator review - send an update back.
-if [[ -n "${ph_target_phid:-}" ]]; then
-  cat << EOF
-- continue_on_failure: true
-  wait: '~'
-- label: ':phabricator: update build status on Phabricator'
-  agents: ${SERVICE_AGENTS}
-  artifact_paths:
-  - 'artifacts/**/*'
-  commands:
-  - export SRC=\$\${BUILDKITE_BUILD_PATH}/llvm-premerge-checks
-  - rm -rf \$\${SRC}
-  - git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "\$\${SRC}"
-  - cd \$\${SRC}
-  - git fetch origin "main":x
-  - git checkout x
-  - echo "llvm-premerge-checks commit"
-  - git rev-parse HEAD
-  - pip install -q -r \$\${SRC}/scripts/requirements.txt
-  - cd "\$\$BUILDKITE_BUILD_CHECKOUT_PATH"
-  - \$\${SRC}/scripts/summary.py
-  timeout_in_minutes: 10
-EOF
-fi

.github/new-prs-labeler.yml

@@ -308,6 +308,9 @@ clang-tidy:
   - clang-tools-extra/docs/clang-tidy/**
   - clang-tools-extra/test/clang-tidy/**
 
+clang-tools-extra:
+  - clang-tools-extra/**
+
 tools:llvm-mca:
   - llvm/tools/llvm-mca/**
   - llvm/include/llvm/MCA/**
@@ -591,13 +594,16 @@ mlgo:
   - llvm/include/llvm/Analysis/*Runner.h
   - llvm/unittests/Analysis/ML*
   - llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
+  - llvm/lib/Analysis/TrainingLogger.cpp
   - llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
+  - llvm/include/llvm/Analysis/Utils/TrainingLogger.h
   - llvm/test/Analysis/FunctionPropertiesAnalysis/*
   - llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
   - llvm/test/Transforms/inline/ML/**
   - llvm/lib/CodeGen/ML*
   - llvm/unittests/CodeGen/ML*
   - llvm/test/CodeGen/MLRegAlloc/**
+  - llvm/utils/mlgo-utils/*
 
 tools:llvm-exegesis:
   - llvm/tools/llvm-exegesis/**

.github/workflows/issue-release-workflow.yml

@@ -33,6 +33,8 @@ jobs:
   backport-commits:
     name: Backport Commits
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
     if: >-
       (github.repository == 'llvm/llvm-project') &&
       !startswith(github.event.comment.body, '<!--IGNORE-->') &&
@@ -58,15 +60,18 @@ jobs:
           printf "%s" "$COMMENT_BODY" |
           ./llvm/utils/git/github-automation.py \
           --repo "$GITHUB_REPOSITORY" \
-          --token ${{ secrets.RELEASE_WORKFLOW_PUSH_SECRET }} \
+          --token ${{ github.token }} \
           release-workflow \
+          --branch-repo-token ${{ secrets.RELEASE_WORKFLOW_PUSH_SECRET }} \
           --issue-number ${{ github.event.issue.number }} \
-          --phab-token ${{ secrets.RELEASE_WORKFLOW_PHAB_TOKEN }} \
           auto
 
   create-pull-request:
     name: Create Pull Request
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     if: >-
       (github.repository == 'llvm/llvm-project') &&
       !startswith(github.event.comment.body, '<!--IGNORE-->') &&
@@ -87,8 +92,8 @@ jobs:
           printf "%s" "$COMMENT_BODY" |
           ./llvm/utils/git/github-automation.py \
           --repo "$GITHUB_REPOSITORY" \
-          --token ${{ secrets.RELEASE_WORKFLOW_PUSH_SECRET }} \
+          --token ${{ github.token }} \
           release-workflow \
+          --branch-repo-token ${{ secrets.RELEASE_WORKFLOW_PUSH_SECRET }} \
           --issue-number ${{ github.event.issue.number }} \
-          --phab-token ${{ secrets.RELEASE_WORKFLOW_PHAB_TOKEN }} \
           auto

.github/workflows/issue-subscriber.yml

@@ -22,7 +22,6 @@ jobs:
       - name: Setup Automation Script
         working-directory: ./llvm/utils/git/
         run: |
-          chmod a+x github-automation.py
           pip install -r requirements.txt
 
       - name: Update watchers
@@ -31,7 +30,7 @@ jobs:
         env:
           LABEL_NAME: ${{ github.event.label.name }}
         run: |
-          ./github-automation.py \
+          python3 ./github-automation.py \
             --token '${{ secrets.ISSUE_SUBSCRIBER_TOKEN }}' \
             issue-subscriber \
             --issue-number '${{ github.event.issue.number }}' \

.github/workflows/libclang-python-tests.yml

@@ -6,6 +6,8 @@ permissions:
 on:
   workflow_dispatch:
   push:
+    branches:
+      - 'main'
     paths:
       - 'clang/bindings/python/**'
       - 'clang/tools/libclang/**'
@@ -30,6 +32,7 @@ jobs:
   check-clang-python:
     # Build libclang and then run the libclang Python binding's unit tests.
     name: Build and run Python unit tests
+    if: github.repository == 'llvm/llvm-project'
     strategy:
       fail-fast: false
       matrix:

.github/workflows/libcxx-build-and-test.yaml

@@ -23,8 +23,8 @@ on:
       - 'cmake/**'
       - '.github/workflows/libcxx-build-and-test.yaml'
   schedule:
-    # Run nightly at 8 AM UTC (or roughly 3 AM eastern)
-    - cron: '0 3 * * *'
+    # Run nightly at 08:00 UTC (aka 00:00 Pacific, aka 03:00 Eastern)
+    - cron: '0 8 * * *'
 
 permissions:
   contents: read # Default everything to read-only

.github/workflows/new-prs.yml

@@ -43,13 +43,12 @@ jobs:
       - name: Setup Automation Script
         working-directory: ./llvm/utils/git/
         run: |
-          chmod a+x github-automation.py
           pip install -r requirements.txt
 
       - name: Greet Author
         working-directory: ./llvm/utils/git/
         run: |
-          ./github-automation.py \
+          python3 ./github-automation.py \
             --token '${{ secrets.GITHUB_TOKEN }}' \
             pr-greeter \
             --issue-number "${{ github.event.pull_request.number }}"

.github/workflows/pr-subscriber.yml

@@ -22,13 +22,12 @@ jobs:
       - name: Setup Automation Script
         working-directory: ./llvm/utils/git/
         run: |
-          chmod a+x github-automation.py
           pip install -r requirements.txt
 
       - name: Update watchers
         working-directory: ./llvm/utils/git/
         run: |
-          ./github-automation.py \
+          python3 ./github-automation.py \
             --token '${{ secrets.ISSUE_SUBSCRIBER_TOKEN }}' \
             pr-subscriber \
             --issue-number "${{ github.event.number }}" \

bolt/docs/BAT.md

@@ -69,11 +69,15 @@ Header:
 
 The header is followed by Functions table with `NumFuncs` entries.
 Output binary addresses are delta encoded, meaning that only the difference with
-the previous output address is stored. Addresses implicitly start at zero.
+the last previous output address is stored. Addresses implicitly start at zero.
+Output addresses are continuous through function start addresses and function
+internal offsets, and between hot and cold fragments, to better spread deltas
+and save space.
+
 Hot indices are delta encoded, implicitly starting at zero.
 | Entry  | Encoding | Description |
 | ------ | ------| ----------- |
-| `Address` | Delta, ULEB128 | Function address in the output binary |
+| `Address` | Continuous, Delta, ULEB128 | Function address in the output binary |
 | `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table |
 | `NumEntries` | ULEB128 | Number of address translation entries for a function |
 
@@ -82,10 +86,10 @@ function.
 
 ### Address translation table
 Delta encoding means that only the difference with the previous corresponding
-entry is encoded. Offsets implicitly start at zero.
+entry is encoded. Input offsets implicitly start at zero.
 | Entry  | Encoding | Description |
 | ------ | ------| ----------- |
-| `OutputOffset` | Delta, ULEB128 | Function offset in output binary |
+| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary |
 | `InputOffset` | Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit |
 
 `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source

bolt/include/bolt/Profile/BoltAddressTranslation.h

@@ -121,13 +121,14 @@ class BoltAddressTranslation {
 
   /// Write the serialized address translation table for a function.
   template <bool Cold>
-  void writeMaps(std::map<uint64_t, MapTy> &Maps, raw_ostream &OS);
+  void writeMaps(std::map<uint64_t, MapTy> &Maps, uint64_t &PrevAddress,
+                 raw_ostream &OS);
 
   /// Read the serialized address translation table for a function.
   /// Return a parse error if failed.
   template <bool Cold>
-  void parseMaps(std::vector<uint64_t> &HotFuncs, DataExtractor &DE,
-                 uint64_t &Offset, Error &Err);
+  void parseMaps(std::vector<uint64_t> &HotFuncs, uint64_t &PrevAddress,
+                 DataExtractor &DE, uint64_t &Offset, Error &Err);
 
   std::map<uint64_t, MapTy> Maps;