Skip to content

Commit

Permalink
Merge branch 'master' into nacho/people-chain-spec-with-migation
Browse files Browse the repository at this point in the history
  • Loading branch information
NachoPal committed Apr 17, 2024
2 parents 2365851 + 4e10d3b commit 62d38aa
Show file tree
Hide file tree
Showing 261 changed files with 11,556 additions and 8,385 deletions.
7 changes: 5 additions & 2 deletions .config/lychee.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# Run with `lychee -c .config/lychee.toml ./**/*.rs ./**/*.prdoc`

cache = true
max_cache_age = "1d"
max_cache_age = "10d"
max_redirects = 10
max_retries = 6
max_retries = 3

# Exclude localhost et.al.
exclude_all_private = true
Expand Down Expand Up @@ -51,4 +51,7 @@ exclude = [
# Behind a captcha (code 403):
"https://iohk.io/en/blog/posts/2023/11/03/partner-chains-are-coming-to-cardano/",
"https://www.reddit.com/r/rust/comments/3spfh1/does_collect_allocate_more_than_once_while/",
# 403 rate limited:
"https://etherscan.io/block/11090290",
"https://substrate.stackexchange.com/.*",
]
30 changes: 30 additions & 0 deletions .github/workflows/auto-add-parachain-issues.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# If there are new issues related to the async backing feature,
# add it to the parachain team's board and set a custom "meta" field.

name: Add selected issues to Parachain team board
on:
issues:
types:
- labeled

jobs:
add-parachain-issues:
if: github.event.label.name == 'T16-async_backing'
runs-on: ubuntu-latest
steps:
- name: Generate token
id: generate_token
uses: tibdex/github-app-token@v2.1.0
with:
app_id: ${{ secrets.PROJECT_APP_ID }}
private_key: ${{ secrets.PROJECT_APP_KEY }}
- name: Sync issues
uses: paritytech/github-issue-sync@v0.3.2
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PROJECT_TOKEN: ${{ steps.generate_token.outputs.token }}
project: 119 # Parachain team board
project_field: 'meta'
project_value: 'async backing'
labels: |
T16-async_backing
37 changes: 37 additions & 0 deletions .github/workflows/review-trigger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,43 @@ jobs:
- name: Skip merge queue
if: ${{ contains(github.ref, 'gh-readonly-queue') }}
run: exit 0
- name: Get PR data
id: comments
run: |
echo "bodies=$(gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --json comments --jq '[.comments[].body]')" >> "$GITHUB_OUTPUT"
echo "reviews=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews --jq '[.[].state]')" >> "$GITHUB_OUTPUT"
env:
GH_TOKEN: ${{ github.token }}
- name: Fail when author pushes new code
# Require new reviews when the author is pushing and he is not a member
if: |
contains(fromJson(steps.comments.outputs.reviews), 'APPROVED') &&
github.event_name == 'pull_request_target' &&
github.event.action == 'synchronize' &&
github.event.sender.login == github.event.pull_request.user.login &&
github.event.pull_request.author_association != 'CONTRIBUTOR' &&
github.event.pull_request.author_association != 'MEMBER'
run: |
echo "User's association is ${{ github.event.pull_request.author_association }}"
# We get the list of reviewers who approved the PR
REVIEWERS=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews \
--jq '{reviewers: [.[] | select(.state == "APPROVED") | .user.login]}')
# We request them to review again
echo $REVIEWERS | gh api --method POST repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/requested_reviewers --input -
echo "::error::Project needs to be reviewed again"
exit 1
env:
GH_TOKEN: ${{ github.token }}
- name: Comment requirements
# If the previous step failed and github-actions hasn't commented yet we comment instructions
if: failure() && !contains(fromJson(steps.comments.outputs.bodies), 'Review required! Latest push from author must always be reviewed')
run: |
gh pr comment ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --body "Review required! Latest push from author must always be reviewed"
env:
GH_TOKEN: ${{ github.token }}
COMMENTS: ${{ steps.comments.outputs.users }}
- name: Get PR number
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
Expand Down
159 changes: 159 additions & 0 deletions .github/workflows/sync-templates.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
name: Synchronize templates


# This job is used to keep the repository templates up-to-date.
# The code of the templates exist inside the monorepo, and upon releases we synchronize the repositories:
# - https://github.com/paritytech/polkadot-sdk-minimal-template
# - https://github.com/paritytech/polkadot-sdk-parachain-template
# - https://github.com/paritytech/polkadot-sdk-solochain-template
#
# The job moves the template code out of the monorepo,
# replaces any references to the monorepo workspace using psvm and toml-cli,
# checks that it builds successfully,
# and commits and pushes the result to each respective repository.
# If the build fails, a PR is created instead for manual inspection.


on:
# A manual dispatch for now - automatic on releases later.
workflow_dispatch:
inputs:
crate_release_version:
description: 'A release version to use, e.g. 1.9.0'
required: true


jobs:
sync-templates:
runs-on: ubuntu-latest
environment: master
strategy:
fail-fast: false
matrix:
template: ["minimal", "solochain", "parachain"]
env:
template-path: "polkadot-sdk-${{ matrix.template }}-template"
steps:

# 1. Prerequisites.

- name: Configure git identity
run: |
git config --global user.name "Template Bot"
git config --global user.email "163342540+paritytech-polkadotsdk-templatebot[bot]@users.noreply.github.com"
- uses: actions/checkout@v3
with:
path: polkadot-sdk
ref: "release-crates-io-v${{ github.event.inputs.crate_release_version }}"
- name: Generate a token for the template repository
id: app_token
uses: actions/create-github-app-token@v1.9.3
with:
owner: "paritytech"
repositories: "polkadot-sdk-${{ matrix.template }}-template"
app-id: ${{ secrets.TEMPLATE_APP_ID }}
private-key: ${{ secrets.TEMPLATE_APP_KEY }}
- uses: actions/checkout@v3
with:
repository: "paritytech/polkadot-sdk-${{ matrix.template }}-template"
path: "${{ env.template-path }}"
token: ${{ steps.app_token.outputs.token }}
- name: Install toml-cli
run: cargo install --git https://github.com/gnprice/toml-cli --rev ea69e9d2ca4f0f858110dc7a5ae28bcb918c07fb # v0.2.3
- name: Install Polkadot SDK Version Manager
run: cargo install --git https://github.com/paritytech/psvm --rev c41261ffb52ab0c115adbbdb17e2cb7900d2bdfd psvm # master
- name: Rust compilation prerequisites
run: |
sudo apt update
sudo apt install -y \
protobuf-compiler
rustup target add wasm32-unknown-unknown
rustup component add rustfmt clippy rust-src
# 2. Yanking the template out of the monorepo workspace.

- name: Use psvm to replace git references with released creates.
run: find . -type f -name 'Cargo.toml' -exec psvm -o -v ${{ github.event.inputs.crate_release_version }} -p {} \;
working-directory: polkadot-sdk/templates/${{ matrix.template }}/
- name: Create a new workspace Cargo.toml
run: |
cat << EOF > Cargo.toml
[workspace.package]
license = "MIT-0"
authors = ["Parity Technologies <admin@parity.io>"]
homepage = "https://substrate.io"
[workspace]
members = [
"node",
"pallets/template",
"runtime",
]
resolver = "2"
EOF
shell: bash
working-directory: polkadot-sdk/templates/${{ matrix.template }}/
- name: Update workspace configuration
run: |
set -euo pipefail
# toml-cli has no overwrite functionality yet, so we use temporary files.
# We cannot pipe the output straight to the same file while the CLI still reads and processes it.
toml set templates/${{ matrix.template }}/Cargo.toml 'workspace.package.repository' "https://github.com/paritytech/polkadot-sdk-${{ matrix.template }}-template.git" > Cargo.temp
mv Cargo.temp ./templates/${{ matrix.template }}/Cargo.toml
toml set templates/${{ matrix.template }}/Cargo.toml 'workspace.package.edition' "$(toml get --raw Cargo.toml 'workspace.package.edition')" > Cargo.temp
mv Cargo.temp ./templates/${{ matrix.template }}/Cargo.toml
toml get Cargo.toml 'workspace.lints' --output-toml >> ./templates/${{ matrix.template }}/Cargo.toml
toml get Cargo.toml 'workspace.dependencies' --output-toml >> ./templates/${{ matrix.template }}/Cargo.toml
working-directory: polkadot-sdk
- name: Print the result Cargo.tomls for debugging
if: runner.debug == '1'
run: find . -type f -name 'Cargo.toml' -exec cat {} \;
working-directory: polkadot-sdk/templates/${{ matrix.template }}/

- name: Clean the destination repository
run: rm -rf ./*
working-directory: "${{ env.template-path }}"
- name: Copy over the new changes
run: |
cp -r polkadot-sdk/templates/${{ matrix.template }}/* "${{ env.template-path }}/"
# 3. Verify the build. Push the changes or create a PR.

# We've run into out-of-disk error when compiling in the next step, so we free up some space this way.
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # 1.3.1
with:
android: true # This alone is a 12 GB save.
# We disable the rest because it caused some problems. (they're enabled by default)
# The Android removal is enough.
dotnet: false
haskell: false
large-packages: false
swap-storage: false

- name: Check if it compiles
id: check-compilation
run: cargo check && cargo test
working-directory: "${{ env.template-path }}"
timeout-minutes: 90
- name: Create PR on failure
if: failure() && steps.check-compilation.outcome == 'failure'
uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5 # v5
with:
path: "${{ env.template-path }}"
token: ${{ steps.app_token.outputs.token }}
add-paths: |
./*
title: "[Don't merge] Update the ${{ matrix.template }} template"
body: "The template has NOT been successfully built and needs to be inspected."
branch: "update-template/${{ github.event_name }}"
- name: Push changes
run: |
git add -A .
git commit --allow-empty -m "Update template triggered by ${{ github.event_name }}"
git push
working-directory: "${{ env.template-path }}"
8 changes: 6 additions & 2 deletions .gitlab/pipeline/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ publish-subsystem-benchmarks:
- .kubernetes-env
- .publish-gh-pages-refs
needs:
- job: subsystem-regression-tests
- job: subsystem-benchmark-availability-recovery
artifacts: true
- job: subsystem-benchmark-availability-distribution
artifacts: true
- job: publish-rustdoc
artifacts: false
Expand Down Expand Up @@ -109,7 +111,9 @@ trigger_workflow:
needs:
- job: publish-subsystem-benchmarks
artifacts: false
- job: subsystem-regression-tests
- job: subsystem-benchmark-availability-recovery
artifacts: true
- job: subsystem-benchmark-availability-distribution
artifacts: true
script:
- echo "Triggering workflow"
Expand Down
25 changes: 21 additions & 4 deletions .gitlab/pipeline/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -511,21 +511,38 @@ test-syscalls:
fi
allow_failure: false # this rarely triggers in practice

subsystem-regression-tests:
subsystem-benchmark-availability-recovery:
stage: test
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: always
expire_in: 1 days
expire_in: 1 hour
paths:
- charts/
extends:
- .docker-env
- .common-refs
- .run-immediately
script:
- cargo bench --profile=testnet -p polkadot-availability-recovery --bench availability-recovery-regression-bench --features subsystem-benchmarks
- cargo bench --profile=testnet -p polkadot-availability-distribution --bench availability-distribution-regression-bench --features subsystem-benchmarks
- cargo bench -p polkadot-availability-recovery --bench availability-recovery-regression-bench --features subsystem-benchmarks
tags:
- benchmark
allow_failure: true

subsystem-benchmark-availability-distribution:
stage: test
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: always
expire_in: 1 hour
paths:
- charts/
extends:
- .docker-env
- .common-refs
- .run-immediately
script:
- cargo bench -p polkadot-availability-distribution --bench availability-distribution-regression-bench --features subsystem-benchmarks
tags:
- benchmark
allow_failure: true
2 changes: 2 additions & 0 deletions .gitlab/pipeline/zombienet/polkadot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,8 @@ zombienet-polkadot-functional-0011-async-backing-6-seconds-rate:
zombienet-polkadot-elastic-scaling-0001-basic-3cores-6s-blocks:
extends:
- .zombienet-polkadot-common
variables:
FORCED_INFRA_INSTANCE: "spot-iops"
script:
- /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
--local-dir="${LOCAL_DIR}/elastic_scaling"
Expand Down
Loading

0 comments on commit 62d38aa

Please sign in to comment.