diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/Processor_Cores.jpeg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/Processor_Cores.jpeg new file mode 100644 index 000000000..9040f55d2 Binary files /dev/null and b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/Processor_Cores.jpeg differ diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/Train.jpeg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/Train.jpeg new file mode 100644 index 000000000..0ad0f9965 Binary files /dev/null and b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/Train.jpeg differ diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/advantage.png b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/advantage.png new file mode 100644 index 000000000..c0529c52c Binary files /dev/null and b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/advantage.png differ diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/cores_big_picture.svg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/cores_big_picture.svg new file mode 100644 index 000000000..a6247c459 --- /dev/null +++ b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/cores_big_picture.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/dice.jpeg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/dice.jpeg new file mode 100644 index 000000000..473f84442 Binary files /dev/null and b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/dice.jpeg differ diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/exotic_core_scheduling.svg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/exotic_core_scheduling.svg new file mode 100644 index 000000000..9d8cc0abf --- /dev/null +++ b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/exotic_core_scheduling.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/occupying_assigned_cores_on_demand.svg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/occupying_assigned_cores_on_demand.svg new file mode 100644 index 000000000..544bdd8b5 --- /dev/null +++ b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/occupying_assigned_cores_on_demand.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/occupying_assigned_cores_with_lease.svg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/occupying_assigned_cores_with_lease.svg new file mode 100644 index 000000000..e998d2a6e --- /dev/null +++ b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/occupying_assigned_cores_with_lease.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/pairing_backing_groups_with_cores.svg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/pairing_backing_groups_with_cores.svg new file mode 100644 index 000000000..817f27dbf --- /dev/null +++ b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/pairing_backing_groups_with_cores.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/pairing_leases_and_claims_with_cores.svg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/pairing_leases_and_claims_with_cores.svg new file mode 100644 index 000000000..9a29040e3 --- /dev/null +++ b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/pairing_leases_and_claims_with_cores.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/road.jpeg b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/road.jpeg new file mode 100644 index 000000000..23dff3bc2 Binary files /dev/null and b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/road.jpeg differ diff --git a/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/validator-groups.png b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/validator-groups.png new file mode 100644 index 000000000..8df741461 Binary files /dev/null and b/assets/img/5-Polkadot/Availability_Cores_Deep_Dive/validator-groups.png differ diff --git a/syllabus/7-Polkadot/Availability_Cores/Deep_Dive_Availability_Cores_Slides.md b/syllabus/7-Polkadot/Availability_Cores/Deep_Dive_Availability_Cores_Slides.md new file mode 100644 index 000000000..132e4021e --- /dev/null +++ b/syllabus/7-Polkadot/Availability_Cores/Deep_Dive_Availability_Cores_Slides.md @@ -0,0 +1,575 @@ +--- +title: Deep Dive, Availability Cores +description: The Polkadot Abstraction for Flexible Blockspace Allocation +duration: 1 hour +--- + +# Deep Dive, Availability Cores + +Notes: + +Hello! + +I'm Bradley Olson + +Was student at first Academy in Cambridge + +Currently on Parachains Core Team at Parity + +Will present 3 lectures providing a window into Polkadot core, a slice of where we're at and where we're headed. + +First a look at availability cores, the abstraction enabling flexible purchases of blockspace under the umbrella of Polkadot shared security. + +Lets get to it + +--- + +## Addressing the Dual Naming + + + +- In the code: Availability core +- Outside the code: Execution core + + + +--- + +## Overview + + + +- What do availability cores represent? +- How do cores map parachain leases and claims to validator subsets? + +- How do cores gate each step of the parachains protocol? + +- What advantages do cores give us now? + +- What roadmap items do cores accommodate? + + + + +--- + +## Review, Blockspace + +> Blockspace is the capacity of a blockchain
to finalize and commit operations + +Polkadot's primary product is _blockspace_. + +--- + +## Blockspace, Use It or Lose It + +Polkadot blockspace is consumed in two ways: + + + +1. When the relay chain validates, includes,
and finalizes a parachain block +1. When the capacity to validate a parachain block
is left unused and expires + + +
+ + + + +--- + +## Availability Core Defined + + + + +- Availability cores are the abstraction we use to allocate Polkadot's blockspace. +- Allocated via leases and on-demand claims + +- Cores divide blockspace supply into discrete units, 1 parachain block per relay chain block per core + +- Why "availability"? + +- Why "core"? + + + + + + + + + + +Notes: + +- "Availability", because a core is considered occupied by a parachain block candidate while that candidate is being made available. But cores mediate access to the entire parablock validation pipeline, not just availability. +- "Core", because many candidates can be made available in parallel, mimicking the parallel computation per core in a computer processor. + +--- + +## Availability + +Though cores gate the entire parachain block pipeline,
the availability process alone determines when these cores are considered occupied vs free. + +To recap, the goals of availability are: + + + +1. To ensure that approvers will always be able to recover the PoVs to validate their assigned blocks + +2. To ensure that parachain nodes can recover blocks in the case that a parablock author fails to share them, through malintent or malfunction + + +--- + +## Core States + + + + +Free + + + + + + +Scheduled + + + + + + + +Occupied + + + + + + +Notes: + +- Before going any farther we need to talk about core states +- CoreState: Free -> core has not been assigned a parachain via lease or on-demand claim +- CoreState: Scheduled -> Core has an assigned parachain and is currently unoccupied +- CoreState: Occupied -> Core has assignment and is occupied by a parablock pending availability + +--- + +## The Availability Process + + + +1. Block author places a candidate on-chain as backed, immediately occupying its scheduled core + +1. Candidate backers distribute erasure coded PoV chunks + +1. Validators distribute statements as to which candidates they have chunks for + +1. Availability threshold is met (2/3 vs the 1/3 needed to reconstruct POV) + +1. Candidate marked as included on chain and core freed + +1. Approvers or parachain nodes retrieve PoV chunks as needed + + + + +--- + +## Cores and Blockspace Over Time + + + +Notes: + +- Metaphor: Freight trains +- Relay chain: Train loading bay +- Relay block: Train leaving station every 6 seconds +- Parachain block: One train car worth of cargo +- Availability core: Car index within all trains + +If you have a lease on core 4, then you have the right to fill train car 4 on each train with whatever you want to ship. + +Q: How would an on-demand claim be represented in this metaphor? + +--- + +# Mapping Leases and Claims to Validator Subsets + +--- + +## The Big Picture + + + +Notes: + +- Which steps of the parachains protocol are missing, and why? +- Going to dive into each piece +- Questions? + +--- + +## Assigning Leases and Claims to Cores + + + +Notes: + +- Leases have indices and pair to cores with the same index +- Cores not designated as on-demand and without a paired lease are left free, their blockspace wasted +- When on-demand claims are queued, they are each assigned a designated core in ascending order, looping when reaching the last core + +--- + +## Assigning Backing Groups to Cores + + + +Notes: + +- Round robin, fixed intervals + +This prevents a byzantine backing group from interrupting the liveness of any one parachain for too long. + +--- + +## Backing Group Formation + + + +Notes: + +- Validators randomly assigned to groups at start of session. +- Group count is active validator count / max group size rounded up +- Groups are partitioned such that the largest group and smallest group have a size difference of 1 + +--- + + + +## Assigning Approvers to Cores + + + +- Randomness via schnorrkel::vrf +- Approver assignments activated with delay tranches until threshold met + +- Results in 30-40 approvers checking each block + +- Different assignments each block prevent DOS + + + + +--- + +## Putting the Pieces Together + + + +--- + +## Occupying Assigned Cores: With Lease + + + +Notes: + +Q: What step of the parachains protocol takes place between "Supplied backable candidate" and "availability process? + +--- + +## Occupying Assigned Cores: On Demand + + + +--- + +## Core States in the Runtime + +In file: polkadot/runtime/parachains/src/scheduler.rs + +
+ +```rust[1|3-8|5,10-16] +pub(crate) type AvailabilityCores = StorageValue<_, Vec>, ValueQuery>; + +pub enum CoreOccupied { + /// A parathread (on-demand parachain). + Parathread(ParathreadEntry), + /// A lease holding parachain. + Parachain, +} + +/// Parathread = on-demand parachain +pub struct ParathreadEntry { + /// The claim. + pub claim: ParathreadClaim, + /// Number of retries. + pub retries: u32, +} +``` + +
+ +Notes: + +Q: When Option is None, what does that indicate? + +--- + +## Core Assignments in The Runtime + +In file: polkadot/runtime/parachains/src/scheduler.rs + +
+ +```rust[1|3-10|9,12-17] +pub(crate) type Scheduled = StorageValue<_, Vec, ValueQuery>; + +pub struct CoreAssignment { + /// The core that is assigned. + pub core: CoreIndex, + /// The unique ID of the para that is assigned to the core. + pub para_id: ParaId, + /// The kind of the assignment. + pub kind: AssignmentKind, +} + +pub enum AssignmentKind { + /// A parachain. + Parachain, + /// A parathread (on-demand parachain). + Parathread(CollatorId, u32), +} +``` + +
+ +Notes: + +- Vec of all core assignments +- Pairs ParaId with CoreIndex +- AssignmentKind carries retry info for on-demand + +--- + +## How Cores Gate Each Step of the Parachains Protocol + +--- + +## How Core Assignments Mediate Backing + +Each parablock candidate is built in the context of a particular `relay_parent`. + +Validators query their core assignment as of `relay_parent` and refuse to second candidates not assiciated with their backing group. + + + +Notes: + +- Relay parent context: max PoV size, current parachain runtime code, and backing group assignments. + +--- + +### How Core Assignments Mediate Backing (Cont.) + +`handle_second_message()` in the Backing Subsystem. + +```rust[1|6] +if Some(candidate.descriptor().para_id) != rp_state.assignment { + gum::debug!( + target: LOG_TARGET, + our_assignment = ?rp_state.assignment, + collation = ?candidate.descriptor().para_id, + "Subsystem asked to second for para outside of our assignment", + ); + + return Ok(()) +} +``` + +--- + +## Cores and Backing On-Chain + + + +- For each core that is either unoccupied or is about to be a new candidate is found +- A candidate for the parachain scheduled next on that core is provided to the block author + +- Backed on-chain -> immediately occupies core + + + + +Notes: + +- Review possible core states +- Mention time-out vs made available + +Q: What does "immediately occupies core" imply? + +--- + +## Cores and Backing On-Chain, Code + +
+ +Code determining whether to back a candidate and which, greatly simplified + +```rust[1|25|2-4|5-13|13-23|1,28-29] + let (para_id) = match core { + CoreState::Scheduled(scheduled_core) => { + scheduled_core.para_id + }, + CoreState::Occupied(occupied_core) => { + if current_occupant_made_available(core_idx, &occupied_core.availability) + { + if let Some(ref scheduled_core) = occupied_core.next_up_on_available { + scheduled_core.para_id + } else { + continue + } + } else { + if occupied_core.time_out_at != current_block { + continue + } + if let Some(ref scheduled_core) = occupied_core.next_up_on_time_out { + scheduled_core.para_id + } else { + continue + } + } + }, + CoreState::Free => continue, + }; + + let candidate_hash = + get_backable_candidate(relay_parent, para_id, required_path, sender).await?; +``` + +
+ +Notes: + +- Discuss core freeing criteria +- `bitfields_indicate_availability` + - next_up_on_available +- availability time out + - next_up_on_timeout + +--- + +## Cores and Approvals, Disputes, Finality + +Each included candidate has already occupied an availability core + +Approvals, Disputes, and Finality are only provided to included candidates + +--- + +## Advantages Cores Give us Now + + + +1. Predictability of parachain execution +1. Predictability of allocation + + + + + + +Notes: + +- Regularly rotating pairings between cores and backing groups lessen the impact of potential bad validator subsets +- Cores accommodate advance allocation making costs for parachains predictable +- Core time can be resold or split for precise allocation + +--- + +# Cores and Roadmap Items + + + +--- + +## Exotic core scheduling + + + +- Multiple cores per parachain +- Overlapping leases of many lengths + +- Lease + On-demand + + + + + + +Notes: + +- Each color represents a parachain + Metaphor: +- Multiple cars on the same train +- Overlapping time spans of rights to fill different train cars +- Long term rights to one car and buy rights to others for just one train + +--- + +## Divisible and Marketable Blockspace + +We want Parachains to buy, sell, and split blockspace such that they are allocated exactly as much as fits their needs.

+ + + +- Core sharing via use-threshold blockspace regions +- Secondary markets for blockspace regions + + + +Notes: + +- Example: Region spanning 100 blocks. Split the region use so that each of two parties can submit up to 50 parablocks. Ownership proportion is enforced throughout the region such that each party can't submit more than 5 out of the first 10 blocks. + +--- + +## Framing Shift: Blockspace vs Core Time + +Blockspace is a universal term for the product of blockchains, while core time is Polkadot's particular abstraction for allocating blockspace or other computation. + + + +Cores can secure computation other than blocks. For example, a smart contract could be deployed on a core directly. + + + +Allocating cores by time rather than for a fixed block size could allow for smaller or larger parachain blocks to be handled seamlessly. + + + +--- + +## Resources + + + +1. [Implementers Guide: Scheduler Pallet](https://paritytech.github.io/polkadot/book/runtime/scheduler.html) +1. [RFC: Agile Coretime](https://github.com/polkadot-fellows/RFCs/pull/1) +1. [RFC: Coretime Scheduling Regions](https://github.com/polkadot-fellows/rfcs/pull/3) +1. [Rob Habermeier's Blog: Polkadot Blockspace Over Blockchains](https://www.rob.tech/polkadot-blockspace-over-blockchains/) + + + +--- + + + +# Questions diff --git a/syllabus/7-Polkadot/Polkadot_Decisions_slides.md b/syllabus/7-Polkadot/Polkadot_Decisions_slides.md index 59806a287..b6662527e 100644 --- a/syllabus/7-Polkadot/Polkadot_Decisions_slides.md +++ b/syllabus/7-Polkadot/Polkadot_Decisions_slides.md @@ -141,9 +141,7 @@ Polkadot tries to accomplish that mission by solving three problems: 1. Shared Security 1. Interoperability - - ---- +## ## The Decisions diff --git a/syllabus/7-Polkadot/assets/Processor_Cores.jpeg b/syllabus/7-Polkadot/assets/Processor_Cores.jpeg new file mode 100644 index 000000000..9040f55d2 Binary files /dev/null and b/syllabus/7-Polkadot/assets/Processor_Cores.jpeg differ diff --git a/syllabus/7-Polkadot/assets/Train.jpeg b/syllabus/7-Polkadot/assets/Train.jpeg new file mode 100644 index 000000000..0ad0f9965 Binary files /dev/null and b/syllabus/7-Polkadot/assets/Train.jpeg differ diff --git a/syllabus/7-Polkadot/assets/advantage.png b/syllabus/7-Polkadot/assets/advantage.png new file mode 100644 index 000000000..c0529c52c Binary files /dev/null and b/syllabus/7-Polkadot/assets/advantage.png differ diff --git a/syllabus/7-Polkadot/assets/cores_big_picture.svg b/syllabus/7-Polkadot/assets/cores_big_picture.svg new file mode 100644 index 000000000..a6247c459 --- /dev/null +++ b/syllabus/7-Polkadot/assets/cores_big_picture.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/syllabus/7-Polkadot/assets/dice.jpeg b/syllabus/7-Polkadot/assets/dice.jpeg new file mode 100644 index 000000000..473f84442 Binary files /dev/null and b/syllabus/7-Polkadot/assets/dice.jpeg differ diff --git a/syllabus/7-Polkadot/assets/exotic_core_scheduling.svg b/syllabus/7-Polkadot/assets/exotic_core_scheduling.svg new file mode 100644 index 000000000..9d8cc0abf --- /dev/null +++ b/syllabus/7-Polkadot/assets/exotic_core_scheduling.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/syllabus/7-Polkadot/assets/free.png b/syllabus/7-Polkadot/assets/free.png new file mode 100644 index 000000000..3a9c33b89 Binary files /dev/null and b/syllabus/7-Polkadot/assets/free.png differ diff --git a/syllabus/7-Polkadot/assets/occupied.png b/syllabus/7-Polkadot/assets/occupied.png new file mode 100644 index 000000000..5d82d08c9 Binary files /dev/null and b/syllabus/7-Polkadot/assets/occupied.png differ diff --git a/syllabus/7-Polkadot/assets/occupying_assigned_cores_on_demand.svg b/syllabus/7-Polkadot/assets/occupying_assigned_cores_on_demand.svg new file mode 100644 index 000000000..544bdd8b5 --- /dev/null +++ b/syllabus/7-Polkadot/assets/occupying_assigned_cores_on_demand.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/syllabus/7-Polkadot/assets/occupying_assigned_cores_with_lease.svg b/syllabus/7-Polkadot/assets/occupying_assigned_cores_with_lease.svg new file mode 100644 index 000000000..e998d2a6e --- /dev/null +++ b/syllabus/7-Polkadot/assets/occupying_assigned_cores_with_lease.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/syllabus/7-Polkadot/assets/pairing_backing_groups_with_cores.svg b/syllabus/7-Polkadot/assets/pairing_backing_groups_with_cores.svg new file mode 100644 index 000000000..817f27dbf --- /dev/null +++ b/syllabus/7-Polkadot/assets/pairing_backing_groups_with_cores.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/syllabus/7-Polkadot/assets/pairing_leases_and_claims_with_cores.svg b/syllabus/7-Polkadot/assets/pairing_leases_and_claims_with_cores.svg new file mode 100644 index 000000000..9a29040e3 --- /dev/null +++ b/syllabus/7-Polkadot/assets/pairing_leases_and_claims_with_cores.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/syllabus/7-Polkadot/assets/road.jpeg b/syllabus/7-Polkadot/assets/road.jpeg new file mode 100644 index 000000000..23dff3bc2 Binary files /dev/null and b/syllabus/7-Polkadot/assets/road.jpeg differ diff --git a/syllabus/7-Polkadot/assets/scheduled.png b/syllabus/7-Polkadot/assets/scheduled.png new file mode 100644 index 000000000..aea7cb1ae Binary files /dev/null and b/syllabus/7-Polkadot/assets/scheduled.png differ