feat: Revamped sequencer timetable and tx processing timeout

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -196,6 +196,7 @@ export class AztecNodeService implements AztecNode, Traceable {
           l2BlockSource: archiver,
           l1ToL2MessageSource: archiver,
           telemetry,
+          dateProvider,
           ...deps,
         });
 
@@ -230,6 +231,10 @@ export class AztecNodeService implements AztecNode, Traceable {
     return this.blockSource;
   }
 
+  public getContractDataSource(): ContractDataSource {
+    return this.contractDataSource;
+  }
+
   public getP2P(): P2P {
     return this.p2pClient;
   }
@@ -815,7 +820,11 @@ export class AztecNodeService implements AztecNode, Traceable {
       feeRecipient,
     );
     const prevHeader = (await this.blockSource.getBlock(-1))?.header;
-    const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.telemetry);
+    const publicProcessorFactory = new PublicProcessorFactory(
+      this.contractDataSource,
+      new DateProvider(),
+      this.telemetry,
+    );
     const fork = await this.worldStateSynchronizer.fork();
 
     this.log.verbose(`Simulating public calls for tx ${tx.getTxHash()}`, {

yarn-project/aztec.js/src/utils/anvil_test_watcher.ts

@@ -1,6 +1,7 @@
 import { type EthCheatCodes, type Logger, createLogger } from '@aztec/aztec.js';
 import { type EthAddress } from '@aztec/circuits.js';
 import { RunningPromise } from '@aztec/foundation/running-promise';
+import { type TestDateProvider } from '@aztec/foundation/timer';
 import { RollupAbi } from '@aztec/l1-artifacts';
 
 import { type GetContractReturnType, type HttpTransport, type PublicClient, getAddress, getContract } from 'viem';
@@ -24,6 +25,7 @@ export class AnvilTestWatcher {
     private cheatcodes: EthCheatCodes,
     rollupAddress: EthAddress,
     publicClient: PublicClient<HttpTransport, chains.Chain>,
+    private dateProvider?: TestDateProvider,
   ) {
     this.rollup = getContract({
       address: getAddress(rollupAddress.toString()),
@@ -69,6 +71,7 @@ export class AnvilTestWatcher {
         const timestamp = await this.rollup.read.getTimestampForSlot([currentSlot + 1n]);
         try {
           await this.cheatcodes.warp(Number(timestamp));
+          this.dateProvider?.setTime(Number(timestamp) * 1000);
         } catch (e) {
           this.logger.error(`Failed to warp to timestamp ${timestamp}: ${e}`);
         }

yarn-project/aztec.js/src/utils/cheat_codes.ts

@@ -108,8 +108,17 @@ export class RollupCheatCodes {
     const slotsUntilNextEpoch = epochDuration - (slot % epochDuration) + 1n;
     const timeToNextEpoch = slotsUntilNextEpoch * slotDuration;
     const l1Timestamp = BigInt((await this.client.getBlock()).timestamp);
-    await this.ethCheatCodes.warp(Number(l1Timestamp + timeToNextEpoch));
-    this.logger.verbose(`Advanced to next epoch`);
+    await this.ethCheatCodes.warp(Number(l1Timestamp + timeToNextEpoch), true);
+    this.logger.warn(`Advanced to next epoch`);
+  }
+
+  /** Warps time in L1 until the beginning of the next slot. */
+  public async advanceToNextSlot() {
+    const currentSlot = await this.getSlot();
+    const timestamp = await this.rollup.read.getTimestampForSlot([currentSlot + 1n]);
+    await this.ethCheatCodes.warp(Number(timestamp));
+    this.logger.warn(`Advanced to slot ${currentSlot + 1n}`);
+    return [timestamp, currentSlot + 1n];
   }
 
   /**
@@ -120,9 +129,9 @@ export class RollupCheatCodes {
     const l1Timestamp = (await this.client.getBlock()).timestamp;
     const slotDuration = await this.rollup.read.SLOT_DURATION();
     const timeToWarp = BigInt(howMany) * slotDuration;
-    await this.ethCheatCodes.warp(l1Timestamp + timeToWarp);
+    await this.ethCheatCodes.warp(l1Timestamp + timeToWarp, true);
     const [slot, epoch] = await Promise.all([this.getSlot(), this.getEpoch()]);
-    this.logger.verbose(`Advanced ${howMany} slots up to slot ${slot} in epoch ${epoch}`);
+    this.logger.warn(`Advanced ${howMany} slots up to slot ${slot} in epoch ${epoch}`);
   }
 
   /** Returns the current proof claim (if any) */
@@ -163,7 +172,7 @@ export class RollupCheatCodes {
 
     await this.asOwner(async account => {
       await this.rollup.write.setAssumeProvenThroughBlockNumber([blockNumber], { account, chain: this.client.chain });
-      this.logger.verbose(`Marked ${blockNumber} as proven`);
+      this.logger.warn(`Marked ${blockNumber} as proven`);
     });
   }
 

yarn-project/circuit-types/src/interfaces/configs.ts

@@ -38,6 +38,8 @@ export interface SequencerConfig {
   governanceProposerPayload?: EthAddress;
   /** Whether to enforce the time table when building blocks */
   enforceTimeTable?: boolean;
+  /** How many seconds into an L1 slot we can still send a tx and get it mined. */
+  maxL1TxInclusionTimeIntoSlot?: number;
 }
 
 const AllowedElementSchema = z.union([
@@ -59,4 +61,5 @@ export const SequencerConfigSchema = z.object({
   maxBlockSizeInBytes: z.number().optional(),
   enforceFees: z.boolean().optional(),
   gerousiaPayload: schemas.EthAddress.optional(),
+  maxL1TxInclusionTimeIntoSlot: z.number().optional(),
 }) satisfies ZodFor<SequencerConfig>;

yarn-project/end-to-end/src/e2e_block_building.test.ts

@@ -1,4 +1,5 @@
 import { getSchnorrAccount } from '@aztec/accounts/schnorr';
+import { type AztecNodeService } from '@aztec/aztec-node';
 import {
   type AztecAddress,
   type AztecNode,
@@ -7,6 +8,7 @@ import {
   ContractFunctionInteraction,
   Fq,
   Fr,
+  type GlobalVariables,
   L1EventPayload,
   L1NotePayload,
   type Logger,
@@ -17,12 +19,20 @@ import {
   retryUntil,
   sleep,
 } from '@aztec/aztec.js';
+// eslint-disable-next-line no-restricted-imports
+import { type MerkleTreeWriteOperations, type Tx } from '@aztec/circuit-types';
 import { getL1ContractsConfigEnvVars } from '@aztec/ethereum';
-import { times } from '@aztec/foundation/collection';
+import { asyncMap } from '@aztec/foundation/async-map';
+import { times, unique } from '@aztec/foundation/collection';
 import { poseidon2Hash } from '@aztec/foundation/crypto';
+import { type TestDateProvider } from '@aztec/foundation/timer';
 import { StatefulTestContract, StatefulTestContractArtifact } from '@aztec/noir-contracts.js/StatefulTest';
 import { TestContract } from '@aztec/noir-contracts.js/Test';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
+import { type Sequencer, type SequencerClient, SequencerState } from '@aztec/sequencer-client';
+import { PublicProcessorFactory, type PublicTxResult, PublicTxSimulator, type WorldStateDB } from '@aztec/simulator';
+import { type TelemetryClient } from '@aztec/telemetry-client';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 
 import { jest } from '@jest/globals';
 import 'jest-extended';
@@ -38,6 +48,9 @@ describe('e2e_block_building', () => {
   let owner: Wallet;
   let minter: Wallet;
   let aztecNode: AztecNode;
+  let sequencer: TestSequencerClient;
+  let dateProvider: TestDateProvider | undefined;
+  let cheatCodes: CheatCodes;
   let teardown: () => Promise<void>;
 
   const { aztecEpochProofClaimWindowInL2Slots } = getL1ContractsConfigEnvVars();
@@ -46,13 +59,19 @@ describe('e2e_block_building', () => {
     const artifact = StatefulTestContractArtifact;
 
     beforeAll(async () => {
+      let sequencerClient;
       ({
         teardown,
         pxe,
         logger,
         aztecNode,
         wallets: [owner, minter],
+        sequencer: sequencerClient,
+        dateProvider,
+        cheatCodes,
       } = await setup(2));
+      // Bypass accessibility modifiers in sequencer
+      sequencer = sequencerClient! as unknown as TestSequencerClient;
     });
 
     afterEach(() => aztecNode.setConfig({ minTxsPerBlock: 1 }));
@@ -106,7 +125,7 @@ describe('e2e_block_building', () => {
 
       // Assemble N contract deployment txs
       // We need to create them sequentially since we cannot have parallel calls to a circuit
-      const TX_COUNT = 8;
+      const TX_COUNT = 4;
       await aztecNode.setConfig({ minTxsPerBlock: TX_COUNT });
 
       const methods = times(TX_COUNT, i => contract.methods.increment_public_value(ownerAddress, i));
@@ -127,6 +146,57 @@ describe('e2e_block_building', () => {
       expect(receipts.map(r => r.blockNumber)).toEqual(times(TX_COUNT, () => receipts[0].blockNumber));
     });
 
+    it('processes txs until hitting timetable', async () => {
+      const TX_COUNT = 32;
+
+      const ownerAddress = owner.getCompleteAddress().address;
+      const contract = await StatefulTestContract.deploy(owner, ownerAddress, ownerAddress, 1).send().deployed();
+      logger.info(`Deployed stateful test contract at ${contract.address}`);
+
+      // We have to set minTxsPerBlock to 1 or we could end with dangling txs.
+      // We also set enforceTimetable so the deadline makes sense, otherwise we may be starting the
+      // block too late into the slot, and start processing when the deadline has already passed.
+      logger.info(`Updating aztec node config`);
+      await aztecNode.setConfig({ minTxsPerBlock: 1, maxTxsPerBlock: TX_COUNT, enforceTimeTable: true });
+
+      // We tweak the sequencer so it uses a fake simulator that adds a 200ms delay to every public tx.
+      const archiver = (aztecNode as AztecNodeService).getContractDataSource();
+      sequencer.sequencer.publicProcessorFactory = new TestPublicProcessorFactory(
+        archiver,
+        dateProvider!,
+        new NoopTelemetryClient(),
+      );
+
+      // We also cheat the sequencer's timetable so it allocates little time to processing.
+      // This will leave the sequencer with just 2s to build the block, so it shouldn't be
+      // able to squeeze in more than 10 txs in each. This is sensitive to the time it takes
+      // to pick up and validate the txs, so we may need to bump it to work on CI.
+      sequencer.sequencer.timeTable[SequencerState.WAITING_FOR_TXS] = 2;
+      sequencer.sequencer.timeTable[SequencerState.CREATING_BLOCK] = 2;
+      sequencer.sequencer.processTxTime = 1;
+
+      // Flood the mempool with TX_COUNT simultaneous txs
+      const methods = times(TX_COUNT, i => contract.methods.increment_public_value(ownerAddress, i));
+      const provenTxs = await asyncMap(methods, method => method.prove({ skipPublicSimulation: true }));
+      logger.info(`Sending ${TX_COUNT} txs to the node`);
+      const txs = await Promise.all(provenTxs.map(tx => tx.send()));
+      logger.info(`All ${TX_COUNT} txs have been sent`, { txs: await Promise.all(txs.map(tx => tx.getTxHash())) });
+
+      // We forcefully mine a block to make the L1 timestamp move and sync to it, otherwise the sequencer will
+      // stay continuously trying to build a block for the same slot, even if the time for it has passed.
+      // Keep in mind the anvil test watcher only moves the anvil blocks when there is a block mined.
+      // This is quite ugly, and took me a very long time to realize it was needed.
+      // Maybe we should change it? And have it always mine a block every 12s even if there is no activity?
+      const [timestamp] = await cheatCodes.rollup.advanceToNextSlot();
+      dateProvider!.setTime(Number(timestamp) * 1000);
+
+      // Await txs to be mined and assert they are mined across multiple different blocks.
+      const receipts = await Promise.all(txs.map(tx => tx.wait()));
+      const blockNumbers = receipts.map(r => r.blockNumber!).sort((a, b) => a - b);
+      logger.info(`Txs mined on blocks: ${unique(blockNumbers)}`);
+      expect(blockNumbers.at(-1)! - blockNumbers[0]).toBeGreaterThan(1);
+    });
+
     it.skip('can call public function from different tx in same block as deployed', async () => {
       // Ensure both txs will land on the same block
       await aztecNode.setConfig({ minTxsPerBlock: 2 });
@@ -503,3 +573,36 @@ async function sendAndWait(calls: ContractFunctionInteraction[]) {
       .map(p => p.wait()),
   );
 }
+
+type TestSequencer = Omit<Sequencer, 'publicProcessorFactory' | 'timeTable'> & {
+  publicProcessorFactory: PublicProcessorFactory;
+  timeTable: Record<SequencerState, number>;
+  processTxTime: number;
+};
+type TestSequencerClient = Omit<SequencerClient, 'sequencer'> & { sequencer: TestSequencer };
+
+class TestPublicTxSimulator extends PublicTxSimulator {
+  public override async simulate(tx: Tx): Promise<PublicTxResult> {
+    await sleep(200);
+    return super.simulate(tx);
+  }
+}
+class TestPublicProcessorFactory extends PublicProcessorFactory {
+  protected override createPublicTxSimulator(
+    db: MerkleTreeWriteOperations,
+    worldStateDB: WorldStateDB,
+    telemetryClient: TelemetryClient,
+    globalVariables: GlobalVariables,
+    doMerkleOperations: boolean,
+    enforceFeePayment: boolean,
+  ): PublicTxSimulator {
+    return new TestPublicTxSimulator(
+      db,
+      worldStateDB,
+      telemetryClient,
+      globalVariables,
+      doMerkleOperations,
+      enforceFeePayment,
+    );
+  }
+}

yarn-project/end-to-end/src/fixtures/utils.ts

@@ -416,10 +416,13 @@ export async function setup(
     await ethCheatCodes.warp(opts.l2StartTime);
   }
 
+  const dateProvider = new TestDateProvider();
+
   const watcher = new AnvilTestWatcher(
     new EthCheatCodesWithState(config.l1RpcUrl),
     deployL1ContractsValues.l1ContractAddresses.rollupAddress,
     deployL1ContractsValues.publicClient,
+    dateProvider,
   );
 
   await watcher.start();
@@ -441,7 +444,6 @@ export async function setup(
 
   const telemetry = await telemetryPromise;
   const publisher = new TestL1Publisher(config, telemetry);
-  const dateProvider = new TestDateProvider();
   const aztecNode = await AztecNodeService.createAndSync(config, { telemetry, publisher, dateProvider });
   const sequencer = aztecNode.getSequencer();
 

yarn-project/ethereum/src/config.ts

@@ -13,13 +13,13 @@ export type L1ContractsConfig = {
   aztecEpochProofClaimWindowInL2Slots: number;
 };
 
-export const DefaultL1ContractsConfig: L1ContractsConfig = {
+export const DefaultL1ContractsConfig = {
   ethereumSlotDuration: 12,
   aztecSlotDuration: 24,
   aztecEpochDuration: 16,
   aztecTargetCommitteeSize: 48,
   aztecEpochProofClaimWindowInL2Slots: 13,
-};
+} satisfies L1ContractsConfig;
 
 export const l1ContractsConfigMappings: ConfigMappingsType<L1ContractsConfig> = {
   ethereumSlotDuration: {