Re-enable recursion contexts now that we can rely on depth being a distinguisher

Author: dberlin

IncrementalParsing.md

@@ -57,10 +57,8 @@ Tree fixup is actually the most expensive part of the parser data right now, and
 - We use the parselistener interface as an easy way to ensure we get to see entry/
   exit events at the right time. This turned out to be easier than handling
   recursion/left factoring through overriding the relevant parser interface pieces.
-- It is not possible to distinguish between recursive contexts because they only
-  store the invoking state, not the state of the parser. That means it is not apparently
-  possible to get the kind of unique key we need. As a result, we will not reuse recursive
-  contexts (but will reuse pieces from below them).
+- Top level recursion contexts are now reused, but we won't reuse individaul recursion contexts yet.
+
 
 #### References
 

src/IncrementalParser.ts

@@ -82,6 +82,7 @@ export abstract class IncrementalParser extends Parser
 		}
 		// See if we have seen this state before at this starting point.
 		let existingCtx = this.parseData.tryGetContext(
+			parentCtx ? parentCtx.depth() + 1 : 1,
 			this.state,
 			ruleIndex,
 			this._input.LT(1).tokenIndex,

src/IncrementalParserData.ts

@@ -242,23 +242,37 @@ export class IncrementalParserData {
 	/**
 	 * Try to see if we have existing context for this state, rule and token position that may be reused.
 	 *
+	 * @param depth Current rule depth
 	 * @param state Parser state number - currently ignored.
 	 * @param ruleIndex Rule number
 	 * @param tokenIndex Token index in the *new* token stream
 	 */
-	public tryGetContext(state: number, ruleIndex: number, tokenIndex: number) {
-		return this.ruleStartMap.get(this.getKey(state, ruleIndex, tokenIndex));
+	public tryGetContext(
+		depth: number,
+		state: number,
+		ruleIndex: number,
+		tokenIndex: number,
+	) {
+		return this.ruleStartMap.get(
+			this.getKey(depth, state, ruleIndex, tokenIndex),
+		);
 	}
 
 	private getKeyFromContext(ctx: IncrementalParserRuleContext) {
 		return this.getKey(
+			ctx.depth(),
 			ctx.invokingState,
 			ctx.ruleIndex,
 			ctx.start.tokenIndex,
 		);
 	}
-	private getKey(state: number, rule: number, tokenindex: number) {
-		return `${state},${rule},${tokenindex}`;
+	private getKey(
+		depth: number,
+		state: number,
+		rule: number,
+		tokenindex: number,
+	) {
+		return `${depth},${rule},${tokenindex}`;
 	}
 	/**
 	 * 	Index a given parse tree and adjust the min/max ranges

src/IncrementalParserRuleContext.ts

@@ -5,8 +5,13 @@
 
 import { ParserRuleContext } from "./ParserRuleContext";
 import { Interval } from "./misc/Interval";
+import { RuleContext } from "./RuleContext";
 
 export class IncrementalParserRuleContext extends ParserRuleContext {
+	/* Avoid having to recompute depth on every single depth call */
+	private cachedDepth: number | undefined = undefined;
+	private cachedParent: RuleContext | undefined = undefined;
+
 	// This is an epoch number that can be used to tell which pieces were
 	// modified during a given incremental parse. The incremental parser
 	// adds the current epoch number to all rule contexts it creates.
@@ -40,4 +45,28 @@ export class IncrementalParserRuleContext extends ParserRuleContext {
 	set minMaxTokenIndex(index: Interval) {
 		this._minMaxTokenIndex = index;
 	}
+
+	/**
+	 * Compute the depth of this context in the parse tree.
+	 *
+	 * @notes The incremental parser uses a caching implemntation.
+	 *
+	 */
+	public depth(): number {
+		if (
+			this.cachedParent !== undefined &&
+			this.cachedParent === this._parent
+		) {
+			return this.cachedDepth as number;
+		}
+		let n = 1;
+		if (this._parent) {
+			let parentDepth = this._parent.depth();
+			this.cachedParent = this._parent;
+			this.cachedDepth = n = parentDepth + 1;
+		} else {
+			this.cachedDepth = n = 1;
+		}
+		return n;
+	}
 }

test/tool/TestIncremental.ts

@@ -174,8 +174,9 @@ const JAVA_PROGRAM_2_EXPECTATIONS: XPathExpectation[] = [
 		text: "publicstatic",
 		xpathRule: "//modifiers",
 	},
-	// This requires making recursive contexts work
-	/*{
+	/* This requires reusing individual recursion contexts */
+	/*
+	{
 		class: ExpressionContext,
 		text: "System.out.println",
 		xpathRule: "//statementExpression/expression/expression",
@@ -590,8 +591,9 @@ export class TestIncremental {
 		for (let i = 0; i < JAVA_PROGRAM_2_EXPECTATIONS.length - 1; ++i) {
 			JAVA_PROGRAM_2_EXPECTATIONS[i].epoch = startingEpoch + 1;
 		}
-		JAVA_PROGRAM_2_EXPECTATIONS[JAVA_PROGRAM_2_EXPECTATIONS.length - 1].epoch =
-			startingEpoch + 2;
+		JAVA_PROGRAM_2_EXPECTATIONS[
+			JAVA_PROGRAM_2_EXPECTATIONS.length - 1
+		].epoch = startingEpoch + 2;
 		this.verifyXPathExpectations(
 			parser,
 			secondTree,

tool/resources/org/antlr/v4/tool/templates/codegen/TypeScript/TypeScript.stg

@@ -424,8 +424,7 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
 	// Check whether we need to execute this rule.	
 	let guardResult = this.guardRule(this._ctx as IncrementalParserRuleContext, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>) as <currentRule.ctxType>;
 	// If we found an existing context that is valid, return it.
-	// NOTE: Recursive context reuse is currently disabled
-	if (0 && guardResult) {
+	if (guardResult) {
 		this._input.seek(guardResult.stop!.tokenIndex + 1);
 		return guardResult;
 	}