fix/indented code parsing

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -50,8 +50,12 @@ ILoggerFactory logger
 	}
 
 
-	public async Task ResolveDirectoryTree(Cancel ctx) =>
+	public async Task ResolveDirectoryTree(Cancel ctx)
+	{
+		_logger.LogInformation("Resolving tree");
 		await DocumentationSet.Tree.Resolve(ctx);
+		_logger.LogInformation("Resolved tree");
+	}
 
 	public async Task GenerateAll(Cancel ctx)
 	{
@@ -62,11 +66,30 @@ public async Task GenerateAll(Cancel ctx)
 		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
 			return;
 
-		_logger.LogInformation("Resolving tree");
 		await ResolveDirectoryTree(ctx);
-		_logger.LogInformation("Resolved tree");
 
+		await ProcessDocumentationFiles(offendingFiles, outputSeenChanges, ctx);
+
+		await ExtractEmbeddedStaticResources(ctx);
+
+
+		_logger.LogInformation($"Completing diagnostics channel");
+		Context.Collector.Channel.TryComplete();
 
+		_logger.LogInformation($"Generating documentation compilation state");
+		await GenerateDocumentationState(ctx);
+		_logger.LogInformation($"Generating links.json");
+		await GenerateLinkReference(ctx);
+
+		_logger.LogInformation($"Completing diagnostics channel");
+
+		await Context.Collector.StopAsync(ctx);
+
+		_logger.LogInformation($"Completed diagnostics channel");
+	}
+
+	private async Task ProcessDocumentationFiles(HashSet<string> offendingFiles, DateTimeOffset outputSeenChanges, Cancel ctx)
+	{
 		var processedFileCount = 0;
 		var exceptionCount = 0;
 		_ = Context.Collector.StartAsync(ctx);
@@ -91,7 +114,10 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 			if (processedFiles % 100 == 0)
 				_logger.LogInformation($"-> Handled {processedFiles} files");
 		});
+	}
 
+	private async Task ExtractEmbeddedStaticResources(Cancel ctx)
+	{
 		_logger.LogInformation($"Copying static files to output directory");
 		var embeddedStaticFiles = Assembly.GetExecutingAssembly()
 			.GetManifestResourceNames()
@@ -111,21 +137,6 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 			await resourceStream.CopyToAsync(stream, ctx);
 			_logger.LogInformation($"Copied static embedded resource {path}");
 		}
-
-
-		_logger.LogInformation($"Completing diagnostics channel");
-		Context.Collector.Channel.TryComplete();
-
-		_logger.LogInformation($"Generating documentation compilation state");
-		await GenerateDocumentationState(ctx);
-		_logger.LogInformation($"Generating links.json");
-		await GenerateLinkReference(ctx);
-
-		_logger.LogInformation($"Completing diagnostics channel");
-
-		await Context.Collector.StopAsync(ctx);
-
-		_logger.LogInformation($"Completed diagnostics channel");
 	}
 
 	private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, CancellationToken token)

src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs

@@ -39,19 +39,18 @@ public string? RepositoryName
 	// manual read because libgit2sharp is not yet AOT ready
 	public static GitCheckoutInformation Create(IFileSystem fileSystem)
 	{
-		// filesystem is not real so return a dummy
-		var fakeRef = Guid.NewGuid().ToString().Substring(0, 16);
 		if (fileSystem is not FileSystem)
 		{
 			return new GitCheckoutInformation
 			{
-				Branch = $"test-{fakeRef}",
+				Branch = $"test-e35fcb27-5f60-4e",
 				Remote = "elastic/docs-builder",
-				Ref = fakeRef,
+				Ref = "e35fcb27-5f60-4e",
 				RepositoryName = "docs-builder"
 			};
 		}
 
+		var fakeRef = Guid.NewGuid().ToString()[..16];
 		var gitConfig = Git(".git/config");
 		if (!gitConfig.Exists)
 			return Unavailable;

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -63,8 +63,8 @@ public string? NavigationTitle
 	private readonly Dictionary<string, PageTocItem> _tableOfContent = new(StringComparer.OrdinalIgnoreCase);
 	public IReadOnlyDictionary<string, PageTocItem> TableOfContents => _tableOfContent;
 
-	private readonly HashSet<string> _additionalLabels = new(StringComparer.OrdinalIgnoreCase);
-	public IReadOnlySet<string> AdditionalLabels => _additionalLabels;
+	private readonly HashSet<string> _anchors = new(StringComparer.OrdinalIgnoreCase);
+	public IReadOnlySet<string> Anchors => _anchors;
 
 	public string FilePath { get; }
 	public string FileName { get; }
@@ -89,7 +89,7 @@ public MarkdownFile[] YieldParents()
 		return parents.ToArray();
 	}
 
-	public async Task<MarkdownDocument> MinimalParse(Cancel ctx)
+	public async Task<MarkdownDocument> MinimalParseAsync(Cancel ctx)
 	{
 		var document = await MarkdownParser.MinimalParseAsync(SourceFile, ctx);
 		ReadDocumentInstructions(document);
@@ -99,7 +99,7 @@ public async Task<MarkdownDocument> MinimalParse(Cancel ctx)
 	public async Task<MarkdownDocument> ParseFullAsync(Cancel ctx)
 	{
 		if (!_instructionsParsed)
-			await MinimalParse(ctx);
+			await MinimalParseAsync(ctx);
 
 		var document = await MarkdownParser.ParseAsync(SourceFile, YamlFrontMatter, ctx);
 		return document;
@@ -171,22 +171,24 @@ private void ReadDocumentInstructions(MarkdownDocument document)
 				Slug = (h.Item2 ?? h.Item1).Slugify()
 			})
 			.ToList();
+
 		_tableOfContent.Clear();
 		foreach (var t in contents)
 			_tableOfContent[t.Slug] = t;
 
-		var labels = document.Descendants<DirectiveBlock>()
+		var inlineAnchros = document.Descendants<InlineAnchor>().ToList();
+
+		var anchors = document.Descendants<DirectiveBlock>()
 			.Select(b => b.CrossReferenceName)
 			.Where(l => !string.IsNullOrWhiteSpace(l))
 			.Select(s => s.Slugify())
 			.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
+			.Concat(_tableOfContent.Values.Select(t => t.Slug))
+			.Where(anchor => !string.IsNullOrEmpty(anchor))
 			.ToArray();
 
-		foreach (var label in labels)
-		{
-			if (!string.IsNullOrEmpty(label))
-				_additionalLabels.Add(label);
-		}
+		foreach (var label in anchors)
+			_anchors.Add(label);
 
 		_instructionsParsed = true;
 	}

src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs

@@ -146,10 +146,10 @@ public async Task Resolve(Cancel ctx = default)
 		if (_resolved)
 			return;
 
-		await Parallel.ForEachAsync(FilesInOrder, ctx, async (file, token) => await file.MinimalParse(token));
+		await Parallel.ForEachAsync(FilesInOrder, ctx, async (file, token) => await file.MinimalParseAsync(token));
 		await Parallel.ForEachAsync(GroupsInOrder, ctx, async (group, token) => await group.Resolve(token));
 
-		await (Index?.MinimalParse(ctx) ?? Task.CompletedTask);
+		await (Index?.MinimalParseAsync(ctx) ?? Task.CompletedTask);
 
 		_resolved = true;
 	}

src/Elastic.Markdown/IO/State/LinkReference.cs

@@ -7,6 +7,13 @@
 
 namespace Elastic.Markdown.IO.State;
 
+public record LinkMetadata
+{
+	[JsonPropertyName("anchors")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+	public required string[]? Anchors { get; init; } = [];
+}
+
 public record LinkReference
 {
 	[JsonPropertyName("origin")]
@@ -15,8 +22,9 @@ public record LinkReference
 	[JsonPropertyName("url_path_prefix")]
 	public required string? UrlPathPrefix { get; init; }
 
+	/// Mapping of relative filepath and all the page's anchors for deeplinks
 	[JsonPropertyName("links")]
-	public required string[] Links { get; init; } = [];
+	public required Dictionary<string, LinkMetadata> Links { get; init; } = [];
 
 	[JsonPropertyName("cross_links")]
 	public required string[] CrossLinks { get; init; } = [];
@@ -25,7 +33,12 @@ public static LinkReference Create(DocumentationSet set)
 	{
 		var crossLinks = set.Context.Collector.CrossLinks.ToHashSet().ToArray();
 		var links = set.MarkdownFiles.Values
-			.Select(m => m.RelativePath).ToArray();
+			.Select(m => (m.RelativePath, m.Anchors))
+			.ToDictionary(k => k.RelativePath, v =>
+			{
+				var anchors = v.Anchors.Count == 0 ? null : v.Anchors.ToArray();
+				return new LinkMetadata { Anchors = anchors };
+			});
 		return new LinkReference
 		{
 			UrlPathPrefix = set.Context.UrlPathPrefix,

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -135,11 +135,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
 
 			if (!string.IsNullOrEmpty(anchor))
 			{
-				if (markdown == null || (!markdown.TableOfContents.TryGetValue(anchor, out var heading)
-					&& !markdown.AdditionalLabels.Contains(anchor)))
+				if (markdown == null || !markdown.Anchors.Contains(anchor))
 					processor.EmitError(line, column, length, $"`{anchor}` does not exist in {markdown?.FileName}.");
-
-				else if (link.FirstChild == null && heading != null)
+				else if (link.FirstChild == null && markdown.TableOfContents.TryGetValue(anchor, out var heading))
 					title += " > " + heading.Heading;
 
 			}

src/Elastic.Markdown/Myst/MarkdownParser.cs

@@ -14,6 +14,7 @@
 using Elastic.Markdown.Myst.Substitution;
 using Markdig;
 using Markdig.Extensions.EmphasisExtras;
+using Markdig.Parsers;
 using Markdig.Syntax;
 
 namespace Elastic.Markdown.Myst;
@@ -28,34 +29,56 @@ public class MarkdownParser(
 
 	private BuildContext Context { get; } = context;
 
-	public static MarkdownPipeline MinimalPipeline { get; } =
-		new MarkdownPipelineBuilder()
-			.UseYamlFrontMatter()
-			.UseInlineAnchors()
-			.UseHeadingsWithSlugs()
-			.UseDirectives()
-			.Build();
+	// ReSharper disable once InconsistentNaming
+	private static MarkdownPipeline? _minimalPipeline;
+	public static MarkdownPipeline MinimalPipeline
+	{
+		get
+		{
+			if (_minimalPipeline is not null) return _minimalPipeline;
+			var builder = new MarkdownPipelineBuilder()
+				.UseYamlFrontMatter()
+				.UseInlineAnchors()
+				.UseHeadingsWithSlugs()
+				.UseDirectives();
+
+			builder.BlockParsers.TryRemove<IndentedCodeBlockParser>();
+			_minimalPipeline = builder.Build();
+			return _minimalPipeline;
+
+		}
+	}
+
+	// ReSharper disable once InconsistentNaming
+	private static MarkdownPipeline? _pipeline;
+	public static MarkdownPipeline Pipeline {
+		get
+		{
+			if (_pipeline is not null) return _pipeline;
 
-	public static MarkdownPipeline Pipeline { get; } =
-		new MarkdownPipelineBuilder()
-			.EnableTrackTrivia()
-			.UseInlineAnchors()
-			.UsePreciseSourceLocation()
-			.UseDiagnosticLinks()
-			.UseHeadingsWithSlugs()
-			.UseEmphasisExtras(EmphasisExtraOptions.Default)
-			.UseSoftlineBreakAsHardlineBreak()
-			.UseSubstitution()
-			.UseComments()
-			.UseYamlFrontMatter()
-			.UseGridTables()
-			.UsePipeTables()
-			.UseDirectives()
-			.UseDefinitionLists()
-			.UseEnhancedCodeBlocks()
-			.DisableHtml()
-			.UseHardBreaks()
-			.Build();
+			var builder = new MarkdownPipelineBuilder()
+				.EnableTrackTrivia()
+				.UseInlineAnchors()
+				.UsePreciseSourceLocation()
+				.UseDiagnosticLinks()
+				.UseHeadingsWithSlugs()
+				.UseEmphasisExtras(EmphasisExtraOptions.Default)
+				.UseSoftlineBreakAsHardlineBreak()
+				.UseSubstitution()
+				.UseComments()
+				.UseYamlFrontMatter()
+				.UseGridTables()
+				.UsePipeTables()
+				.UseDirectives()
+				.UseDefinitionLists()
+				.UseEnhancedCodeBlocks()
+				.DisableHtml()
+				.UseHardBreaks();
+			builder.BlockParsers.TryRemove<IndentedCodeBlockParser>();
+			_pipeline = builder.Build();
+			return _pipeline;
+		}
+	}
 
 	public ConfigurationFile Configuration { get; } = configuration;
 

tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs

@@ -26,7 +26,7 @@ public void EmitsLinks() =>
 
 	[Fact]
 	public void ShouldNotIncludeSnippets() =>
-		Reference.Links.Should().NotContain(l => l.Contains("_snippets/"));
+		Reference.Links.Should().NotContain(l => l.Key.Contains("_snippets/"));
 }
 
 public class GitCheckoutInformationTests(ITestOutputHelper output) : NavigationTestsBase(output)

tests/authoring/Container/DefinitionLists.fs

@@ -41,7 +41,7 @@ This is my `definition`
 """
 
     [<Fact>]
-    let ``validate HTML`` () =
+    let ``validate HTML 2`` () =
         markdown |> convertsToHtml """
              <dl>
                 <dt>This is my <code>definition</code> </dt>
@@ -56,4 +56,4 @@ This is my `definition`
              </dl>
             """
     [<Fact>]
-    let ``has no errors`` () = markdown |> hasNoErrors
+    let ``has no errors 2`` () = markdown |> hasNoErrors

tests/authoring/Framework/ErrorCollectorAssertions.fs

@@ -14,11 +14,14 @@ open Swensen.Unquote
 module DiagnosticsCollectorAssertions =
 
     [<DebuggerStepThrough>]
-    let hasNoErrors (actual: GenerateResult) =
-        test <@ actual.Context.Collector.Errors = 0 @>
+    let hasNoErrors (actual: Lazy<GeneratorResults>) =
+        let actual = actual.Value
+        let errors = actual.Context.Collector.Errors
+        test <@ errors = 0 @>
 
     [<DebuggerStepThrough>]
-    let hasError (expected: string) (actual: GenerateResult) =
+    let hasError (expected: string) (actual: Lazy<GeneratorResults>) =
+        let actual = actual.Value
         actual.Context.Collector.Errors |> shouldBeGreaterThan 0
         let errorDiagnostics = actual.Context.Collector.Diagnostics
                                    .Where(fun d -> d.Severity = Severity.Error)