Add anchors to link json artifact (#353)

* Add anchors to link json artifact

Refactored `LinkReference` to use a dictionary for better anchor tracking and adjusted anchor validations. Added extensive test coverage, revamped test setup, and improved generator logic for markdown files.```

* rename union creation methods

* add license headers

* make links on object holding anchors

* dotnet format

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -50,8 +50,12 @@ ILoggerFactory logger
 	}
 
 
-	public async Task ResolveDirectoryTree(Cancel ctx) =>
+	public async Task ResolveDirectoryTree(Cancel ctx)
+	{
+		_logger.LogInformation("Resolving tree");
 		await DocumentationSet.Tree.Resolve(ctx);
+		_logger.LogInformation("Resolved tree");
+	}
 
 	public async Task GenerateAll(Cancel ctx)
 	{
@@ -62,11 +66,30 @@ public async Task GenerateAll(Cancel ctx)
 		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
 			return;
 
-		_logger.LogInformation("Resolving tree");
 		await ResolveDirectoryTree(ctx);
-		_logger.LogInformation("Resolved tree");
 
+		await ProcessDocumentationFiles(offendingFiles, outputSeenChanges, ctx);
+
+		await ExtractEmbeddedStaticResources(ctx);
+
+
+		_logger.LogInformation($"Completing diagnostics channel");
+		Context.Collector.Channel.TryComplete();
 
+		_logger.LogInformation($"Generating documentation compilation state");
+		await GenerateDocumentationState(ctx);
+		_logger.LogInformation($"Generating links.json");
+		await GenerateLinkReference(ctx);
+
+		_logger.LogInformation($"Completing diagnostics channel");
+
+		await Context.Collector.StopAsync(ctx);
+
+		_logger.LogInformation($"Completed diagnostics channel");
+	}
+
+	private async Task ProcessDocumentationFiles(HashSet<string> offendingFiles, DateTimeOffset outputSeenChanges, Cancel ctx)
+	{
 		var processedFileCount = 0;
 		var exceptionCount = 0;
 		_ = Context.Collector.StartAsync(ctx);
@@ -91,7 +114,10 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 			if (processedFiles % 100 == 0)
 				_logger.LogInformation($"-> Handled {processedFiles} files");
 		});
+	}
 
+	private async Task ExtractEmbeddedStaticResources(Cancel ctx)
+	{
 		_logger.LogInformation($"Copying static files to output directory");
 		var embeddedStaticFiles = Assembly.GetExecutingAssembly()
 			.GetManifestResourceNames()
@@ -111,21 +137,6 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 			await resourceStream.CopyToAsync(stream, ctx);
 			_logger.LogInformation($"Copied static embedded resource {path}");
 		}
-
-
-		_logger.LogInformation($"Completing diagnostics channel");
-		Context.Collector.Channel.TryComplete();
-
-		_logger.LogInformation($"Generating documentation compilation state");
-		await GenerateDocumentationState(ctx);
-		_logger.LogInformation($"Generating links.json");
-		await GenerateLinkReference(ctx);
-
-		_logger.LogInformation($"Completing diagnostics channel");
-
-		await Context.Collector.StopAsync(ctx);
-
-		_logger.LogInformation($"Completed diagnostics channel");
 	}
 
 	private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, CancellationToken token)

src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs

@@ -39,19 +39,18 @@ public string? RepositoryName
 	// manual read because libgit2sharp is not yet AOT ready
 	public static GitCheckoutInformation Create(IFileSystem fileSystem)
 	{
-		// filesystem is not real so return a dummy
-		var fakeRef = Guid.NewGuid().ToString().Substring(0, 16);
 		if (fileSystem is not FileSystem)
 		{
 			return new GitCheckoutInformation
 			{
-				Branch = $"test-{fakeRef}",
+				Branch = $"test-e35fcb27-5f60-4e",
 				Remote = "elastic/docs-builder",
-				Ref = fakeRef,
+				Ref = "e35fcb27-5f60-4e",
 				RepositoryName = "docs-builder"
 			};
 		}
 
+		var fakeRef = Guid.NewGuid().ToString()[..16];
 		var gitConfig = Git(".git/config");
 		if (!gitConfig.Exists)
 			return Unavailable;

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -63,8 +63,8 @@ public string? NavigationTitle
 	private readonly Dictionary<string, PageTocItem> _tableOfContent = new(StringComparer.OrdinalIgnoreCase);
 	public IReadOnlyDictionary<string, PageTocItem> TableOfContents => _tableOfContent;
 
-	private readonly HashSet<string> _additionalLabels = new(StringComparer.OrdinalIgnoreCase);
-	public IReadOnlySet<string> AdditionalLabels => _additionalLabels;
+	private readonly HashSet<string> _anchors = new(StringComparer.OrdinalIgnoreCase);
+	public IReadOnlySet<string> Anchors => _anchors;
 
 	public string FilePath { get; }
 	public string FileName { get; }
@@ -171,22 +171,22 @@ private void ReadDocumentInstructions(MarkdownDocument document)
 				Slug = (h.Item2 ?? h.Item1).Slugify()
 			})
 			.ToList();
+
 		_tableOfContent.Clear();
 		foreach (var t in contents)
 			_tableOfContent[t.Slug] = t;
 
-		var labels = document.Descendants<DirectiveBlock>()
+		var anchors = document.Descendants<DirectiveBlock>()
 			.Select(b => b.CrossReferenceName)
 			.Where(l => !string.IsNullOrWhiteSpace(l))
 			.Select(s => s.Slugify())
 			.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
+			.Concat(_tableOfContent.Values.Select(t => t.Slug))
+			.Where(anchor => !string.IsNullOrEmpty(anchor))
 			.ToArray();
 
-		foreach (var label in labels)
-		{
-			if (!string.IsNullOrEmpty(label))
-				_additionalLabels.Add(label);
-		}
+		foreach (var label in anchors)
+			_anchors.Add(label);
 
 		_instructionsParsed = true;
 	}

src/Elastic.Markdown/IO/State/LinkReference.cs

@@ -7,6 +7,13 @@
 
 namespace Elastic.Markdown.IO.State;
 
+public record LinkMetadata
+{
+	[JsonPropertyName("anchors")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+	public required string[]? Anchors { get; init; } = [];
+}
+
 public record LinkReference
 {
 	[JsonPropertyName("origin")]
@@ -15,8 +22,9 @@ public record LinkReference
 	[JsonPropertyName("url_path_prefix")]
 	public required string? UrlPathPrefix { get; init; }
 
+	/// Mapping of relative filepath and all the page's anchors for deeplinks
 	[JsonPropertyName("links")]
-	public required string[] Links { get; init; } = [];
+	public required Dictionary<string, LinkMetadata> Links { get; init; } = [];
 
 	[JsonPropertyName("cross_links")]
 	public required string[] CrossLinks { get; init; } = [];
@@ -25,7 +33,12 @@ public static LinkReference Create(DocumentationSet set)
 	{
 		var crossLinks = set.Context.Collector.CrossLinks.ToHashSet().ToArray();
 		var links = set.MarkdownFiles.Values
-			.Select(m => m.RelativePath).ToArray();
+			.Select(m => (m.RelativePath, m.Anchors))
+			.ToDictionary(k => k.RelativePath, v =>
+			{
+				var anchors = v.Anchors.Count == 0 ? null : v.Anchors.ToArray();
+				return new LinkMetadata { Anchors = anchors };
+			});
 		return new LinkReference
 		{
 			UrlPathPrefix = set.Context.UrlPathPrefix,

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -135,11 +135,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
 
 			if (!string.IsNullOrEmpty(anchor))
 			{
-				if (markdown == null || (!markdown.TableOfContents.TryGetValue(anchor, out var heading)
-					&& !markdown.AdditionalLabels.Contains(anchor)))
+				if (markdown == null || !markdown.Anchors.Contains(anchor))
 					processor.EmitError(line, column, length, $"`{anchor}` does not exist in {markdown?.FileName}.");
-
-				else if (link.FirstChild == null && heading != null)
+				else if (link.FirstChild == null && markdown.TableOfContents.TryGetValue(anchor, out var heading))
 					title += " > " + heading.Heading;
 
 			}

tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs

@@ -26,7 +26,7 @@ public void EmitsLinks() =>
 
 	[Fact]
 	public void ShouldNotIncludeSnippets() =>
-		Reference.Links.Should().NotContain(l => l.Contains("_snippets/"));
+		Reference.Links.Should().NotContain(l => l.Key.Contains("_snippets/"));
 }
 
 public class GitCheckoutInformationTests(ITestOutputHelper output) : NavigationTestsBase(output)

tests/authoring/Container/DefinitionLists.fs

@@ -41,7 +41,7 @@ This is my `definition`
 """
 
     [<Fact>]
-    let ``validate HTML`` () =
+    let ``validate HTML 2`` () =
         markdown |> convertsToHtml """
              <dl>
                 <dt>This is my <code>definition</code> </dt>
@@ -56,4 +56,4 @@ This is my `definition`
              </dl>
             """
     [<Fact>]
-    let ``has no errors`` () = markdown |> hasNoErrors
+    let ``has no errors 2`` () = markdown |> hasNoErrors

tests/authoring/Framework/ErrorCollectorAssertions.fs

@@ -14,11 +14,13 @@ open Swensen.Unquote
 module DiagnosticsCollectorAssertions =
 
     [<DebuggerStepThrough>]
-    let hasNoErrors (actual: GenerateResult) =
+    let hasNoErrors (actual: Lazy<GeneratorResults>) =
+        let actual = actual.Value
         test <@ actual.Context.Collector.Errors = 0 @>
 
     [<DebuggerStepThrough>]
-    let hasError (expected: string) (actual: GenerateResult) =
+    let hasError (expected: string) (actual: Lazy<GeneratorResults>) =
+        let actual = actual.Value
         actual.Context.Collector.Errors |> shouldBeGreaterThan 0
         let errorDiagnostics = actual.Context.Collector.Diagnostics
                                    .Where(fun d -> d.Severity = Severity.Error)

tests/authoring/Framework/HtmlAssertions.fs

@@ -89,35 +89,36 @@ actual: {actual}
         |> Seq.iter _.ToHtml(sw, PrettyMarkupFormatter())
         sw.ToString()
 
-    [<DebuggerStepThrough>]
-    let convertsToHtml ([<LanguageInjection("html")>]expected: string) (actual: GenerateResult) =
+    let private createDiff expected actual =
         let diffs =
             DiffBuilder
-                .Compare(actual.Html)
+                .Compare(actual)
                 .WithTest(expected)
                 .Build()
 
-        let diff = htmlDiffString diffs
-        match diff with
+        let deepComparision = htmlDiffString diffs
+        match deepComparision with
         | s when String.IsNullOrEmpty s -> ()
         | s ->
             let expectedHtml = prettyHtml expected
-            let actualHtml = prettyHtml actual.Html
-            let textDiff =
-                InlineDiffBuilder.Diff(expectedHtml, actualHtml).Lines
-                |> Seq.map(fun l ->
-                    match l.Type with
-                    | ChangeType.Deleted -> "- " + l.Text
-                    | ChangeType.Modified -> "+ " + l.Text
-                    | ChangeType.Inserted -> "+ " + l.Text
-                    | _ -> " " + l.Text
-                )
-                |> String.concat "\n"
+            let actualHtml = prettyHtml actual
+            let textDiff = diff expectedHtml actualHtml
             let msg = $"""Html was not equal
 {textDiff}
 
-{diff}
+{deepComparision}
 """
             raise (XunitException(msg))
 
+    [<DebuggerStepThrough>]
+    let convertsToHtml ([<LanguageInjection("html")>]expected: string) (actual: Lazy<GeneratorResults>) =
+        let actual = actual.Value
+
+        let defaultFile = actual.MarkdownResults |> Seq.head
+        createDiff expected defaultFile.Html
+
+    [<DebuggerStepThrough>]
+    let toHtml ([<LanguageInjection("html")>]expected: string) (actual: MarkdownResult) =
+        createDiff expected actual.Html
+
 

tests/authoring/Framework/MarkdownResultsAssertions.fs

@@ -0,0 +1,81 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace authoring
+
+open System.Diagnostics
+open System.Text.Json
+open DiffPlex.DiffBuilder
+open DiffPlex.DiffBuilder.Model
+open FsUnit.Xunit
+open JetBrains.Annotations
+open Xunit.Sdk
+
+[<AutoOpen>]
+module ResultsAssertions =
+
+    let diff expected actual =
+        let diffLines = InlineDiffBuilder.Diff(expected, actual).Lines
+
+        let mutatedCount =
+            diffLines
+            |> Seq.filter (fun l ->
+                match l.Type with
+                | ChangeType.Modified -> true
+                | ChangeType.Inserted -> true
+                | _ -> false
+            )
+            |> Seq.length
+
+        let actualLineLength = actual.Split("\n").Length
+        match mutatedCount with
+        | 0 -> ""
+        | _ when mutatedCount >= actualLineLength -> $"Mutations {mutatedCount} on all {actualLineLength} showing actual: \n\n{actual}"
+        | _ ->
+            diffLines
+            |> Seq.map(fun l ->
+                match l.Type with
+                | ChangeType.Deleted -> "- " + l.Text
+                | ChangeType.Modified -> "+ " + l.Text
+                | ChangeType.Inserted -> "+ " + l.Text
+                | _ -> " " + l.Text
+            )
+            |> String.concat "\n"
+
+
+    [<DebuggerStepThrough>]
+    let converts file (results: Lazy<GeneratorResults>) =
+        let results = results.Value
+
+        let result =
+            results.MarkdownResults
+            |> Seq.tryFind (fun m -> m.File.RelativePath = file)
+
+        match result with
+        | None ->
+            raise (XunitException($"{file} not part of the markdown results"))
+        | Some result -> result
+
+[<AutoOpen>]
+module JsonAssertions =
+
+    [<DebuggerStepThrough>]
+    let convertsToJson artifact ([<LanguageInjection("json")>]expected: string) (actual: Lazy<GeneratorResults>) =
+        let actual = actual.Value
+        let fs = actual.Context.ReadFileSystem
+
+        let fi = fs.FileInfo.New(artifact)
+        if not <| fi.Exists then
+            raise (XunitException($"{artifact} is not part of the output"))
+
+        let actual = fs.File.ReadAllText(fi.FullName)
+        use actualJson = JsonDocument.Parse(actual);
+        let actual = JsonSerializer.Serialize(actualJson, JsonSerializerOptions(WriteIndented = true))
+
+        use expectedJson = JsonDocument.Parse(expected);
+        let expected = JsonSerializer.Serialize(expectedJson, JsonSerializerOptions(WriteIndented = true))
+
+        diff expected actual |> should be NullOrEmptyString
+
+