From d4e3e021dd35851718ae35fb775b121a4a0db07e Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Wed, 26 Nov 2025 13:44:10 +0100 Subject: [PATCH 1/5] initial c# cut --- .gitignore | 14 +- dotnet/ConnectedWorkbooks.sln | 56 +++ .../ConnectedWorkbooks.csproj | 17 + .../Internal/ArrayReader.cs | 51 +++ .../Internal/ByteHelpers.cs | 31 ++ .../Internal/CellReferenceHelper.cs | 78 +++++ .../Internal/EmbeddedTemplateLoader.cs | 28 ++ .../Internal/ExcelArchive.cs | 125 +++++++ .../ConnectedWorkbooks/Internal/GridParser.cs | 119 +++++++ .../Internal/MashupDocumentParser.cs | 138 ++++++++ .../Internal/PowerQueryGenerator.cs | 13 + .../Internal/PqUtilities.cs | 63 ++++ .../Internal/WorkbookConstants.cs | 37 ++ .../Internal/WorkbookEditor.cs | 329 ++++++++++++++++++ .../Internal/XmlEncodingHelper.cs | 51 +++ .../ConnectedWorkbooks/Internal/XmlNames.cs | 68 ++++ .../Models/DocumentProperties.cs | 20 ++ .../Models/FileConfiguration.cs | 26 ++ dotnet/src/ConnectedWorkbooks/Models/Grid.cs | 10 + .../ConnectedWorkbooks/Models/GridConfig.cs | 21 ++ .../ConnectedWorkbooks/Models/QueryInfo.cs | 27 ++ .../ConnectedWorkbooks/Models/TableData.cs | 13 + .../Models/TemplateSettings.cs | 14 + .../SIMPLE_BLANK_TABLE_TEMPLATE.xlsx | Bin 0 -> 10234 bytes .../SIMPLE_QUERY_WORKBOOK_TEMPLATE.xlsx | Bin 0 -> 17284 bytes .../src/ConnectedWorkbooks/WorkbookManager.cs | 46 +++ .../ConnectedWorkbooks.Tests.csproj | 24 ++ .../MSTestSettings.cs | 5 + .../WorkbookManagerTests.cs | 92 +++++ .eslintrc.js => typescript/.eslintrc.js | 0 .prettierrc => typescript/.prettierrc | 0 README.md => typescript/README.md | 0 babel.config.js => typescript/babel.config.js | 0 .../jest.config.jsdom.js | 0 .../jest.config.node.js | 0 .../package-lock.json | 22 +- package.json => typescript/package.json | 0 {src => typescript/src}/generators.ts | 0 {src => typescript/src}/index.ts | 0 {src => typescript/src}/types.ts | 0 {src => typescript/src}/utils/arrayUtils.ts | 0 {src => typescript/src}/utils/constants.ts | 0 .../src}/utils/documentUtils.ts | 0 {src => typescript/src}/utils/gridUtils.ts | 0 {src => typescript/src}/utils/htmlUtils.ts | 0 {src => typescript/src}/utils/index.ts | 0 .../src}/utils/mashupDocumentParser.ts | 0 {src => typescript/src}/utils/pqUtils.ts | 0 {src => typescript/src}/utils/tableUtils.ts | 0 .../src}/utils/xmlInnerPartsUtils.ts | 0 .../src}/utils/xmlPartsUtils.ts | 0 {src => typescript/src}/workbookManager.ts | 0 {src => typescript/src}/workbookTemplate.ts | 0 .../tests}/arrayUtils.test.ts | 0 .../tests}/documentUtils.test.ts | 0 {tests => typescript/tests}/gridUtils.test.ts | 0 {tests => typescript/tests}/htmlUtils.test.ts | 0 .../tests}/mashupDocumentParser.test.ts | 0 {tests => typescript/tests}/mocks/PqMock.ts | 0 {tests => typescript/tests}/mocks/index.ts | 0 .../tests}/mocks/section1mSimpleQueryMock.ts | 0 .../tests}/mocks/workbookMocks.ts | 0 {tests => typescript/tests}/mocks/xmlMocks.ts | 0 .../tests}/tableUtils.test.ts | 0 .../tests}/workbookQueryTemplate.test.ts | 0 .../tests}/workbookTableTemplate.test.ts | 0 .../tests}/xmlInnerPartsUtils.test.ts | 0 tsconfig.json => typescript/tsconfig.json | 0 .../tsconfig.test.json | 0 .../webpack.config.js | 0 70 files changed, 1529 insertions(+), 9 deletions(-) create mode 100644 dotnet/ConnectedWorkbooks.sln create mode 100644 dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/ByteHelpers.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/PowerQueryGenerator.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/WorkbookConstants.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/XmlEncodingHelper.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/XmlNames.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Models/DocumentProperties.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Models/FileConfiguration.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Models/Grid.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Models/GridConfig.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Models/QueryInfo.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Models/TableData.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Models/TemplateSettings.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Templates/SIMPLE_BLANK_TABLE_TEMPLATE.xlsx create mode 100644 dotnet/src/ConnectedWorkbooks/Templates/SIMPLE_QUERY_WORKBOOK_TEMPLATE.xlsx create mode 100644 dotnet/src/ConnectedWorkbooks/WorkbookManager.cs create mode 100644 dotnet/tests/ConnectedWorkbooks.Tests/ConnectedWorkbooks.Tests.csproj create mode 100644 dotnet/tests/ConnectedWorkbooks.Tests/MSTestSettings.cs create mode 100644 dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs rename .eslintrc.js => typescript/.eslintrc.js (100%) rename .prettierrc => typescript/.prettierrc (100%) rename README.md => typescript/README.md (100%) rename babel.config.js => typescript/babel.config.js (100%) rename jest.config.jsdom.js => typescript/jest.config.jsdom.js (100%) rename jest.config.node.js => typescript/jest.config.node.js (100%) rename package-lock.json => typescript/package-lock.json (99%) rename package.json => typescript/package.json (100%) rename {src => typescript/src}/generators.ts (100%) rename {src => typescript/src}/index.ts (100%) rename {src => typescript/src}/types.ts (100%) rename {src => typescript/src}/utils/arrayUtils.ts (100%) rename {src => typescript/src}/utils/constants.ts (100%) rename {src => typescript/src}/utils/documentUtils.ts (100%) rename {src => typescript/src}/utils/gridUtils.ts (100%) rename {src => typescript/src}/utils/htmlUtils.ts (100%) rename {src => typescript/src}/utils/index.ts (100%) rename {src => typescript/src}/utils/mashupDocumentParser.ts (100%) rename {src => typescript/src}/utils/pqUtils.ts (100%) rename {src => typescript/src}/utils/tableUtils.ts (100%) rename {src => typescript/src}/utils/xmlInnerPartsUtils.ts (100%) rename {src => typescript/src}/utils/xmlPartsUtils.ts (100%) rename {src => typescript/src}/workbookManager.ts (100%) rename {src => typescript/src}/workbookTemplate.ts (100%) rename {tests => typescript/tests}/arrayUtils.test.ts (100%) rename {tests => typescript/tests}/documentUtils.test.ts (100%) rename {tests => typescript/tests}/gridUtils.test.ts (100%) rename {tests => typescript/tests}/htmlUtils.test.ts (100%) rename {tests => typescript/tests}/mashupDocumentParser.test.ts (100%) rename {tests => typescript/tests}/mocks/PqMock.ts (100%) rename {tests => typescript/tests}/mocks/index.ts (100%) rename {tests => typescript/tests}/mocks/section1mSimpleQueryMock.ts (100%) rename {tests => typescript/tests}/mocks/workbookMocks.ts (100%) rename {tests => typescript/tests}/mocks/xmlMocks.ts (100%) rename {tests => typescript/tests}/tableUtils.test.ts (100%) rename {tests => typescript/tests}/workbookQueryTemplate.test.ts (100%) rename {tests => typescript/tests}/workbookTableTemplate.test.ts (100%) rename {tests => typescript/tests}/xmlInnerPartsUtils.test.ts (100%) rename tsconfig.json => typescript/tsconfig.json (100%) rename tsconfig.test.json => typescript/tsconfig.test.json (100%) rename webpack.config.js => typescript/webpack.config.js (100%) diff --git a/.gitignore b/.gitignore index 635044a..150df22 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,10 @@ PublishScripts/ node_modules/ +# TypeScript artifacts +typescript/node_modules/ +typescript/dist/ + # Build files dist/ @@ -20,4 +24,12 @@ dist/ coverage/ # bundles -*.tgz \ No newline at end of file +*.tgz + +# .NET artifacts +dotnet/**/bin/ +dotnet/**/obj/ + +# Python artifacts +python/.venv/ +python/**/__pycache__/ \ No newline at end of file diff --git a/dotnet/ConnectedWorkbooks.sln b/dotnet/ConnectedWorkbooks.sln new file mode 100644 index 0000000..83b9b23 --- /dev/null +++ b/dotnet/ConnectedWorkbooks.sln @@ -0,0 +1,56 @@ +ο»Ώ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConnectedWorkbooks", "src\ConnectedWorkbooks\ConnectedWorkbooks.csproj", "{5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05-4346-4AA6-1389-037BE0695223}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConnectedWorkbooks.Tests", "tests\ConnectedWorkbooks.Tests\ConnectedWorkbooks.Tests.csproj", "{2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Debug|x64.ActiveCfg = Debug|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Debug|x64.Build.0 = Debug|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Debug|x86.ActiveCfg = Debug|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Debug|x86.Build.0 = Debug|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Release|Any CPU.Build.0 = Release|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Release|x64.ActiveCfg = Release|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Release|x64.Build.0 = Release|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Release|x86.ActiveCfg = Release|Any CPU + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8}.Release|x86.Build.0 = Release|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Debug|x64.ActiveCfg = Debug|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Debug|x64.Build.0 = Debug|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Debug|x86.ActiveCfg = Debug|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Debug|x86.Build.0 = Debug|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Release|Any CPU.Build.0 = Release|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Release|x64.ActiveCfg = Release|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Release|x64.Build.0 = Release|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Release|x86.ActiveCfg = Release|Any CPU + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {5C9D0FC6-1585-4A60-B814-1A2A0B28E9D8} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {2A8D653E-7A73-4DC9-BED0-EA15DAB20D0A} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + EndGlobalSection +EndGlobal diff --git a/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj b/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj new file mode 100644 index 0000000..a00bdbf --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj @@ -0,0 +1,17 @@ +ο»Ώ + + + net8.0 + enable + enable + + + + + + + + + + + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs b/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs new file mode 100644 index 0000000..3a0eb1d --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Buffers.Binary; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal sealed class ArrayReader +{ + private readonly ReadOnlyMemory _buffer; + private int _offset; + + public ArrayReader(byte[] buffer) + { + _buffer = new ReadOnlyMemory(buffer); + _offset = 0; + } + + public byte[] ReadBytes(int count) + { + if (_offset + count > _buffer.Length) + { + throw new InvalidOperationException("Attempted to read beyond the length of the buffer."); + } + + var slice = _buffer.Slice(_offset, count).ToArray(); + _offset += count; + return slice; + } + + public int ReadInt32() + { + var span = _buffer.Span; + if (_offset + sizeof(int) > span.Length) + { + throw new InvalidOperationException("Attempted to read beyond the length of the buffer."); + } + + var value = BinaryPrimitives.ReadInt32LittleEndian(span.Slice(_offset, sizeof(int))); + _offset += sizeof(int); + return value; + } + + public byte[] ReadToEnd() + { + var slice = _buffer.Slice(_offset).ToArray(); + _offset = _buffer.Length; + return slice; + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/ByteHelpers.cs b/dotnet/src/ConnectedWorkbooks/Internal/ByteHelpers.cs new file mode 100644 index 0000000..4de3293 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/ByteHelpers.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Buffers.Binary; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class ByteHelpers +{ + public static byte[] Concat(params byte[][] arrays) + { + var total = arrays.Sum(a => a.Length); + var result = new byte[total]; + var offset = 0; + foreach (var array in arrays) + { + Buffer.BlockCopy(array, 0, result, offset, array.Length); + offset += array.Length; + } + + return result; + } + + public static byte[] GetInt32Bytes(int value) + { + var buffer = new byte[4]; + BinaryPrimitives.WriteInt32LittleEndian(buffer, value); + return buffer; + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs b/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs new file mode 100644 index 0000000..37aa54a --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Linq; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class CellReferenceHelper +{ + public static (int Row, int Column) GetStartPosition(string reference) + { + // Reference format "A1" or "A1:B5"; we only care about the first cell + var start = reference.Split(':')[0]; + var letters = new string(start.TakeWhile(char.IsLetter).ToArray()); + var digits = new string(start.SkipWhile(char.IsLetter).ToArray()); + var column = ColumnNameToNumber(letters); + var row = int.TryParse(digits, out var parsedRow) ? parsedRow : 1; + return (row, column); + } + + public static string ColumnNumberToName(int columnIndex) + { + columnIndex++; // zero-based to one-based + var columnName = string.Empty; + while (columnIndex > 0) + { + var remainder = (columnIndex - 1) % 26; + columnName = (char)('A' + remainder) + columnName; + columnIndex = (columnIndex - remainder - 1) / 26; + } + + return columnName; + } + + public static string BuildReference((int Row, int Column) start, int columnCount, int rowCount) + { + var endColumnIndex = start.Column - 1 + columnCount; + var endRow = start.Row - 1 + rowCount; + var startRef = $"{ColumnNumberToName(start.Column - 1)}{start.Row}"; + var endRef = $"{ColumnNumberToName(endColumnIndex - 1)}{endRow}"; + return $"{startRef}:{endRef}"; + } + + public static string WithAbsolute(string reference) + { + var (row, column) = GetStartPosition(reference); + var (endRow, endColumn) = GetEndPosition(reference); + return $"!${ColumnNumberToName(column - 1)}${row}:${ColumnNumberToName(endColumn - 1)}${endRow}"; + } + + private static (int Row, int Column) GetEndPosition(string reference) + { + var parts = reference.Split(':'); + var target = parts.Length == 2 ? parts[1] : parts[0]; + var letters = new string(target.TakeWhile(char.IsLetter).ToArray()); + var digits = new string(target.SkipWhile(char.IsLetter).ToArray()); + var column = ColumnNameToNumber(letters); + var row = int.TryParse(digits, out var parsedRow) ? parsedRow : 1; + return (row, column); + } + + private static int ColumnNameToNumber(string columnName) + { + if (string.IsNullOrWhiteSpace(columnName)) + { + return 1; + } + + var result = 0; + foreach (var ch in columnName.ToUpperInvariant()) + { + result = result * 26 + (ch - 'A' + 1); + } + + return result; + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs b/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs new file mode 100644 index 0000000..7526f3a --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Reflection; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class EmbeddedTemplateLoader +{ + private const string ResourcePrefix = "ConnectedWorkbooks.Templates."; + private const string SimpleQueryTemplateResource = ResourcePrefix + "SIMPLE_QUERY_WORKBOOK_TEMPLATE.xlsx"; + private const string BlankTableTemplateResource = ResourcePrefix + "SIMPLE_BLANK_TABLE_TEMPLATE.xlsx"; + + public static byte[] LoadSimpleQueryTemplate() => LoadTemplate(SimpleQueryTemplateResource); + + public static byte[] LoadBlankTableTemplate() => LoadTemplate(BlankTableTemplateResource); + + private static byte[] LoadTemplate(string resourceName) + { + var assembly = Assembly.GetExecutingAssembly(); + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Unable to locate embedded template '{resourceName}'."); + using var memory = new MemoryStream(); + stream.CopyTo(memory); + return memory.ToArray(); + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs b/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs new file mode 100644 index 0000000..4c5fce8 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.IO.Compression; +using System.Text; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal sealed class ExcelArchive : IDisposable +{ + private readonly MemoryStream _stream; + private ZipArchive? _zipArchive; + private bool _disposed; + + private ExcelArchive(byte[] template) + { + _stream = new MemoryStream(); + _stream.Write(template, 0, template.Length); + _stream.Position = 0; + _zipArchive = new ZipArchive(_stream, ZipArchiveMode.Update, leaveOpen: true); + } + + public static ExcelArchive Load(byte[] template) => new(template); + + public byte[] ToArray() + { + _zipArchive?.Dispose(); + _zipArchive = null; + return _stream.ToArray(); + } + + public string ReadText(string path) + { + var entry = GetEntry(path); + using var stream = entry.Open(); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false); + return reader.ReadToEnd(); + } + + public byte[] ReadBytes(string path) + { + var entry = GetEntry(path); + using var stream = entry.Open(); + using var memory = new MemoryStream(); + stream.CopyTo(memory); + return memory.ToArray(); + } + + public void WriteText(string path, string content, Encoding? encoding = null) + { + var entry = GetOrCreateEntry(path); + using var stream = entry.Open(); + stream.SetLength(0); + using var writer = new StreamWriter(stream, encoding ?? new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), leaveOpen: true); + writer.Write(content); + writer.Flush(); + } + + public void WriteBytes(string path, byte[] content) + { + var entry = GetOrCreateEntry(path); + using var stream = entry.Open(); + stream.SetLength(0); + stream.Write(content, 0, content.Length); + } + + public IEnumerable EnumerateEntries(string folderPrefix) + { + EnsureNotDisposed(); + + foreach (var entry in _zipArchive!.Entries) + { + if (entry.FullName.StartsWith(folderPrefix, StringComparison.OrdinalIgnoreCase)) + { + yield return entry.FullName; + } + } + } + + public bool EntryExists(string path) + { + EnsureNotDisposed(); + return _zipArchive!.GetEntry(path) is not null; + } + + public void Remove(string path) + { + EnsureNotDisposed(); + _zipArchive!.GetEntry(path)?.Delete(); + } + + private ZipArchiveEntry GetEntry(string path) + { + EnsureNotDisposed(); + return _zipArchive?.GetEntry(path) ?? throw new InvalidOperationException($"'{path}' was not found inside the workbook template."); + } + + private ZipArchiveEntry GetOrCreateEntry(string path) + { + EnsureNotDisposed(); + return _zipArchive?.GetEntry(path) ?? _zipArchive!.CreateEntry(path, CompressionLevel.Optimal); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _zipArchive?.Dispose(); + _zipArchive = null; + _stream.Dispose(); + } + + private void EnsureNotDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(ExcelArchive)); + } + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs b/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs new file mode 100644 index 0000000..666a560 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.ConnectedWorkbooks.Models; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class GridParser +{ + public static TableData Parse(Grid grid) + { + grid ??= new Grid(Array.Empty>()); + var data = grid.Data?.Select(row => row.Select(value => value?.ToString() ?? string.Empty).ToArray()).ToList() + ?? new List { Array.Empty() }; + + var promoteHeaders = grid.Config?.PromoteHeaders ?? false; + var adjustColumns = grid.Config?.AdjustColumnNames ?? true; + + CorrectGrid(data, ref promoteHeaders); + ValidateGrid(data, promoteHeaders, adjustColumns); + + string[] columnNames; + if (promoteHeaders && adjustColumns) + { + columnNames = AdjustColumnNames(data[0]); + data.RemoveAt(0); + } + else if (promoteHeaders) + { + columnNames = data[0]; + data.RemoveAt(0); + } + else + { + columnNames = Enumerable.Range(1, data[0].Length).Select(i => $"Column {i}").ToArray(); + } + + return new TableData(columnNames, data); + } + + private static void CorrectGrid(IList data, ref bool promoteHeaders) + { + if (data.Count == 0) + { + promoteHeaders = false; + data.Add(new[] { string.Empty }); + return; + } + + if (data[0].Length == 0) + { + data[0] = new[] { string.Empty }; + } + + var width = data[0].Length; + for (var i = 0; i < data.Count; i++) + { + if (data[i].Length == 0) + { + data[i] = Enumerable.Repeat(string.Empty, width).ToArray(); + } + } + + if (promoteHeaders && data.Count == 1) + { + data.Add(Enumerable.Repeat(string.Empty, width).ToArray()); + } + } + + private static void ValidateGrid(IReadOnlyList data, bool promoteHeaders, bool adjustColumns) + { + if (data.Count == 0 || data[0].Length == 0) + { + throw new InvalidOperationException("The provided grid is empty."); + } + + if (data.Any(row => row.Length != data[0].Length)) + { + throw new InvalidOperationException("The provided grid is not a rectangular MxN matrix."); + } + + if (promoteHeaders && !adjustColumns) + { + if (data[0].Any(string.IsNullOrWhiteSpace)) + { + throw new InvalidOperationException("Headers cannot be promoted when empty values exist."); + } + + var uniqueCount = data[0].Select(name => name.ToLowerInvariant()).Distinct().Count(); + if (uniqueCount != data[0].Length) + { + throw new InvalidOperationException("Headers must be unique when column adjustments are disabled."); + } + } + } + + private static string[] AdjustColumnNames(string[] columnNames) + { + var unique = new HashSet(StringComparer.OrdinalIgnoreCase); + var result = new string[columnNames.Length]; + for (var i = 0; i < columnNames.Length; i++) + { + var baseName = string.IsNullOrWhiteSpace(columnNames[i]) ? $"Column {i + 1}" : columnNames[i]; + var candidate = baseName; + var suffix = 1; + while (!unique.Add(candidate)) + { + candidate = $"{baseName} ({suffix++})"; + } + + result[i] = candidate; + } + + return result; + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs b/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs new file mode 100644 index 0000000..cd90360 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class MashupDocumentParser +{ + public static string ReplaceSingleQuery(string base64, string queryName, string queryMashupDocument) + { + var buffer = Convert.FromBase64String(base64); + var reader = new ArrayReader(buffer); + var versionBytes = reader.ReadBytes(4); + var packageSize = reader.ReadInt32(); + var packageOpc = reader.ReadBytes(packageSize); + var permissionsSize = reader.ReadInt32(); + var permissions = reader.ReadBytes(permissionsSize); + var metadataSize = reader.ReadInt32(); + var metadataBytes = reader.ReadBytes(metadataSize); + var endBuffer = reader.ReadToEnd(); + + var newPackage = EditSingleQueryPackage(packageOpc, queryMashupDocument); + var newMetadata = EditSingleQueryMetadata(metadataBytes, queryName); + + var finalBytes = ByteHelpers.Concat( + versionBytes, + ByteHelpers.GetInt32Bytes(newPackage.Length), + newPackage, + ByteHelpers.GetInt32Bytes(permissionsSize), + permissions, + ByteHelpers.GetInt32Bytes(newMetadata.Length), + newMetadata, + endBuffer); + + return Convert.ToBase64String(finalBytes); + } + + private static byte[] EditSingleQueryPackage(byte[] packageOpc, string queryMashupDocument) + { + using var packageStream = new MemoryStream(); + packageStream.Write(packageOpc, 0, packageOpc.Length); + packageStream.Position = 0; + using var zip = new ZipArchive(packageStream, ZipArchiveMode.Update, leaveOpen: true); + var entry = zip.GetEntry(WorkbookConstants.Section1mPath) + ?? throw new InvalidOperationException("Formula section was not found in the Power Query package."); + + using (var entryStream = entry.Open()) + using (var writer = new StreamWriter(entryStream, new UTF8Encoding(false), leaveOpen: true)) + { + entryStream.SetLength(0); + writer.Write(queryMashupDocument); + writer.Flush(); + } + + zip.Dispose(); + return packageStream.ToArray(); + } + + private static byte[] EditSingleQueryMetadata(byte[] metadataBytes, string queryName) + { + var reader = new ArrayReader(metadataBytes); + var metadataVersion = reader.ReadBytes(4); + var metadataXmlSize = reader.ReadInt32(); + var metadataXmlBytes = reader.ReadBytes(metadataXmlSize); + var endBuffer = reader.ReadToEnd(); + + var metadataXmlString = Encoding.UTF8.GetString(metadataXmlBytes).TrimStart('\uFEFF'); + XDocument metadataDoc; + try + { + metadataDoc = XDocument.Parse(metadataXmlString, LoadOptions.PreserveWhitespace); + } + catch (Exception ex) + { + var preview = Convert.ToHexString(metadataXmlBytes.AsSpan(0, Math.Min(metadataXmlBytes.Length, 64))); + throw new InvalidOperationException($"Failed to parse metadata XML. Hex preview: {preview}", ex); + } + UpdateItemPaths(metadataDoc, queryName); + UpdateEntries(metadataDoc); + + var newMetadataXml = Encoding.UTF8.GetBytes(metadataDoc.ToString(SaveOptions.DisableFormatting)); + return ByteHelpers.Concat( + metadataVersion, + ByteHelpers.GetInt32Bytes(newMetadataXml.Length), + newMetadataXml, + endBuffer); + } + + private static void UpdateItemPaths(XDocument doc, string queryName) + { + if (doc.Root is null) + { + return; + } + + foreach (var itemPathElement in doc.Descendants().Where(e => e.Name.LocalName == XmlNames.Elements.ItemPath)) + { + var content = itemPathElement.Value; + if (!content.Contains("Section1/", StringComparison.Ordinal)) + { + continue; + } + + var parts = content.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2) + { + continue; + } + + parts[1] = Uri.EscapeDataString(queryName); + itemPathElement.Value = string.Join('/', parts); + } + } + + private static void UpdateEntries(XDocument doc) + { + var now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture); + var lastUpdatedValue = ($"d{now}").Replace("Z", "0000Z", StringComparison.Ordinal); + + foreach (var entry in doc.Descendants().Where(e => e.Name.LocalName == XmlNames.Elements.Entry)) + { + var typeValue = entry.Attribute(XmlNames.Attributes.Type)?.Value; + if (string.Equals(typeValue, "ResultType", StringComparison.Ordinal)) + { + entry.SetAttributeValue(XmlNames.Attributes.Value, "sTable"); + } + else if (string.Equals(typeValue, "FillLastUpdated", StringComparison.Ordinal)) + { + entry.SetAttributeValue(XmlNames.Attributes.Value, lastUpdatedValue); + } + } + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/PowerQueryGenerator.cs b/dotnet/src/ConnectedWorkbooks/Internal/PowerQueryGenerator.cs new file mode 100644 index 0000000..50d7742 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/PowerQueryGenerator.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class PowerQueryGenerator +{ + public static string GenerateSingleQueryMashup(string queryName, string queryBody) + { + return $"section Section1;\n\nshared \"{queryName}\" = \n{queryBody};"; + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs b/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs new file mode 100644 index 0000000..f7d3efa --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Globalization; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class PqUtilities +{ + public static (string Path, string Base64) GetDataMashup(ExcelArchive archive) + { + foreach (var entryPath in archive.EnumerateEntries(WorkbookConstants.CustomXmlFolder)) + { + var match = WorkbookConstants.CustomXmlItemRegex.Match(entryPath); + if (!match.Success) + { + continue; + } + + var bytes = archive.ReadBytes(entryPath); + var xml = XmlEncodingHelper.DecodeToString(bytes).TrimStart('\uFEFF'); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + if (!string.Equals(doc.Root?.Name.NamespaceName, WorkbookConstants.DataMashupNamespace, StringComparison.Ordinal)) + { + continue; + } + + var base64 = doc.Root?.Value ?? throw new InvalidOperationException("DataMashup element was empty."); + return (entryPath, base64); + } + + throw new InvalidOperationException("DataMashup XML was not found in the workbook template."); + } + + public static void SetDataMashup(ExcelArchive archive, string path, string base64) + { + var xml = $"{base64}"; + var encoded = Encoding.Unicode.GetBytes("\uFEFF" + xml); + archive.WriteBytes(path, encoded); + } + + public static void ValidateQueryName(string queryName) + { + if (string.IsNullOrWhiteSpace(queryName)) + { + throw new ArgumentException("Query name cannot be empty.", nameof(queryName)); + } + + if (queryName.Length > WorkbookConstants.MaxQueryLength) + { + throw new ArgumentException($"Query names are limited to {WorkbookConstants.MaxQueryLength} characters.", nameof(queryName)); + } + + if (queryName.Any(ch => ch == '"' || ch == '.' || char.IsControl(ch))) + { + throw new ArgumentException("Query names cannot contain periods, quotes, or control characters.", nameof(queryName)); + } + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/WorkbookConstants.cs b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookConstants.cs new file mode 100644 index 0000000..40d0112 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookConstants.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Text.RegularExpressions; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class WorkbookConstants +{ + public const string ConnectionsXmlPath = "xl/connections.xml"; + public const string SharedStringsXmlPath = "xl/sharedStrings.xml"; + public const string DefaultSheetPath = "xl/worksheets/sheet1.xml"; + public const string DefaultTablePath = "xl/tables/table1.xml"; + public const string QueryTablesFolder = "xl/queryTables/"; + public const string QueryTablePath = "xl/queryTables/queryTable1.xml"; + public const string WorkbookXmlPath = "xl/workbook.xml"; + public const string WorkbookRelsPath = "xl/_rels/workbook.xml.rels"; + public const string PivotCachesFolder = "xl/pivotCache/"; + public const string Section1mPath = "Formulas/Section1.m"; + public const string DocPropsCoreXmlPath = "docProps/core.xml"; + public const string DocPropsAppXmlPath = "docProps/app.xml"; + public const string ContentTypesPath = "[Content_Types].xml"; + public const string RootRelsPath = "_rels/.rels"; + public const string DocMetadataPath = "docMetadata"; + public const string CustomXmlFolder = "customXml"; + public const string LabelInfoPath = "docMetadata/LabelInfo.xml"; + public const string TablesFolder = "xl/tables/"; + + public const string ConnectedWorkbookNamespace = "http://schemas.microsoft.com/ConnectedWorkbook"; + public const string DataMashupNamespace = "http://schemas.microsoft.com/DataMashup"; + + public const int MaxQueryLength = 80; + public const int MaxCellCharacters = 32767; + + public static readonly Regex CustomXmlItemRegex = new(@"^customXml/item(\d+)\.xml$", RegexOptions.Compiled | RegexOptions.IgnoreCase); +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs new file mode 100644 index 0000000..597b644 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs @@ -0,0 +1,329 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Globalization; +using System.Linq; +using System.Xml.Linq; +using Microsoft.ConnectedWorkbooks.Models; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal sealed class WorkbookEditor +{ + private readonly ExcelArchive _archive; + private readonly DocumentProperties? _documentProperties; + + public WorkbookEditor(ExcelArchive archive, DocumentProperties? documentProperties) + { + _archive = archive; + _documentProperties = documentProperties; + } + + public void UpdatePowerQueryDocument(string queryName, string mashupDocument) + { + var (path, base64) = PqUtilities.GetDataMashup(_archive); + var nextBase64 = MashupDocumentParser.ReplaceSingleQuery(base64, queryName, mashupDocument); + PqUtilities.SetDataMashup(_archive, path, nextBase64); + } + + public string UpdateConnections(string queryName, bool refreshOnOpen) + { + var xml = _archive.ReadText(WorkbookConstants.ConnectionsXmlPath); + var doc = XDocument.Parse(xml); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var connection = doc.Root?.Element(ns + "connection") ?? throw new InvalidOperationException("Connections XML does not contain a connection element."); + var dbPr = connection.Element(ns + "dbPr") ?? throw new InvalidOperationException("Connections XML is missing the dbPr element."); + + connection.SetAttributeValue("name", $"Query - {queryName}"); + connection.SetAttributeValue("description", $"Connection to the '{queryName}' query in the workbook."); + dbPr.SetAttributeValue("connection", $"Provider=Microsoft.Mashup.OleDb.1;Data Source=$Workbook$;Location=\"{queryName}\";"); + dbPr.SetAttributeValue("command", $"SELECT * FROM [{queryName.Replace("]", "]]", StringComparison.Ordinal)}]"); + dbPr.SetAttributeValue("refreshOnLoad", refreshOnOpen ? "1" : "0"); + + _archive.WriteText(WorkbookConstants.ConnectionsXmlPath, doc.ToString(SaveOptions.DisableFormatting)); + return connection.Attribute("id")?.Value ?? "1"; + } + + public int UpdateSharedStrings(string queryName) + { + var xml = _archive.ReadText(WorkbookConstants.SharedStringsXmlPath); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var textElements = doc.Descendants(ns + "t").ToList(); + var sharedStringIndex = textElements.Count; + var existing = textElements.Select((element, index) => (element, index)).FirstOrDefault(tuple => tuple.element.Value == queryName); + if (existing.element is not null) + { + sharedStringIndex = existing.index + 1; + } + else + { + var si = new XElement(ns + "si", new XElement(ns + "t", queryName)); + doc.Root!.Add(si); + IncrementAttribute(doc.Root, "count"); + IncrementAttribute(doc.Root, "uniqueCount"); + } + + _archive.WriteText(WorkbookConstants.SharedStringsXmlPath, doc.ToString(SaveOptions.DisableFormatting)); + return sharedStringIndex; + } + + public void UpdateWorksheet(int sharedStringIndex) + { + var xml = _archive.ReadText(WorkbookConstants.DefaultSheetPath); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var cellValue = doc.Descendants(ns + XmlNames.Elements.CellValue).FirstOrDefault(); + if (cellValue is null) + { + throw new InvalidOperationException("Worksheet XML did not contain a cell value node."); + } + + cellValue.Value = sharedStringIndex.ToString(CultureInfo.InvariantCulture); + _archive.WriteText(WorkbookConstants.DefaultSheetPath, doc.ToString(SaveOptions.DisableFormatting)); + } + + public void UpdateQueryTable(string connectionId, bool refreshOnOpen) + { + var xml = _archive.ReadText(WorkbookConstants.QueryTablePath); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + doc.Root?.SetAttributeValue("connectionId", connectionId); + doc.Root?.SetAttributeValue("refreshOnLoad", refreshOnOpen ? "1" : "0"); + _archive.WriteText(WorkbookConstants.QueryTablePath, doc.ToString(SaveOptions.DisableFormatting)); + } + + public void UpdateTableData(TableData tableData) + { + if (tableData.ColumnNames.Count == 0) + { + return; + } + + UpdateSheetData(tableData); + UpdateTableDefinition(tableData); + UpdateWorkbookDefinedName(tableData); + UpdateQueryTableColumns(tableData); + } + + public void UpdateDocumentProperties() + { + var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); + var xml = _archive.ReadText(WorkbookConstants.DocPropsCoreXmlPath); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + var cp = (XNamespace)"http://schemas.openxmlformats.org/package/2006/metadata/core-properties"; + var dc = (XNamespace)"http://purl.org/dc/elements/1.1/"; + var dcterms = (XNamespace)"http://purl.org/dc/terms/"; + var xsi = (XNamespace)"http://www.w3.org/2001/XMLSchema-instance"; + + SetElement(doc, cp + "coreProperties", dcterms + "created", now, xsi); + SetElement(doc, cp + "coreProperties", dcterms + "modified", now, xsi); + if (_documentProperties is not null) + { + SetElement(doc, cp + "coreProperties", dc + "title", _documentProperties.Title); + SetElement(doc, cp + "coreProperties", dc + "subject", _documentProperties.Subject); + SetElement(doc, cp + "coreProperties", dc + "creator", _documentProperties.CreatedBy); + SetElement(doc, cp + "coreProperties", dc + "description", _documentProperties.Description); + SetElement(doc, cp + "coreProperties", (XNamespace)"http://schemas.openxmlformats.org/package/2006/metadata/core-properties" + "keywords", _documentProperties.Keywords); + SetElement(doc, cp + "coreProperties", cp + "lastModifiedBy", _documentProperties.LastModifiedBy); + SetElement(doc, cp + "coreProperties", cp + "category", _documentProperties.Category); + SetElement(doc, cp + "coreProperties", cp + "revision", _documentProperties.Revision); + } + + _archive.WriteText(WorkbookConstants.DocPropsCoreXmlPath, doc.ToString(SaveOptions.DisableFormatting)); + } + + private void UpdateSheetData(TableData tableData) + { + var xml = _archive.ReadText(WorkbookConstants.DefaultSheetPath); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var x14ac = doc.Root?.GetNamespaceOfPrefix("x14ac") ?? XNamespace.None; + var sheetData = doc.Descendants(ns + XmlNames.Elements.SheetData).FirstOrDefault(); + if (sheetData is null) + { + throw new InvalidOperationException("Worksheet XML is missing sheetData."); + } + + sheetData.RemoveNodes(); + var startCell = "A1"; + var (startRow, startColumn) = CellReferenceHelper.GetStartPosition(startCell); + var spans = $"{startColumn}:{startColumn + tableData.ColumnNames.Count - 1}"; + + var headerRow = new XElement(ns + XmlNames.Elements.Row, + new XAttribute(XmlNames.Attributes.Row, startRow), + new XAttribute(XmlNames.Attributes.Spans, spans), + x14ac == XNamespace.None ? null : new XAttribute(x14ac + "dyDescent", "0.3")); + + for (var columnIndex = 0; columnIndex < tableData.ColumnNames.Count; columnIndex++) + { + headerRow.Add(CreateCell(ns, startColumn + columnIndex, startRow, tableData.ColumnNames[columnIndex], isHeader: true)); + } + + sheetData.Add(headerRow); + + for (var rowIndex = 0; rowIndex < tableData.Rows.Count; rowIndex++) + { + var excelRow = startRow + rowIndex + 1; + var row = new XElement(ns + XmlNames.Elements.Row, + new XAttribute(XmlNames.Attributes.Row, excelRow), + new XAttribute(XmlNames.Attributes.Spans, spans), + x14ac == XNamespace.None ? null : new XAttribute(x14ac + "dyDescent", "0.3")); + + var rowValues = tableData.Rows[rowIndex]; + for (var columnIndex = 0; columnIndex < tableData.ColumnNames.Count; columnIndex++) + { + var value = columnIndex < rowValues.Count ? rowValues[columnIndex] : string.Empty; + row.Add(CreateCell(ns, startColumn + columnIndex, excelRow, value, isHeader: false)); + } + + sheetData.Add(row); + } + + var endReference = CellReferenceHelper.BuildReference((startRow, startColumn), tableData.ColumnNames.Count, tableData.Rows.Count + 1); + doc.Descendants(ns + XmlNames.Elements.Dimension).FirstOrDefault()?.SetAttributeValue(XmlNames.Attributes.Reference, endReference); + doc.Descendants(ns + XmlNames.Elements.Selection).FirstOrDefault()?.SetAttributeValue(XmlNames.Attributes.SqRef, endReference); + + _archive.WriteText(WorkbookConstants.DefaultSheetPath, doc.ToString(SaveOptions.DisableFormatting)); + } + + private void UpdateTableDefinition(TableData tableData) + { + var xml = _archive.ReadText(WorkbookConstants.DefaultTablePath); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var tableColumns = doc.Descendants(ns + XmlNames.Elements.TableColumns).FirstOrDefault(); + if (tableColumns is null) + { + throw new InvalidOperationException("Table definition is missing tableColumns."); + } + + tableColumns.RemoveNodes(); + tableColumns.SetAttributeValue(XmlNames.Attributes.Count, tableData.ColumnNames.Count); + for (var index = 0; index < tableData.ColumnNames.Count; index++) + { + var column = new XElement(ns + XmlNames.Elements.TableColumn); + column.SetAttributeValue(XmlNames.Attributes.Id, index + 1); + column.SetAttributeValue(XmlNames.Attributes.Name, tableData.ColumnNames[index]); + tableColumns.Add(column); + } + + var reference = $"A1:{CellReferenceHelper.ColumnNumberToName(tableData.ColumnNames.Count - 1)}{tableData.Rows.Count + 1}"; + doc.Root?.SetAttributeValue(XmlNames.Attributes.Reference, reference); + doc.Descendants(ns + XmlNames.Elements.AutoFilter).FirstOrDefault()?.SetAttributeValue(XmlNames.Attributes.Reference, reference); + + _archive.WriteText(WorkbookConstants.DefaultTablePath, doc.ToString(SaveOptions.DisableFormatting)); + } + + private void UpdateWorkbookDefinedName(TableData tableData) + { + var xml = _archive.ReadText(WorkbookConstants.WorkbookXmlPath); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var definedName = doc.Descendants(ns + XmlNames.Elements.DefinedName).FirstOrDefault(); + if (definedName is null) + { + _archive.WriteText(WorkbookConstants.WorkbookXmlPath, doc.ToString(SaveOptions.DisableFormatting)); + return; + } + + var reference = $"!$A$1:${CellReferenceHelper.ColumnNumberToName(tableData.ColumnNames.Count - 1)}${tableData.Rows.Count + 1}"; + definedName.Value = reference; + _archive.WriteText(WorkbookConstants.WorkbookXmlPath, doc.ToString(SaveOptions.DisableFormatting)); + } + + private void UpdateQueryTableColumns(TableData tableData) + { + var xml = _archive.ReadText(WorkbookConstants.QueryTablePath); + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var fields = doc.Descendants(ns + XmlNames.Elements.QueryTableFields).FirstOrDefault(); + if (fields is null) + { + throw new InvalidOperationException("Query table definition is missing queryTableFields."); + } + + fields.RemoveNodes(); + for (var index = 0; index < tableData.ColumnNames.Count; index++) + { + var field = new XElement(ns + XmlNames.Elements.QueryTableField); + field.SetAttributeValue(XmlNames.Attributes.Id, index + 1); + field.SetAttributeValue(XmlNames.Attributes.Name, tableData.ColumnNames[index]); + field.SetAttributeValue(XmlNames.Attributes.TableColumnId, index + 1); + fields.Add(field); + } + + fields.SetAttributeValue(XmlNames.Attributes.Count, tableData.ColumnNames.Count); + doc.Descendants(ns + XmlNames.Elements.QueryTableRefresh).FirstOrDefault()?.SetAttributeValue(XmlNames.Attributes.NextId, tableData.ColumnNames.Count + 1); + _archive.WriteText(WorkbookConstants.QueryTablePath, doc.ToString(SaveOptions.DisableFormatting)); + } + + private XElement CreateCell(XNamespace ns, int column, int row, string value, bool isHeader) + { + var reference = $"{CellReferenceHelper.ColumnNumberToName(column - 1)}{row}"; + var cell = new XElement(ns + XmlNames.Elements.Cell, + new XAttribute("r", reference)); + + cell.SetAttributeValue("t", isHeader ? "str" : DetermineValueType(value)); + if (value.StartsWith(" ", StringComparison.Ordinal) || value.EndsWith(" ", StringComparison.Ordinal)) + { + cell.SetAttributeValue(XNamespace.Xml + "space", "preserve"); + } + + var cellValue = new XElement(ns + XmlNames.Elements.CellValue, value); + cell.Add(cellValue); + return cell; + } + + private static string DetermineValueType(string value) + { + if (double.TryParse(value, NumberStyles.Any, CultureInfo.InvariantCulture, out _)) + { + return "n"; + } + + if (bool.TryParse(value, out _)) + { + return "b"; + } + + return "str"; + } + + private static void IncrementAttribute(XElement element, string attributeName) + { + if (element.Attribute(attributeName) is XAttribute attr && int.TryParse(attr.Value, out var parsed)) + { + attr.Value = (parsed + 1).ToString(CultureInfo.InvariantCulture); + } + } + + private static void SetElement(XDocument doc, XName parentName, XName elementName, string? value, XNamespace? xsi = null) + { + if (value is null) + { + return; + } + + var parent = doc.Descendants(parentName).FirstOrDefault(); + if (parent is null) + { + return; + } + + var element = parent.Element(elementName); + if (element is null) + { + element = new XElement(elementName); + parent.Add(element); + } + + if (xsi is not null && elementName.NamespaceName.Contains("dcterms", StringComparison.Ordinal)) + { + element.SetAttributeValue(xsi + "type", "dcterms:W3CDTF"); + } + + element.Value = value; + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/XmlEncodingHelper.cs b/dotnet/src/ConnectedWorkbooks/Internal/XmlEncodingHelper.cs new file mode 100644 index 0000000..470e291 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/XmlEncodingHelper.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Text; +using System.Linq; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class XmlEncodingHelper +{ + public static string DecodeToString(byte[] xmlBytes) + { + if (xmlBytes.Length >= 3 && xmlBytes[0] == 0xEF && xmlBytes[1] == 0xBB && xmlBytes[2] == 0xBF) + { + return Encoding.UTF8.GetString(xmlBytes, 3, xmlBytes.Length - 3); + } + + if (xmlBytes.Length >= 2 && xmlBytes[0] == 0xFF && xmlBytes[1] == 0xFE) + { + return Encoding.Unicode.GetString(xmlBytes, 2, xmlBytes.Length - 2); + } + + if (xmlBytes.Length >= 2 && xmlBytes[0] == 0xFE && xmlBytes[1] == 0xFF) + { + return Encoding.BigEndianUnicode.GetString(xmlBytes, 2, xmlBytes.Length - 2); + } + + return Encoding.UTF8.GetString(xmlBytes); + } + + public static byte[] EncodeWithBom(string content, Encoding encoding) + { + if (encoding == Encoding.Unicode) + { + return Encoding.Unicode.GetPreamble().Concat(encoding.GetBytes(content)).ToArray(); + } + + if (encoding == Encoding.BigEndianUnicode) + { + return Encoding.BigEndianUnicode.GetPreamble().Concat(encoding.GetBytes(content)).ToArray(); + } + + if (encoding == Encoding.UTF8) + { + return Encoding.UTF8.GetPreamble().Concat(encoding.GetBytes(content)).ToArray(); + } + + return encoding.GetBytes(content); + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/XmlNames.cs b/dotnet/src/ConnectedWorkbooks/Internal/XmlNames.cs new file mode 100644 index 0000000..f8ab742 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/XmlNames.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal static class XmlNames +{ + internal static class Elements + { + public const string SharedStringTable = "sst"; + public const string SharedStringItem = "si"; + public const string Text = "t"; + public const string CellValue = "v"; + public const string DatabaseProperties = "dbPr"; + public const string QueryTable = "queryTable"; + public const string CacheSource = "cacheSource"; + public const string Table = "table"; + public const string TableColumns = "tableColumns"; + public const string TableColumn = "tableColumn"; + public const string AutoFilter = "autoFilter"; + public const string SheetData = "sheetData"; + public const string Row = "row"; + public const string Cell = "c"; + public const string DefinedName = "definedName"; + public const string QueryTableFields = "queryTableFields"; + public const string QueryTableField = "queryTableField"; + public const string QueryTableRefresh = "queryTableRefresh"; + public const string Relationships = "Relationships"; + public const string Relationship = "Relationship"; + public const string Item = "Item"; + public const string ItemPath = "ItemPath"; + public const string Entry = "Entry"; + public const string Items = "Items"; + public const string Worksheet = "worksheet"; + public const string Dimension = "dimension"; + public const string Selection = "selection"; + } + + internal static class Attributes + { + public const string Count = "count"; + public const string UniqueCount = "uniqueCount"; + public const string RefreshOnLoad = "refreshOnLoad"; + public const string ConnectionId = "connectionId"; + public const string Name = "name"; + public const string Description = "description"; + public const string Connection = "connection"; + public const string Command = "command"; + public const string Id = "id"; + public const string RelId = "r:id"; + public const string Target = "Target"; + public const string PartName = "PartName"; + public const string ContentType = "ContentType"; + public const string Reference = "ref"; + public const string SqRef = "sqref"; + public const string TableColumnId = "tableColumnId"; + public const string UniqueName = "uniqueName"; + public const string QueryTableFieldId = "queryTableFieldId"; + public const string NextId = "nextId"; + public const string Row = "r"; + public const string Spans = "spans"; + public const string X14acDyDescent = "x14ac:dyDescent"; + public const string Type = "Type"; + public const string Value = "Value"; + public const string ResultType = "ResultType"; + } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Models/DocumentProperties.cs b/dotnet/src/ConnectedWorkbooks/Models/DocumentProperties.cs new file mode 100644 index 0000000..ffcb9e6 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Models/DocumentProperties.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Models; + +/// +/// Optional document metadata used to stamp core properties in the generated workbook. +/// +public sealed record DocumentProperties +{ + public string? Title { get; init; } + public string? Subject { get; init; } + public string? Keywords { get; init; } + public string? CreatedBy { get; init; } + public string? Description { get; init; } + public string? LastModifiedBy { get; init; } + public string? Category { get; init; } + public string? Revision { get; init; } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Models/FileConfiguration.cs b/dotnet/src/ConnectedWorkbooks/Models/FileConfiguration.cs new file mode 100644 index 0000000..0a9a405 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Models/FileConfiguration.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Models; + +/// +/// Optional knobs used when generating a workbook from the .NET implementation. +/// +public sealed record FileConfiguration +{ + /// + /// When provided, the workbook will be generated using the supplied template bytes instead of the built-in one. + /// + public byte[]? TemplateBytes { get; init; } + + /// + /// Document metadata that should be applied to docProps/core.xml. + /// + public DocumentProperties? DocumentProperties { get; init; } + + /// + /// Fine grained instructions that help the generator locate the right sheet/table inside a custom template. + /// + public TemplateSettings? TemplateSettings { get; init; } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Models/Grid.cs b/dotnet/src/ConnectedWorkbooks/Models/Grid.cs new file mode 100644 index 0000000..9660a77 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Models/Grid.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Models; + +/// +/// Simple 2D grid abstraction used for data ingestion. +/// +public sealed record Grid(IReadOnlyList> Data, GridConfig? Config = null); + diff --git a/dotnet/src/ConnectedWorkbooks/Models/GridConfig.cs b/dotnet/src/ConnectedWorkbooks/Models/GridConfig.cs new file mode 100644 index 0000000..6ace473 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Models/GridConfig.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Models; + +/// +/// Controls how incoming grid data should be interpreted when converted into an Excel table. +/// +public sealed record GridConfig +{ + /// + /// Treat the first row of as the header row. + /// + public bool PromoteHeaders { get; init; } = true; + + /// + /// Automatically fix duplicate/blank headers by appending numeric suffixes. + /// + public bool AdjustColumnNames { get; init; } = true; +} + diff --git a/dotnet/src/ConnectedWorkbooks/Models/QueryInfo.cs b/dotnet/src/ConnectedWorkbooks/Models/QueryInfo.cs new file mode 100644 index 0000000..151b41b --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Models/QueryInfo.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Models; + +/// +/// Describes a single Power Query definition that should be injected into the generated workbook. +/// +public sealed record QueryInfo +{ + public QueryInfo(string queryMashup, string? queryName = null, bool refreshOnOpen = true) + { + QueryMashup = string.IsNullOrWhiteSpace(queryMashup) + ? throw new ArgumentException("Query mashup cannot be null or empty.", nameof(queryMashup)) + : queryMashup; + + QueryName = string.IsNullOrWhiteSpace(queryName) ? null : queryName; + RefreshOnOpen = refreshOnOpen; + } + + public string QueryMashup { get; } + + public string? QueryName { get; } + + public bool RefreshOnOpen { get; } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Models/TableData.cs b/dotnet/src/ConnectedWorkbooks/Models/TableData.cs new file mode 100644 index 0000000..9bb59fd --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Models/TableData.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Models; + +/// +/// Normalized representation of tabular data that can be written into the workbook. +/// +public sealed record TableData(IReadOnlyList ColumnNames, IReadOnlyList> Rows) +{ + public static TableData Empty { get; } = new(Array.Empty(), Array.Empty>()); +} + diff --git a/dotnet/src/ConnectedWorkbooks/Models/TemplateSettings.cs b/dotnet/src/ConnectedWorkbooks/Models/TemplateSettings.cs new file mode 100644 index 0000000..aa9a32b --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Models/TemplateSettings.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Microsoft.ConnectedWorkbooks.Models; + +/// +/// Optional overrides used when supplying a custom workbook template. +/// +public sealed record TemplateSettings +{ + public string? TableName { get; init; } + public string? SheetName { get; init; } +} + diff --git a/dotnet/src/ConnectedWorkbooks/Templates/SIMPLE_BLANK_TABLE_TEMPLATE.xlsx b/dotnet/src/ConnectedWorkbooks/Templates/SIMPLE_BLANK_TABLE_TEMPLATE.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..ec392d78da30e8df3900c74e143df6714695154a GIT binary patch literal 10234 zcmeHt1y`KQ()9#)cMT8-?iyT!1PPYlIyemO!2$#bZb5=;@Zjz)!$1fG2=49@^qu6K zdvg!xeBU3q_gRbX#hR(Trn`4l?XG%M6ky@-00;mi002M*$cO7cjd}zC1i%9TxBw&= z9Z8U#Gtkc2K;6S0=%mN$Zfo;68xDp(695DK{{No;;vFbapH*66$9iKB>Kk$shEpe$ ziElLsONfl@8e=GGNTN(q9BJ#354pQl(iKB;nJwFV|2)zhERa0A>iD&386J0f(VC}B zrbui7&g8aP(>EP7lu`5;0n`p3RijPwX=rO8YT)DLzoH-B(;YnUaL2W z*A}ALT9a~TX`RRVhZ#$-Pf$4?6xkG0l=x1yycHFU1Dko9Qgz(}7log_-i~;dvG5_< z=o|~DoP7NAmvr$94!3BCkVEVMf1}^%w`)}lt^vRG4zi|QVrRm((>+*DB16apyQJI3 zti2eICnH~k0S`t<$*@@ZOFlzyw5?h*!rneS>H)Cq<-Loq0g9wEF+38kj!^i^^N~7I z^mh?v5Syh*kGD$qP)EGKhXtto#g942Lr18gemoET6cy^n297`*CpOlfayhMp1hoG<0~c5E0l)xneKo8 z*bp~hvr^)omxOk$8Ck52avWB3^oA*e&@IQBK~<)Ftg=AUj$vT>^v32`ionTIo(VUW zo>ipAF~O<+k*t+8hFXum_a|pE@^!~>VK{Z`UF4L;)_}`7NhQQ|<1!uMjVp<;9^Uv5 z_si&!oL0j~VKfmE%%&WvobO*7?lRP=R(Uc{S(X=~)C8`GjbMs_3;RQxbQxUR^~@*i zhVz3PiIq?Wjpg%^&XKisW65KM$m|c?Y8nnM@y0GMu-_QYReCzCp2VSFhGlI_9Ahb&eV9i3M-IgztNZIn zIoa8ft@u4VDI1$8N>Q#8cmDh8Ql`Dou*f4i$Pw{4&&+DG;fQ3Wua;S_`qVSIlmPYS zEdRR~g1298Mn%DVxKH*RaVBbipI4Q&q+{DRc=ahEeXTr>`gMTM z#`tlgu!Ut`8+?x-_TrlO^({SRwfKkbhkZyS)*_#!z3Aw5nIVLiC@ceSD!0o*nQQdb z{VA?544&EF>mTCV?&Hn~%g@iZjnN9U!ii?Rh-Gq9Jap@`ytvCb5L3L&xhl(pEZ$N# z1fqS9e%g;{+Vag{3Ki+{bmzT!5Ca&;H1JEPEq_cJQH-q-ER~SMG6Zb)@)ErnD?P($ ze8@4wB#3wugMMlG*EKlOSf;xpYsAq}}zq?Az1FBOUaEXKF-;0 zy%#K6$h7ml$)r!D#dc(ny+g{#g4cVV3ro9429pjNklE*!;8k(v{78(EP8E{3L1Y&@ zy$|=)VcV?SG~s#&b29ytz;3?Oxjoz01hU>%tE$^wBgdB1ZyQ^4c_)OSgTp=-_r&vl z7x=e|qV^GT&*=gwW24^dNd-1oP_W0rNT2Hk55_Vq?pX&@ zVHggaC$7zbAUtT~WCRw;c(0Tq;5j@l&hGY3@g6EhiW6>wNNhiIIv)S}B|RU~Vfgy9=k46NG$FQEPsvde!eLP;Iv~pt`Q^aSDUk+I@rM>^VY4fCuH+2{E#q z5V{9oY;%H&>9Rxm9O=0E#!S%zY~7J_g|&>$3y9o1_uBTLBH{5NO!UT0-c@MeoY)d zWBbQAMbI)kR`3DSo^ORWW+bfThY*#|5Gya8z2{!M;K0}2u9L1P7B%;8`m!MMx+=7N zDKB<6(n1YJ_s<(?*%q5C^K-r9Rbgaib zqO$;i3xHcum5%`!2vR~%aD!unBx9jrW5L54>o2z721SJs+LTIw=r`T=u zn2?#6ywK^$s4%p#D|{9T!5J3X`ftB<+FrB9Xu5^GF4hd{0^~^zEjyT$;O-Dy&!2=w zn;zJkUWO}kO5RDxF>RI9xe!q$lUuji_zvd3=;zw1yf0?=?j&n2boT=l?wZgZ-3$-T z8?In>+JP%Bn{>yDi*kMYEPcG%KCTXMk5-w+vV!8u84PH>6IeZ3U$bi7+)<{DG6p>* zYof;3j%DQso=h92KZO^(XQdM_@s&X31fbZ+O2dxe~}k*(iISL;;Zl9L;e1R zYjA6nG7RQh|29EYah5CIn~Ek-CaOr{D)w1JtqVGeZ2$w2EC z%kg{CX#)j@(&^XmB3uWhQw&YI4yO_@OHCbLd+&oZC$XK&B}vJU95U;Mhj81pVP8Kd zo_>%kvX88pUfOr%EDhO|oTlrW(#N9Xh$~ewbS$BKMl(#UnMv|2$8s%Z0!L`IVWS-3 zp4oy_FUr#ieZBE&jaWZh0cDuC85*|!&g65$YVmqR005i%=e_+GlbtPqwm`OD<1Z5L zYL7&b*5S7y_TP&Q>rD=CdKn2o3i!_VJ(m*nR%PYlgZ07oVyaC{c#lvvCKUW(EXNf5 z=RN7e5G?!<8djq?3~3baLxdbELd?aXeUw+Fypnk4btQt-ShZ-!n#a zOT32BoIQMvJUtf9N5>x91isE#D-G2f+0#T5{Y{%z1c#zf{36kWh&JxLO!dV`%)3u_9|i_%ig>aE(`8k z*CZ2#%-!*wV9@TOm(RC@3FhWU zz{_=vR*~(R&X^#+%#g8FZ^Aj7n%NPumLO(+LFZ1vk^BXc^+Oo=v5E**;hun}Rt6Kfldv-{oI`;B{#qaH%G^m|nDF<>sIM&frWr9@x@<=tK1 z{q33CX50PUET+NTK*R^&O|kFY-PgK>+j!5{RzVE8Tg(Zv-w`$Q5rm}zJNF7c)Ug6-^FHm3u+`f(4n(-|pRZOc?Xx00|dinX(P82!1qBxBa@< zoM5U-wgk^*_=Y`mB0C7pna69p3g>fm&>GO65=*~%kw%vc`GhYU1fh!PV6!F^ zx32-Ft`roGChvTDx7sPwS=l};?jw-|?7eo+v)@$}=-^-?y+JDOmp-;(F#~>=Wjs8M zWsj6`qRvxTkMY}J5xAua?p}Ak=FsWmRi*dbANL3pV550=tOKYG>5{T(UNypF58VcZ z%)g5GHpoy{L1?>#Kp$S>`80oECoBV(8LsWd!zu$)^Q<1fO`)ZVt(;81*ztVHC#Yrw zt2|Z>W_G}Cxsk=zPJ#5`zO{O~E)j&9G8?d7o9WOJ@FqGSztAYNnL%4P{|NVB@0yw3 z#N2A6KZd)M61kk`gaa)lJ>K$#-L;=3+=nX=MM{@VL`G-Q^4nI;>Ed7GulT@{C{OcO7pVu^)gFtFfF$QC z^ini;g8dR(kVD5!c|%j)N?CJVlHfEto%_6YCS#t_zU|cz%m-NMl(&2F1=O9DO%HVYmhR-IMP*72JW60V zG=&3oc{nFt?~CHw^PRaA*ng5%PBpF~K_MwUe|K1;L_OsQt1y@xK)Z=E>5lXsEXnoJ zwFUk(Vu@3U%01WTqSFTX9UB*ICH_6{Ct7?}^v~g(?^8?Semu<&L+ZBpes+2S>hh!jR|XB0~@YA z!Fx8b*-@R8bcUdM1c}BMTP(R-L*s zzd_N+zL+UW7TFh3F-`_3+rJ=5&~M`{?xnqkAH$n)>jAs0Qw(^-pSd+2I%xtr{-$dTF^W8Z@gF&r5f1lSjtJJ7f8gR zu^m>d5!@z<-vLLnQ&?082?M(KF}Dti`pTEiBA?fr4O984@OD{mq!3fp>T=2L1=%G9 zq-m4Nku#5ked}e{*7B7_!FjioBs++|k8M_YLy09fa*~fM&qBI=8H>8_N=_hC`Ejl^ zp4XYd&x$m{B6P;W#&{w{G8TMI7qyx(tzTK^Twp41;UNLDdO;4|>yDfDWhPseAp6vl4$ps# z$L%~^2t!~265S(N%v@|dW7qpGn*AzxKbxPJ?6F>_ke-!V$C(Jv+Tx&EhZUj}xv^;7(=>Fm>iKU=Fh+ddJs>sI~~pl}W1Yfl_X52LA%3ip|wv zd1c_kQb%^d!raSD;Z>1{z9ko&AaMs`nGTB=PdiM#li{Rsdfp#d_@v?u_<0?JwXkF9 zw+kYlW(<>kcwz4X+0KMOP^mkP*`%Z2;;f!VMYlnj`*-nLtEb_e2IX=c^bhY(PFol` z0!`JN9WCw5e^Hw|LL8-&9ZUKmzzwOY&QdQ#HiZ=>T?`Xz<}><}xRm6O`T2VLA9ou^ z!-tXLE$mgw6OvQanbG;&UfQ1Z3uW)P=n0-zp@vqx!{kJBZQ~lwE>ig3#aO9Vy>8jF zSuaPb`i6qx`BVIpY>%N$Gkt7Ygf}*`agku$LI(WU8F-C;QPfD^+Ta!*<32q3T(HGw znR`0Du5;9XT2A4|Ge9$R8MIKPPV>idewHBqyjA{DllXnN{8f<%jOzHgI;n$zd#rQ2 zaZjw6{FV?%@g`!)RxNt1T))g%8{eX|_#yW$#tV2;TwdC!-L z=8hZY!bVktwQWM^(*AyyWg%8YpP>tvhE_`dJWFRIV`zM0`!)S>omN$_n`OuQPPj>q zv~T%RRGWYqF;VZ)qjA=hQ$V=XwA3ZGEaT^wr~6q?rj!;HW61@&hntC8*G9xq4u`|n zwDH+L8X+YWxpaLk^Yf)(j2AZ~*FeYD{tQEr9K=_Au#gxjf`TBVGYzw!wsZWr;w)Xx z%DmBJu&sJVcKhb*F3c>k3~7`3upG7&QJk1+*Yrzq)Q^Fn&qC?Kg`t z1)zAT*BZj8+a;~Dw|v?ni&ic$AeJ7iqMAsQ@Ln+#C)QR?wM;EmV`C!D9jQf&eSdyn zZ`I)-GGMMqvyH_P&bKo7VtmTz&3+i^4&6GWJ!Ec-$--iQc%Gwewsp&2AO=j|idN|f z?E?S(h^tS+fM?LdmxLZOF?2QsnJ5CCjZBT4jo4lo83S$P?94zv(~)}gkX0AdXMKAt z*oIWoUh~$%&f$-1w24K%WQNeZ{sK|GI!I`;a{GYl#P7B~^c-yj_I%k%8wn&hF2*^1 z<*d^$_>56bO@+2tLa8EJnSQkrOt;AnpGT$prXycMH969eigfvnaY+(-HlyvS*(j}` zjbbFT;08MTSZJiQZyQD6D={EG<1yi^ds8&q6lz|&r}!Eq&5|uQcg9gk+}eYrPwi>$ z50UT$3>8d%OX|f$YwzoXdE2iSNNk9|8|EQwET! zma6(HqpzaKjeW>YQ>LF)-Oo-@{-Zl&`8kk_K=(-&8u+jQ7>}TPrQ!&(cVaUEIRby~ z(c^5C|J5R)TldEkA2Xr^WlZ3KY_sI5mvMnVO34#xm@_yW9O=h)z4Nu2E0oj0)u5=j zYAPgFvc2RVLOU#kUSFSNEH-mDN_fbqz@(5;o#ji676Uzla43@qN;pUuodHnY6TMoXA4oegBVP{GhQ)k5K3PB4#35*~ z9Lm#sl_*Idg|m<+ITUxS`hC7Mu`oSGD<;+>B~HUf`n|RMab4sY&B7TGtNQmX?Yk&< z4-TqyfJ>q6OWUNpq^{}Cl7z=h;29I3#Z3Yve3vFxkU3ZqA{wdRWXCRvqFLP}zGr=n`dCZ;{NBUZiKucYMW$etOT zMvwKVgsx@d!NZYS6P_oyf1A_u!H17Ut~ufmqmA@imND>FO(nxMb@YRO4p$jZ z7icA4>*AZMf-e)V3aSK*Z3-`aDi#wyz7N}~H8iNq$ze_SSZ8G&@s0;^r`H{hb^&3V zVz{C@!hyJE+w$w%yd+d-Uw7MA*_XCh2p{E7=>%~@J|oOdgx4s`YKivTgtGv$Vr`Ow zO$08$))&%j&tVBSdZ`RyjF!k8Hu_=jta$0Z2zCz%9QCswDe=$xgPu8o`Sg>Oubz65 zHa&xkg+Jhxc0|x8gh8CwRHkz&74NRTp0u$G-L0H< zogI6=^kMy>XU;{ta94TvZx!_jh6O4b{`p;`e}#yD&HwObl8VA#0si_D#J>!G&izoI z_>Wg29vVJu_5L;`h1OUfG=3i%|FsYG+Z5V4M)_s@|L96R#Ch19_>HuW_U|9@w^qeN zl!t}k-zfFaCN^{_56i?40UlmBe*4K*{st5$`(ryEiq(e*57p0ageMgLA1(9{=C8uwHwXZr0#y?Kj@AE? n2@lQx8r=WWe2D5l%>NAaDhlw>a>37n0Xje%O8u}WKmYqbVM(;| literal 0 HcmV?d00001 diff --git a/dotnet/src/ConnectedWorkbooks/Templates/SIMPLE_QUERY_WORKBOOK_TEMPLATE.xlsx b/dotnet/src/ConnectedWorkbooks/Templates/SIMPLE_QUERY_WORKBOOK_TEMPLATE.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..d9e7aa8f42e5e008a44c768ad26420496050abcd GIT binary patch literal 17284 zcmeHubyyrpzb+o!LvVMOAi>?;-QC?KxVr^+*Wj+fogl#-f(3WGlkA?GU3SlT?*03G z%`?+I(^D^9Rd4;gWFoq{fskhdfdGF0pW}b=2$Uy|STE5d3f+L;BY-pp3Xz59l2e<+*75ps zgUo=}6b!`*eX6qWFL}BlDft2gJu9wjB`8LG!i62j6kgWOJj*3Gv&wvwn^_B;ektB6 z)$3isJ;7ZKEyhL?;WAkbir=c<;Z>`O1EQN~%wef2g34_lRRc;@eGRO6VCGflH*K#x9+w1C&+G73i8-ys3H)Tg0mm>qnORWPd2DlU=s*zBjrD!Ock>c*4e+1#2Bn+0KnfcT(PRGr zZ^#Y8b{3aK5lxSe11o8@79RD%h0yO_jA3Pb2sqOs+NxEx2-0?VY(}IiBkc5V@jD)#=6|Ofr=WY>6g?522^`JKS^OoE~V}sC#(^ltLy0 z@9aM-WJ9#MQB;m}0{Hv#0tzJiHzB})6FNo+5CW@!H4Xz10y_3amJW2ZKi~he`2QEL z_`h_$JZ?%HlmSs_4g7^*yPac6Nict@BScmfu>_A^%D6E{jFebzsgtz=O{-XFa=`G% za(Z+8kmXjnYr!Yj2j!?Tb?Ec3`ZF)8tamPX78LTrRZ}%Z%GMM^3zv_U=YnVsX5v&R zapbhz_4bJlO~9h&LhuTGzMdtH*f_iP5nN!37DrI2EggQhD+1DQGW9DpF!t{G!}?ef zl3zCAq8QCbA;X9w`Kb*V(ip>4b&n_-%chxNq^A+Mp-HREvNII(R{TY_|7}TukVco5fC2%H0@N%5U>sd7>0E5= zE%a?{EPm?La-}8fEfz#Cx=A0|P%kCo17op}wrXk&xELD0a#lAe#7Sfvq*%e^T>YZ% zXOG|(w(A!1WlOlU_cp;M#^bPbJSU~{9yz&sZBd9F^s&N+5qss{VUvwJ{5P?4qVjwB z+su)ZR92IF`&FHq*3GWkcF*O&yeopmG4?2!!trnG2*<$JCNY)gG!+GVWH~X$gk|Gc z!3%0A;YQwbppL&KezwPRqsbwx!7%95>@_d(e zN9LM^M?T88CCFwZW`?OuVi%B7x`U0Eru=njO-3Z;>Y2efhLSh-9Y~nPS=Xu~e#|%i z-ZIBJj$1(8V;?=K(Kvs*@bf>f*SI$s0SgQO zUGYW8CQ%br9)pVwoC{;*zc}5{0JU~29t?XHIt&i5IgArH;KDXdN+%ukk);_*7=rk= zd~xvdV#+g#vQa#WMe+XDCs9+tu_NwX$XjwkJo!6lKi}g4b`%0{&x?%ET0lzvXe;m_`75hI)m1l-NxT)9y9vWK6pCL zzD!0~tR;4ruCFk(SgnHR1ug!_oc*vSEIJ^*TQ$&Y)#y55uO>Kn%>~nN>CmZgtq5cQ z6IF7Fn52)E(?|?o%@GZ{A`{2!0%B1(kq!mnvp&-+%fZmVA>0pKP+133Upaq3i_=kR zAD$Cbru_lqAPI67XOnV33vs|8+eZ1TrWhVCyfCO@%i)X`sbHMQM`}0WVR)SlkSD*L z4@y;#sn_$**JjsH!IP@V{lK^(JGZo5LyDVx^ZWQ+g>T8N9m%S&7Ot|bAfmdXKBRr} z%}XmeK8V6oZ8`at#O;Di)QS=y$K93ngPH4-+iw z;Rb}aML3V#9%pfqB7-$dV4v!H!fF{bLFCrkTL**`8P2|j!TdF}8~LMqTH=?J>U`S?2P*2H%Gc9WP7Sf9bkvzV0p$*N`$vj2FQ_MGt@rK)(Q|Fpz_wv+q@09F_kI1mu}Z?)lIYGmZ-K=!rVX1Gh)hZwwX#yeL{nF2NO9d(&?1E?zh&2 zTZckcSD9RVpfwpC!mP6e{mK`vLupWddTTyE>Vzh^U^5LTvB_qjUYwK(+1wteS1K;{ z39ET_@*d?2KN08LiTP|ZlhlZYt@^jIhVc9;975$%x#w>J6jO{o+Kv5TO8#DWu9Ucb~TjHF4y!sBJOPKLUy4haxd-1^QQDagx}hxkKOhPq4-^j+6@V)-CMX=U9k80{JNSjDh=*#-r6>mPAp1={C#w17^6jfIqdw7Ikx;A^13*5|BJ$<2csS zO~ThK*h9G&fXH}x0ZjeNOAW=is1%Kz7`0ekEyXaM3VDqZqoO=Ht>`EfL{0g}BWQv6 zFNS3a;>k0c+cRy*8YG}x1|%G5hB5LY*_^WZ=!0~%Ox06#wY5_+5c=6-wp$UR^zg-K z*69SPcK49UKOOo%|C3~d26PWEtWYCz4bT!HVM4=;H{=TcK|YfJ9z{yXk_#%tQ|q2> z(+XeaNj~=vDCNwGfqO@x$?pOSqyzKXh@=eh6|%ba<#_BwqlC)YnCy9GJFzp!oZHQN zRS+-KrdV3h?hbIfE%!SZ+PM#kHHkd`-sg-D5QeLSG$z!!+{Ndgodwj&gpD6pmj*p)* z1O}NE9#Rg981dcnC-I8gTO!wMNk-?*Nvj1y-qu)UjJ=!8tTm74yRnN|7hKXAjy_Gq zq=u*A#I74#c4L&d@x|x%sp^?aE&@4rzJ!9NW4h0$Yt7f!8Y)wn5q5rDF*1F)<`16} zF52zYBJ|KhsVt)uZmou0{fIfw)!cffFuqHfHe-Ta+53qecu194X{TWQP(og(8iu>GN|9@Yx0_X6i{wr26-`L6 z5!`_Hou||D*c{_{hhOnW4D)u-Jzeq$I4PMD7g-ig6&YMFdAZBnJ@dz5tL1ElVuS+= zFU*hq2C7cR;T5%V$AKT_fQ)o-S{@?3EA%(up6Y4RIAZTrUEzi=gEek9s^4hWonj_z zFbwWacGFeH7P~Na|(MAVj{GJ zacSjUBt=1;`&g!eTtjCi-@kWh#{JtbV5@|(g!~|I3;RbL;iF5sib|h~^w)Wn&eq{0Qm}j3SHB zfXIzcoI|}&VZby|bN(&wclel(jcG%O5Kd|0H#GAV-EOct7ya>pX?FQWoa5Ubg8d>4 zsRC)mxmdYOv1OT1HsfH9(U;ZM?VE5bEpO^Y;TMPxKPpp}E~PXp;HyQvnR3TwS7sEL z#}ZPpsnEU=GigC-W#k>|2t(>OY(QcTMv9!p7u>Fjn51e&q*8~cN)>;iR4gN})B-N7 z$OEZb!)>c;d?FuZADR5*FYlGRnZTT)GNshrJSlo&Ps+>$G<>WCff{Q~q2WR+Q$Z_H z?;yCy*dxli1W5#E;7T!G`_-~f$FN*ONR;Lxfl9eGhdyu06XmmVKutH?d*&3s!d11U zs(HeuK#V-Mr7Zp%^bS&oL7be`F2sy(<}uZC%id1-+R^41FXGM7jZ{ZQ7SWI|%X!3R z59W7=mf0n{h*lZ}&;h4a%$4AE=Qd36e%5a=WQts=$EvZ2hQBTl(R=p49oMP9J@ii` z*5GZ5-W*DCk$m?^>xUYS`F5uZ3duH9cBt(GfvrFTX$|2E3w&X^^B5InhQI<_?gjb| zS(w-^hGzWtMeL`#Yu~ekDh%wFW+vCYvsRiTVjYep}J;5I!a zyAX9{qnV1eDxq$tinsH8x$H8b$>}Y7Y>2R5!-lB8CiE@6FK6SxtCwo+^6NSk^a@^^ zGYR;LQsr^(zid2L3aYPUaPVbb!zThw zlSw1*p{jvK+WYv~$gxBQiD5~d?d1B6``k`>t~H@fk&hp_i&cUv)XOyJ>diCsD2`mx z3Y4o>X|4|2g?+xA2#w91#-3!D+kPv#>bcKWD^+oCmsdrYkx#9VCn{Iw(98RZ9grd7 zCI+fKSWJ8a)1WYCWN+}eIue>kGs8e|(sXv=M=*(DC^5QiR1EW@CWv){_adzg?;c!OK zYF@yrVN5+wIWQ#@f~1gG*{r^MduV_W!B7)bnhIT^8s##&S+@DqI@*$GN~muD;Br>ovm3W2uR|sO+I@ z$IA5R!T26Ea%t1KJ`$)$KN+jeur)PLZ;arJhDD35;)4(S z$;l%VBqqFErst&jlxbafetnZjVf6A?um$EBMt7WNk@mAlP)uJcvm?>+)*1ak(G!{S zMPx*@vd*n?D?B0Kb9l1pCJ|ne#EoERmGFT z4v)}73sc~{*}7ze8YT|g+c$MiOBnj-aXt(T2pt}XB$8!G;v`GjIu+06@C(qOLVf83 zo}Wn%KNrZ_ z`4X)VQRH`i9()Lhi&GG(Vq=nhtE-ePo+lUUOTAQAyJAB6#&6P`z=N8GthQVI4|nPo zeUadngSeSKd=gz>Plln&HMUm6b}a41l}nA1k8#2c+1p8yfMz#n&Vvo)r^1OX#62us z)Q67ke(z@JB8B-|sR9hdExh#l2o-xq1TjMFvh(?dZ-Un3haahnTjX_i4&ME1=MNzT z6vqJ_p8!ZM7r1RQ=I}rkzYXNmaWs+5rt=$M>LhE@uH|9#3d`|7F9D< z@||oR&X`)XI(?}q%Iyj-3AzatlH}L1BA@gO?Z`CzA(Nk#*arQm^WKGUnJuo*lWUI# z=XEoNZ1K^dM~H0cau2e$zG^KOXit*JI(2;%qS{jb<7@8*V`LgGCBwRnavz_{l5{_NMSNNPN7F zk7BxHRb)1<9C*f}S;I{+4E4tlW_7wL5pC z-rRQQI-<3UKakn(|8x%jNqxsRWZ6Gw|05vm{^xRb)YAv(EV^H(KjS7V)@k%WfhT0g zIMiz>_>TeEF=4T)VRcfAsazjnNI&o58K<$BF1F4G)tM!vIlJ6uU+kwFsIFwNfkq3O zQ3?U~ssRBb)ms*`p`Om;x?zAV>Q3pBf>W&XzHig;d9Gu;^+nQM;eU5(Dw+JyQ5hXz zz!i^!bS*{ocAy`3wtGHbLhvm|j(L5*? z9JuOYg->)x{_SagmzWUUX?%;XgF01-k%ckZ?npxcbMp{-e+OaojFbbP(94jxjvC85 ztE0K3lJixr=M_V-x^meA@H3XP&G)OWBSy%sVi*NV@LW>|tdfMXz=vAgXj&m0SZr9e>R1*Fv&lFidqR)r4-GSz*ma#&~nFM_s0}voC z=f9fBiOo6W@X>haX?~@8n&x&d!GlmsCrL?8tbEqbblW)}nG;wNN*e1m_eZR`^5C9!F6ND<~1i)G(7>x*2L`Y|yo*-V0DG zYOZ+N(7tZo_1iU3Yn41xks}*D0rovZccnEjH6FkW|8xG~tyub_0rNitSpB$vtbRKu zBYQW+KS=QV`WF)-;xuHo=n(_gVy}5ry=7J|`6XuS4gw*y5zAH2n(Q;g^egDpJml;7 zA6<@;X3+xQta6$h*+h-Ic5s>sU9Y#*k>;6I$v~m&!I+gsJiCv_Q;^^`Q#92;FuH@u zB!}avTy)YjZ85D;-~;P<3R`?J?*H_T+4P4;1277Efm5mhxmKU&*2i}p5WE%Nve!C| zQzZl@0#eY{d`*8m*GUnUTeA&kC?ZFbp^>PM{-83@a!j1^m#BKl*=h!zmuEY)3T+26 zXrM0m7&YM7gMDN)O123*5A5Z^Ya3KXBl65qAgF4a)F`iZ2HJpXV$caBduFY)bEN7O(c29Vat$dHT zs)DK`#=)Z6;U?C=acN3LTv@pH9i6tJ?-kBqX-7T{t@AUA_?r$=2zc}-`0qxn$=T3& z@@8=TZ0uaf@lXsIUp>QgC=}6e8a|iK;xW|pr9Zuxm>iJp_x6)no&9U}tbn@?Y!{%j z?FoNIg2;gkoE#i&tW>Qm>C7CBtbWZOel9T(c{V;^p8x0LC{;~?Py%h>B5%2=s^77; ziDBwyHD(Nr4{Tvu3Qz@TarE(-i~>dud&y3ll$7#gY58caja!`0^H%V~$k@sx z+_H`J0sD_tspa}sC!(9+hZUg})^{5>tg8{4X=f6vQOk+#UXBQrhtK`xTDy-s&yGBr zi(2&V8_=Glm3z1=pJs7;Ly-868Mb`SI$KXPQP!_N^3=UuyKOh##??gO)-v{H8Wq^> z0Dp2h#pTp7Yn-6wBGVemXvRn|)SMHRzxL|83FR=D=w+t z1?d>ixrcaE5zY6NY#A*FX$D=T64lF$Paynh5@iKSi}4p*-{tDq|<6j>C0MJuMiwgpK(Z=g4Q^_wTzs{jSg3 zl3ha6P?Ky;tF3dGv$y7t#vi6YHE%YGAPceZ(XuztsXhoaNYz!3v__nfg{pMFW!1hI zrNZ9;OF)fXgw_XCy=Zoo1RD73%sUIQ2tB|qs(`@6?reT7pcZ&jiXJ1_G;hL9mcYcG53vP?NGie z={iYy!-c!O`(Dzo)y-)Ok7s5a{mB99NuVyX@s@arDt8Kv{{DNt6IJGY+wiAMv_%nj3I#=BnNxr&LZ1olU>+ncu7XT)Qw|;>6kTbftU0m zOtjmK56n#?7aX9u(Hu40c)&!wuKesuY=``NE~jg}8i;48Wsiw^Pryv@U9$nfEOkQ7 z>S#A;x*_~GKv@x>HkN00OT~CS$Mb!i2HD3SDG(B;zksxXZxdzg@g``n|CkMi03RmFcaTrc;aA? zQQuyf+ca&5(mMFgp<{FnFZOWY=+>*b@GQ`u#K&kmg~e?}jYY;yi*sklPi2(L$TJPH@?OYRsR|YyX5;4bDZ_*v zTZ?-G-cAOH@)SXThVHwfb(Pi5?dG{a%o^We(M6W>F67aoK%Ye~O4 zzRM7+iG4C&70(DQ^61$!@(j#s6+fHOX1JxW^Mc6pD*Re0Puoh+t>VOkdz1dzvE_a; zH}_3MCDmoJPCw6`RtRfd`0VYEt^9+uNuM?zp>kYIRk()$5}C*~j81}8C-%9lvOx>; zYbp+fM8X%f=$E&30q#nx@t!gvJf$LU(1N`luiUDiv6IsT3CC>II^^0!k&N2fDJm&!c0y!D zM!ZT6Vkj}$KrGfz99HYQ;+ffoWuA;Wys%{YH>XWb)73q3dFH?7;8|{#Mb1|&7VLsM zf8U?;K7;aM@6A$v5_i)$)4pc%CA%{+-cRPrC*??hy?ma(y|KI7dfZ#*yR!ItR1wVr zp{$dTqjsyrbo*4{j#ui?!T$qvWTMnYaZRgZvh#=-MtaBK3KBJ#xGHd{B&bhEof~Rv zu(NhPY+(YF-a>P+>YW166e}$50R>kEPFX=b_g|Iz#j3 z*h<_(YIr#>Z89Ytc_*n;4mM}3zDF7?H(Sn~!&z%QcU~E5tYy&*?gOjdD-+6R6m%m| z!>VEDVViZ2KiVI2S~Q+X>nt+VAFG_;xy`XVaoU7H*GvH` z->dmtdgpjWUI!k`L0qX~;mIzSsBtPRD|X7<$v==d$=0vUT&Cy(@1#`4UxDVngdlf`zi$&gywD3N^ z-DOx)&5w1N;N_Tv4?ET%sLo&wB zOGo&s;rcMn1>1Eot85~hJNf`2vg{<+dnlx+X5sUuS}QeUX=FclO8T!`kqR7f@4;wR?Bo$_`=E_Y%_wv9(;~ph^wk?s+;L@I`U6I z@1K*ULHxW75Z|r-AQQ}dFeJcTh`xuaX|-Le>Uea|sy26OI+AzFsjBsu>s4weeQuaC zo0PlQi##xTTD?ygiNVb!rNYMZBSS)Q*(gkA+3>M-YWzW-X=m@meG|>hZ~Whd(%OOK0dvl+X*d*n{m7bTx7%b zg?*ZnfA5~P2hm+{3H*JzGxifj2Y!Hw249sMuxkOkGgIFMC&S^gX7B z*@%Rp-5Ucb`m~B@L{yVf|9kc7&|yQ|f}nW6&aY>~XV^POj*UEIoES$~zF%Pwn4Q*Gt#EjaK?Rj&L&fDZObRPT3H?+h@_Q{KXVs(p;GMnQrddz5BaYwluA3?sdYZhHJLE z$-*K{Qy>!uo+FQU{O>?=OTOSOrJSrE2i&S|`ZQ zOPLIqO(fjEw>yC?wT zgai;zNI-bNfU)`|b?6Li?2Z0dI#~ZH4+N0tzc~V8trmA|55zns$To8DP!Jn1I@wsbf%lw2tTC+dl~BpEiiko%-bRanbR)Eco^NL* zEG$|>B-<|f21G%gfdVQ=js*Sz@^f0m_k2Zr`4+mhNDvYzN8PS|Ue)ozz{qu}+ge@8 zO3}Qn)q)J^E$!As&LEU_7)z~uRrQnPE5>D-UG!_>-HeQGsz(ZrX+k) znM6;oK2oLdvV=Wjur8pJNb7B%aDiisOEz;S+VJmsE2|lB!plHBu6}Q0x^WSOpu_Ue z)unzm%DmlbYF4E{G9XevE(HhEa5y~!DUOR}4ppepH~cL`z5cxg0T~`jT~=F!L~I*I zfVC*8I!cAzmtOWI+wX_$_(D$yTO7(j0pc}!rq?^)I4Y0pGp`|Nr6aYvfO>i;Hv8q_ zZWSvf^Lavw3^p_M)^Tj-aMtJ@M(1@>$esqZZ%NS`O*UHH>we&5_4cn}`Sr5Pq?v1_ zK!SI6@$s08-v6mC2aM8x9RU;FVIT~!5bLe$90|)Atwe zZJ|x;^^S~8eP#-&FkWG0V@a>o+^<=H!#MMDjb zY|o&`^(A9GYTe1@+ORC3KHEkq%N1my56#b*J0gWmq7p|RG0es;PhgB@S97T+I&jjm zaeTJoz0oTbu))W>hnTrb5kG%nf%A8{75afjKa7X8xS~p~+2c8y&@{*rO%Bnf>)M!F zx(w4|aW&wx@~;&Mnn`4YKdTY`Cl>rNSO3F;|Dpik3}6Ak-~_P1Bj-fdQ78SHUOe=x z1jaUtFibn9h?u0oh^n~Rns;Fl8U%+p{~8^0u*auaI3|xU2o@zc=)AnufvK*$%d5|z zefkDACSj)QzOcd0?dIOL4f08@=b=h+gjk;hr?oAdQ6NU%l0+CW%EBv)r-)>fw=N=G zHB|CAc{68wF)7Vs4Qh1)JGtk_J$6dHfI1H<9AKm>!z=3i+(qK#QdY_>q?{72(K%7ej`j` zy&i)<=JQou&uhTf)iJ*T$FP3^zN(dZ-Trlb#qahbfOI=xH2zf9zt!|>gx4ARzaZfM zLinAdccA-y{VP`U8td;V;@?O>KuLr^|CTU*4e<9M(O&^(XUezi}uT{|8t7^~3z#Z~YAd1f +/// Entry point for generating Connected Workbooks from .NET. +/// +public sealed class WorkbookManager +{ + public async Task GenerateSingleQueryWorkbookAsync( + QueryInfo query, + Grid? initialDataGrid = null, + FileConfiguration? fileConfiguration = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + + var queryName = string.IsNullOrWhiteSpace(query.QueryName) + ? "Query1" + : query.QueryName!; + + PqUtilities.ValidateQueryName(queryName); + var templateBytes = fileConfiguration?.TemplateBytes ?? EmbeddedTemplateLoader.LoadSimpleQueryTemplate(); + var tableData = initialDataGrid is null ? null : GridParser.Parse(initialDataGrid); + + using var archive = ExcelArchive.Load(templateBytes); + var editor = new WorkbookEditor(archive, fileConfiguration?.DocumentProperties); + var mashup = PowerQueryGenerator.GenerateSingleQueryMashup(queryName, query.QueryMashup); + editor.UpdatePowerQueryDocument(queryName, mashup); + var connectionId = editor.UpdateConnections(queryName, query.RefreshOnOpen); + var sharedStringIndex = editor.UpdateSharedStrings(queryName); + editor.UpdateWorksheet(sharedStringIndex); + editor.UpdateQueryTable(connectionId, query.RefreshOnOpen); + if (tableData is not null) + { + editor.UpdateTableData(tableData); + } + editor.UpdateDocumentProperties(); + + return await Task.FromResult(archive.ToArray()); + } +} diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/ConnectedWorkbooks.Tests.csproj b/dotnet/tests/ConnectedWorkbooks.Tests/ConnectedWorkbooks.Tests.csproj new file mode 100644 index 0000000..0b01c7b --- /dev/null +++ b/dotnet/tests/ConnectedWorkbooks.Tests/ConnectedWorkbooks.Tests.csproj @@ -0,0 +1,24 @@ +ο»Ώ + + + net8.0 + latest + enable + enable + + + + + + + + + + + + + + + + + diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/MSTestSettings.cs b/dotnet/tests/ConnectedWorkbooks.Tests/MSTestSettings.cs new file mode 100644 index 0000000..b8a247d --- /dev/null +++ b/dotnet/tests/ConnectedWorkbooks.Tests/MSTestSettings.cs @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] + diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs b/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs new file mode 100644 index 0000000..7974366 --- /dev/null +++ b/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.IO.Compression; +using System.Text; +using System.Xml.Linq; +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace ConnectedWorkbooks.Tests; + +[TestClass] +public sealed class WorkbookManagerTests +{ + private readonly WorkbookManager _manager = new(); + + [TestMethod] + public async Task GeneratesWorkbookWithMashupAndTable() + { + var queryBody = @"let + Source = Kusto.Contents(""https://help.kusto.windows.net"", ""Samples"", ""StormEvents"") +in + Source"; + var query = new QueryInfo(queryBody, "DataAgentQuery", refreshOnOpen: true); + var grid = new Grid(new List> + { + new List { "City", "Count" }, + new List { "Seattle", 42 }, + new List { "London", 12 } + }, new GridConfig { PromoteHeaders = true, AdjustColumnNames = true }); + + var bytes = await _manager.GenerateSingleQueryWorkbookAsync(query, grid); + Assert.IsTrue(bytes.Length > 0, "The generated workbook should not be empty."); + + using var archiveStream = new MemoryStream(bytes); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read); + + var connections = ReadEntry(archive, "xl/connections.xml"); + StringAssert.Contains(connections, "DataAgentQuery"); + + var sharedStrings = ReadEntry(archive, "xl/sharedStrings.xml"); + StringAssert.Contains(sharedStrings, "DataAgentQuery"); + + var sheet = ReadEntry(archive, "xl/worksheets/sheet1.xml"); + StringAssert.Contains(sheet, "Seattle"); + StringAssert.Contains(sheet, "London"); + + var table = ReadEntry(archive, "xl/tables/table1.xml"); + StringAssert.Contains(table, "City"); + StringAssert.Contains(table, "Count"); + + var mashupXml = ReadEntry(archive, "customXml/item1.xml"); + StringAssert.Contains(mashupXml, "DataMashup"); + var root = XDocument.Parse(mashupXml).Root ?? throw new AssertFailedException("DataMashup XML root was missing."); + var sectionContent = ExtractSection1m(root.Value.Trim()); + StringAssert.Contains(sectionContent, "DataAgentQuery"); + } + + [TestMethod] + public void RejectsInvalidQueryName() + { + var query = new QueryInfo("let Source = 1 in Source", "Invalid.Name", refreshOnOpen: false); + Assert.ThrowsException(() => _manager.GenerateSingleQueryWorkbookAsync(query).GetAwaiter().GetResult()); + } + + private static string ReadEntry(ZipArchive archive, string path) + { + var entry = archive.GetEntry(path) ?? throw new AssertFailedException($"Entry '{path}' was not found in the workbook."); + using var stream = entry.Open(); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); + return reader.ReadToEnd(); + } + + private static string ExtractSection1m(string dataMashupBase64) + { + var bytes = Convert.FromBase64String(dataMashupBase64); + using var memory = new MemoryStream(bytes); + using var binaryReader = new BinaryReader(memory); + binaryReader.ReadBytes(4); // version header + var packageSize = binaryReader.ReadInt32(); + var packageBytes = binaryReader.ReadBytes(packageSize); + + using var packageStream = new MemoryStream(packageBytes); + using var packageZip = new ZipArchive(packageStream, ZipArchiveMode.Read); + var entry = packageZip.GetEntry("Formulas/Section1.m") ?? throw new AssertFailedException("Section1.m was not found in the mashup package."); + using var entryStream = entry.Open(); + using var reader = new StreamReader(entryStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); + return reader.ReadToEnd(); + } +} + diff --git a/.eslintrc.js b/typescript/.eslintrc.js similarity index 100% rename from .eslintrc.js rename to typescript/.eslintrc.js diff --git a/.prettierrc b/typescript/.prettierrc similarity index 100% rename from .prettierrc rename to typescript/.prettierrc diff --git a/README.md b/typescript/README.md similarity index 100% rename from README.md rename to typescript/README.md diff --git a/babel.config.js b/typescript/babel.config.js similarity index 100% rename from babel.config.js rename to typescript/babel.config.js diff --git a/jest.config.jsdom.js b/typescript/jest.config.jsdom.js similarity index 100% rename from jest.config.jsdom.js rename to typescript/jest.config.jsdom.js diff --git a/jest.config.node.js b/typescript/jest.config.node.js similarity index 100% rename from jest.config.node.js rename to typescript/jest.config.node.js diff --git a/package-lock.json b/typescript/package-lock.json similarity index 99% rename from package-lock.json rename to typescript/package-lock.json index 210b67b..2a3fb07 100644 --- a/package-lock.json +++ b/typescript/package-lock.json @@ -87,6 +87,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.6.tgz", "integrity": "sha512-qAHSfAdVyFmIvl0VHELib8xar7ONuSHrE2hLnsaWkYNTI68dmi1x8GYDhJjMI/e7XWal9QBlZkwbOnkcw7Z8gQ==", "dev": true, + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.24.6", @@ -3016,7 +3017,6 @@ "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 18" } @@ -3046,8 +3046,7 @@ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@octokit/core/node_modules/@octokit/types": { "version": "13.10.0", @@ -3055,7 +3054,6 @@ "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/openapi-types": "^24.2.0" } @@ -3097,7 +3095,6 @@ "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/request": "^8.4.1", "@octokit/types": "^13.0.0", @@ -3112,8 +3109,7 @@ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@octokit/graphql/node_modules/@octokit/types": { "version": "13.10.0", @@ -3121,7 +3117,6 @@ "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/openapi-types": "^24.2.0" } @@ -4044,6 +4039,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-4.33.0.tgz", "integrity": "sha512-ZohdsbXadjGBSK0/r+d87X0SBmKzOq4/S5nzK6SBgJspFo9/CUDJ7hjayuze+JK7CZQLDMroqytp7pOcFKTxZA==", "dev": true, + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "4.33.0", "@typescript-eslint/types": "4.33.0", @@ -4415,6 +4411,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", "dev": true, + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4492,6 +4489,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -5071,6 +5069,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001587", "electron-to-chromium": "^1.4.668", @@ -6153,6 +6152,7 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", "integrity": "sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA==", "dev": true, + "peer": true, "dependencies": { "@babel/code-frame": "7.12.11", "@eslint/eslintrc": "^0.4.3", @@ -12268,6 +12268,7 @@ "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.3.2.tgz", "integrity": "sha512-lnJzDfJ66zkMy58OL5/NY5zp70S7Nz6KqcKkXYzn2tMVrNxvbqaBpg7H3qHaLxCJ5lNMsGuM8+ohS7cZrthdLQ==", "dev": true, + "peer": true, "bin": { "prettier": "bin-prettier.js" }, @@ -14464,6 +14465,7 @@ "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", "dev": true, + "peer": true, "dependencies": { "arg": "^4.1.0", "create-require": "^1.1.0", @@ -14895,6 +14897,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "dev": true, + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -15215,6 +15218,7 @@ "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "^1.0.5", "@webassemblyjs/ast": "^1.12.1", @@ -15329,6 +15333,7 @@ "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-5.1.4.tgz", "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==", "dev": true, + "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.5.0", "@webpack-cli/configtest": "^2.1.1", @@ -15407,6 +15412,7 @@ "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, diff --git a/package.json b/typescript/package.json similarity index 100% rename from package.json rename to typescript/package.json diff --git a/src/generators.ts b/typescript/src/generators.ts similarity index 100% rename from src/generators.ts rename to typescript/src/generators.ts diff --git a/src/index.ts b/typescript/src/index.ts similarity index 100% rename from src/index.ts rename to typescript/src/index.ts diff --git a/src/types.ts b/typescript/src/types.ts similarity index 100% rename from src/types.ts rename to typescript/src/types.ts diff --git a/src/utils/arrayUtils.ts b/typescript/src/utils/arrayUtils.ts similarity index 100% rename from src/utils/arrayUtils.ts rename to typescript/src/utils/arrayUtils.ts diff --git a/src/utils/constants.ts b/typescript/src/utils/constants.ts similarity index 100% rename from src/utils/constants.ts rename to typescript/src/utils/constants.ts diff --git a/src/utils/documentUtils.ts b/typescript/src/utils/documentUtils.ts similarity index 100% rename from src/utils/documentUtils.ts rename to typescript/src/utils/documentUtils.ts diff --git a/src/utils/gridUtils.ts b/typescript/src/utils/gridUtils.ts similarity index 100% rename from src/utils/gridUtils.ts rename to typescript/src/utils/gridUtils.ts diff --git a/src/utils/htmlUtils.ts b/typescript/src/utils/htmlUtils.ts similarity index 100% rename from src/utils/htmlUtils.ts rename to typescript/src/utils/htmlUtils.ts diff --git a/src/utils/index.ts b/typescript/src/utils/index.ts similarity index 100% rename from src/utils/index.ts rename to typescript/src/utils/index.ts diff --git a/src/utils/mashupDocumentParser.ts b/typescript/src/utils/mashupDocumentParser.ts similarity index 100% rename from src/utils/mashupDocumentParser.ts rename to typescript/src/utils/mashupDocumentParser.ts diff --git a/src/utils/pqUtils.ts b/typescript/src/utils/pqUtils.ts similarity index 100% rename from src/utils/pqUtils.ts rename to typescript/src/utils/pqUtils.ts diff --git a/src/utils/tableUtils.ts b/typescript/src/utils/tableUtils.ts similarity index 100% rename from src/utils/tableUtils.ts rename to typescript/src/utils/tableUtils.ts diff --git a/src/utils/xmlInnerPartsUtils.ts b/typescript/src/utils/xmlInnerPartsUtils.ts similarity index 100% rename from src/utils/xmlInnerPartsUtils.ts rename to typescript/src/utils/xmlInnerPartsUtils.ts diff --git a/src/utils/xmlPartsUtils.ts b/typescript/src/utils/xmlPartsUtils.ts similarity index 100% rename from src/utils/xmlPartsUtils.ts rename to typescript/src/utils/xmlPartsUtils.ts diff --git a/src/workbookManager.ts b/typescript/src/workbookManager.ts similarity index 100% rename from src/workbookManager.ts rename to typescript/src/workbookManager.ts diff --git a/src/workbookTemplate.ts b/typescript/src/workbookTemplate.ts similarity index 100% rename from src/workbookTemplate.ts rename to typescript/src/workbookTemplate.ts diff --git a/tests/arrayUtils.test.ts b/typescript/tests/arrayUtils.test.ts similarity index 100% rename from tests/arrayUtils.test.ts rename to typescript/tests/arrayUtils.test.ts diff --git a/tests/documentUtils.test.ts b/typescript/tests/documentUtils.test.ts similarity index 100% rename from tests/documentUtils.test.ts rename to typescript/tests/documentUtils.test.ts diff --git a/tests/gridUtils.test.ts b/typescript/tests/gridUtils.test.ts similarity index 100% rename from tests/gridUtils.test.ts rename to typescript/tests/gridUtils.test.ts diff --git a/tests/htmlUtils.test.ts b/typescript/tests/htmlUtils.test.ts similarity index 100% rename from tests/htmlUtils.test.ts rename to typescript/tests/htmlUtils.test.ts diff --git a/tests/mashupDocumentParser.test.ts b/typescript/tests/mashupDocumentParser.test.ts similarity index 100% rename from tests/mashupDocumentParser.test.ts rename to typescript/tests/mashupDocumentParser.test.ts diff --git a/tests/mocks/PqMock.ts b/typescript/tests/mocks/PqMock.ts similarity index 100% rename from tests/mocks/PqMock.ts rename to typescript/tests/mocks/PqMock.ts diff --git a/tests/mocks/index.ts b/typescript/tests/mocks/index.ts similarity index 100% rename from tests/mocks/index.ts rename to typescript/tests/mocks/index.ts diff --git a/tests/mocks/section1mSimpleQueryMock.ts b/typescript/tests/mocks/section1mSimpleQueryMock.ts similarity index 100% rename from tests/mocks/section1mSimpleQueryMock.ts rename to typescript/tests/mocks/section1mSimpleQueryMock.ts diff --git a/tests/mocks/workbookMocks.ts b/typescript/tests/mocks/workbookMocks.ts similarity index 100% rename from tests/mocks/workbookMocks.ts rename to typescript/tests/mocks/workbookMocks.ts diff --git a/tests/mocks/xmlMocks.ts b/typescript/tests/mocks/xmlMocks.ts similarity index 100% rename from tests/mocks/xmlMocks.ts rename to typescript/tests/mocks/xmlMocks.ts diff --git a/tests/tableUtils.test.ts b/typescript/tests/tableUtils.test.ts similarity index 100% rename from tests/tableUtils.test.ts rename to typescript/tests/tableUtils.test.ts diff --git a/tests/workbookQueryTemplate.test.ts b/typescript/tests/workbookQueryTemplate.test.ts similarity index 100% rename from tests/workbookQueryTemplate.test.ts rename to typescript/tests/workbookQueryTemplate.test.ts diff --git a/tests/workbookTableTemplate.test.ts b/typescript/tests/workbookTableTemplate.test.ts similarity index 100% rename from tests/workbookTableTemplate.test.ts rename to typescript/tests/workbookTableTemplate.test.ts diff --git a/tests/xmlInnerPartsUtils.test.ts b/typescript/tests/xmlInnerPartsUtils.test.ts similarity index 100% rename from tests/xmlInnerPartsUtils.test.ts rename to typescript/tests/xmlInnerPartsUtils.test.ts diff --git a/tsconfig.json b/typescript/tsconfig.json similarity index 100% rename from tsconfig.json rename to typescript/tsconfig.json diff --git a/tsconfig.test.json b/typescript/tsconfig.test.json similarity index 100% rename from tsconfig.test.json rename to typescript/tsconfig.test.json diff --git a/webpack.config.js b/typescript/webpack.config.js similarity index 100% rename from webpack.config.js rename to typescript/webpack.config.js From 7b1d634f0188c25a1d7b8cde870f92cf3d781dd8 Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Wed, 26 Nov 2025 14:13:17 +0100 Subject: [PATCH 2/5] support template metadata --- .../Internal/ArrayReader.cs | 38 +- .../Internal/ByteHelpers.cs | 31 - .../Internal/CellReferenceHelper.cs | 2 - .../Internal/EmbeddedTemplateLoader.cs | 14 +- .../Internal/ExcelArchive.cs | 11 +- .../ConnectedWorkbooks/Internal/GridParser.cs | 6 +- .../Internal/MashupDocumentParser.cs | 85 +- .../Internal/PqUtilities.cs | 5 +- .../Internal/TemplateMetadataResolver.cs | 106 + .../Internal/WorkbookEditor.cs | 37 +- .../Internal/XmlEncodingHelper.cs | 51 - .../ConnectedWorkbooks/InternalsVisibleTo.cs | 3 + .../ConnectedWorkbooks/Models/TableData.cs | 1 - .../src/ConnectedWorkbooks/WorkbookManager.cs | 26 +- .../ConnectedWorkbooks.Tests.csproj | 4 + .../PqUtilitiesTests.cs | 73 + .../coverage.cobertura.xml | 3020 +++++++++++++++++ .../WorkbookManagerTests.cs | 171 + 18 files changed, 3514 insertions(+), 170 deletions(-) delete mode 100644 dotnet/src/ConnectedWorkbooks/Internal/ByteHelpers.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/TemplateMetadataResolver.cs delete mode 100644 dotnet/src/ConnectedWorkbooks/Internal/XmlEncodingHelper.cs create mode 100644 dotnet/src/ConnectedWorkbooks/InternalsVisibleTo.cs create mode 100644 dotnet/tests/ConnectedWorkbooks.Tests/PqUtilitiesTests.cs create mode 100644 dotnet/tests/ConnectedWorkbooks.Tests/TestResults/90a856a6-0001-416c-bed0-a68b7b8eb703/coverage.cobertura.xml diff --git a/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs b/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs index 3a0eb1d..123a311 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs @@ -11,41 +11,45 @@ internal sealed class ArrayReader private int _offset; public ArrayReader(byte[] buffer) + : this(new ReadOnlyMemory(buffer)) { - _buffer = new ReadOnlyMemory(buffer); - _offset = 0; } - public byte[] ReadBytes(int count) + public ArrayReader(ReadOnlyMemory buffer) { - if (_offset + count > _buffer.Length) - { - throw new InvalidOperationException("Attempted to read beyond the length of the buffer."); - } + _buffer = buffer; + _offset = 0; + } - var slice = _buffer.Slice(_offset, count).ToArray(); + public ReadOnlyMemory ReadMemory(int count) + { + EnsureAvailable(count); + var slice = _buffer.Slice(_offset, count); _offset += count; return slice; } public int ReadInt32() { - var span = _buffer.Span; - if (_offset + sizeof(int) > span.Length) - { - throw new InvalidOperationException("Attempted to read beyond the length of the buffer."); - } - - var value = BinaryPrimitives.ReadInt32LittleEndian(span.Slice(_offset, sizeof(int))); + EnsureAvailable(sizeof(int)); + var value = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Span.Slice(_offset, sizeof(int))); _offset += sizeof(int); return value; } - public byte[] ReadToEnd() + public ReadOnlyMemory ReadToEnd() { - var slice = _buffer.Slice(_offset).ToArray(); + var slice = _buffer.Slice(_offset); _offset = _buffer.Length; return slice; } + + private void EnsureAvailable(int count) + { + if (_offset + count > _buffer.Length) + { + throw new InvalidOperationException("Attempted to read beyond the length of the buffer."); + } + } } diff --git a/dotnet/src/ConnectedWorkbooks/Internal/ByteHelpers.cs b/dotnet/src/ConnectedWorkbooks/Internal/ByteHelpers.cs deleted file mode 100644 index 4de3293..0000000 --- a/dotnet/src/ConnectedWorkbooks/Internal/ByteHelpers.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Buffers.Binary; - -namespace Microsoft.ConnectedWorkbooks.Internal; - -internal static class ByteHelpers -{ - public static byte[] Concat(params byte[][] arrays) - { - var total = arrays.Sum(a => a.Length); - var result = new byte[total]; - var offset = 0; - foreach (var array in arrays) - { - Buffer.BlockCopy(array, 0, result, offset, array.Length); - offset += array.Length; - } - - return result; - } - - public static byte[] GetInt32Bytes(int value) - { - var buffer = new byte[4]; - BinaryPrimitives.WriteInt32LittleEndian(buffer, value); - return buffer; - } -} - diff --git a/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs b/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs index 37aa54a..5b11897 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Linq; - namespace Microsoft.ConnectedWorkbooks.Internal; internal static class CellReferenceHelper diff --git a/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs b/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs index 7526f3a..320da39 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs @@ -11,17 +11,19 @@ internal static class EmbeddedTemplateLoader private const string SimpleQueryTemplateResource = ResourcePrefix + "SIMPLE_QUERY_WORKBOOK_TEMPLATE.xlsx"; private const string BlankTableTemplateResource = ResourcePrefix + "SIMPLE_BLANK_TABLE_TEMPLATE.xlsx"; - public static byte[] LoadSimpleQueryTemplate() => LoadTemplate(SimpleQueryTemplateResource); + public static Task LoadSimpleQueryTemplateAsync(CancellationToken cancellationToken = default) => + LoadTemplateAsync(SimpleQueryTemplateResource, cancellationToken); - public static byte[] LoadBlankTableTemplate() => LoadTemplate(BlankTableTemplateResource); + public static Task LoadBlankTableTemplateAsync(CancellationToken cancellationToken = default) => + LoadTemplateAsync(BlankTableTemplateResource, cancellationToken); - private static byte[] LoadTemplate(string resourceName) + private static async Task LoadTemplateAsync(string resourceName, CancellationToken cancellationToken) { var assembly = Assembly.GetExecutingAssembly(); - using var stream = assembly.GetManifestResourceStream(resourceName) + await using var stream = assembly.GetManifestResourceStream(resourceName) ?? throw new InvalidOperationException($"Unable to locate embedded template '{resourceName}'."); - using var memory = new MemoryStream(); - stream.CopyTo(memory); + await using var memory = new MemoryStream(); + await stream.CopyToAsync(memory, cancellationToken); return memory.ToArray(); } } diff --git a/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs b/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs index 4c5fce8..c0d8b04 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs @@ -36,16 +36,7 @@ public string ReadText(string path) using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false); return reader.ReadToEnd(); } - - public byte[] ReadBytes(string path) - { - var entry = GetEntry(path); - using var stream = entry.Open(); - using var memory = new MemoryStream(); - stream.CopyTo(memory); - return memory.ToArray(); - } - + public void WriteText(string path, string content, Encoding? encoding = null) { var entry = GetOrCreateEntry(path); diff --git a/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs b/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs index 666a560..fe8e80f 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Collections.Generic; -using System.Linq; using Microsoft.ConnectedWorkbooks.Models; namespace Microsoft.ConnectedWorkbooks.Internal; @@ -45,13 +43,13 @@ private static void CorrectGrid(IList data, ref bool promoteHeaders) if (data.Count == 0) { promoteHeaders = false; - data.Add(new[] { string.Empty }); + data.Add([string.Empty]); return; } if (data[0].Length == 0) { - data[0] = new[] { string.Empty }; + data[0] = [string.Empty]; } var width = data[0].Length; diff --git a/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs b/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs index cd90360..a35ced1 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System.Buffers.Binary; using System.IO.Compression; -using System.Linq; using System.Text; using System.Xml.Linq; @@ -14,35 +14,50 @@ public static string ReplaceSingleQuery(string base64, string queryName, string { var buffer = Convert.FromBase64String(base64); var reader = new ArrayReader(buffer); - var versionBytes = reader.ReadBytes(4); + var versionBytes = reader.ReadMemory(4); var packageSize = reader.ReadInt32(); - var packageOpc = reader.ReadBytes(packageSize); + var packageOpc = reader.ReadMemory(packageSize); var permissionsSize = reader.ReadInt32(); - var permissions = reader.ReadBytes(permissionsSize); + var permissions = reader.ReadMemory(permissionsSize); var metadataSize = reader.ReadInt32(); - var metadataBytes = reader.ReadBytes(metadataSize); + var metadataBytes = reader.ReadMemory(metadataSize); var endBuffer = reader.ReadToEnd(); - var newPackage = EditSingleQueryPackage(packageOpc, queryMashupDocument); + var newPackage = EditSingleQueryPackage(packageOpc.Span, queryMashupDocument); var newMetadata = EditSingleQueryMetadata(metadataBytes, queryName); - var finalBytes = ByteHelpers.Concat( - versionBytes, - ByteHelpers.GetInt32Bytes(newPackage.Length), - newPackage, - ByteHelpers.GetInt32Bytes(permissionsSize), - permissions, - ByteHelpers.GetInt32Bytes(newMetadata.Length), - newMetadata, - endBuffer); + var totalLength = versionBytes.Length + + sizeof(int) + + newPackage.Length + + sizeof(int) + + permissions.Length + + sizeof(int) + + newMetadata.Length + + endBuffer.Length; + + var finalBytes = new byte[totalLength]; + var destination = finalBytes.AsSpan(); + var offset = 0; + + offset += Copy(versionBytes.Span, destination[offset..]); + BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset, sizeof(int)), newPackage.Length); + offset += sizeof(int); + offset += Copy(newPackage, destination[offset..]); + BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset, sizeof(int)), permissionsSize); + offset += sizeof(int); + offset += Copy(permissions.Span, destination[offset..]); + BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset, sizeof(int)), newMetadata.Length); + offset += sizeof(int); + offset += Copy(newMetadata, destination[offset..]); + _ = Copy(endBuffer.Span, destination[offset..]); return Convert.ToBase64String(finalBytes); } - private static byte[] EditSingleQueryPackage(byte[] packageOpc, string queryMashupDocument) + private static byte[] EditSingleQueryPackage(ReadOnlySpan packageOpc, string queryMashupDocument) { using var packageStream = new MemoryStream(); - packageStream.Write(packageOpc, 0, packageOpc.Length); + packageStream.Write(packageOpc); packageStream.Position = 0; using var zip = new ZipArchive(packageStream, ZipArchiveMode.Update, leaveOpen: true); var entry = zip.GetEntry(WorkbookConstants.Section1mPath) @@ -60,15 +75,15 @@ private static byte[] EditSingleQueryPackage(byte[] packageOpc, string queryMash return packageStream.ToArray(); } - private static byte[] EditSingleQueryMetadata(byte[] metadataBytes, string queryName) + private static byte[] EditSingleQueryMetadata(ReadOnlyMemory metadataBytes, string queryName) { var reader = new ArrayReader(metadataBytes); - var metadataVersion = reader.ReadBytes(4); + var metadataVersion = reader.ReadMemory(4); var metadataXmlSize = reader.ReadInt32(); - var metadataXmlBytes = reader.ReadBytes(metadataXmlSize); + var metadataXmlBytes = reader.ReadMemory(metadataXmlSize); var endBuffer = reader.ReadToEnd(); - var metadataXmlString = Encoding.UTF8.GetString(metadataXmlBytes).TrimStart('\uFEFF'); + var metadataXmlString = Encoding.UTF8.GetString(metadataXmlBytes.Span).TrimStart('\uFEFF'); XDocument metadataDoc; try { @@ -76,18 +91,26 @@ private static byte[] EditSingleQueryMetadata(byte[] metadataBytes, string query } catch (Exception ex) { - var preview = Convert.ToHexString(metadataXmlBytes.AsSpan(0, Math.Min(metadataXmlBytes.Length, 64))); + var preview = Convert.ToHexString(metadataXmlBytes.Span[..Math.Min(metadataXmlBytes.Length, 64)]); throw new InvalidOperationException($"Failed to parse metadata XML. Hex preview: {preview}", ex); } UpdateItemPaths(metadataDoc, queryName); UpdateEntries(metadataDoc); var newMetadataXml = Encoding.UTF8.GetBytes(metadataDoc.ToString(SaveOptions.DisableFormatting)); - return ByteHelpers.Concat( - metadataVersion, - ByteHelpers.GetInt32Bytes(newMetadataXml.Length), - newMetadataXml, - endBuffer); + + var totalLength = metadataVersion.Length + sizeof(int) + newMetadataXml.Length + endBuffer.Length; + var buffer = new byte[totalLength]; + var destination = buffer.AsSpan(); + var offset = 0; + + offset += Copy(metadataVersion.Span, destination[offset..]); + BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset, sizeof(int)), newMetadataXml.Length); + offset += sizeof(int); + offset += Copy(newMetadataXml, destination[offset..]); + _ = Copy(endBuffer.Span, destination[offset..]); + + return buffer; } private static void UpdateItemPaths(XDocument doc, string queryName) @@ -119,7 +142,7 @@ private static void UpdateItemPaths(XDocument doc, string queryName) private static void UpdateEntries(XDocument doc) { var now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture); - var lastUpdatedValue = ($"d{now}").Replace("Z", "0000Z", StringComparison.Ordinal); + var lastUpdatedValue = $"d{now}".Replace("Z", "0000Z", StringComparison.Ordinal); foreach (var entry in doc.Descendants().Where(e => e.Name.LocalName == XmlNames.Elements.Entry)) { @@ -134,5 +157,11 @@ private static void UpdateEntries(XDocument doc) } } } + + private static int Copy(ReadOnlySpan source, Span destination) + { + source.CopyTo(destination); + return source.Length; + } } diff --git a/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs b/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs index f7d3efa..de3c7b4 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Globalization; -using System.Linq; using System.Text; using System.Xml.Linq; @@ -20,8 +18,7 @@ public static (string Path, string Base64) GetDataMashup(ExcelArchive archive) continue; } - var bytes = archive.ReadBytes(entryPath); - var xml = XmlEncodingHelper.DecodeToString(bytes).TrimStart('\uFEFF'); + var xml = archive.ReadText(entryPath); var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); if (!string.Equals(doc.Root?.Name.NamespaceName, WorkbookConstants.DataMashupNamespace, StringComparison.Ordinal)) { diff --git a/dotnet/src/ConnectedWorkbooks/Internal/TemplateMetadataResolver.cs b/dotnet/src/ConnectedWorkbooks/Internal/TemplateMetadataResolver.cs new file mode 100644 index 0000000..78a39cc --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/TemplateMetadataResolver.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Linq; +using System.Xml.Linq; +using Microsoft.ConnectedWorkbooks.Models; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +internal sealed record TemplateMetadata(string WorksheetPath, string TablePath, (int Row, int Column) TableStart); + +internal static class TemplateMetadataResolver +{ + private static readonly XNamespace RelationshipsNamespace = "http://schemas.openxmlformats.org/package/2006/relationships"; + private static readonly XNamespace OfficeRelationshipsNamespace = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + + public static TemplateMetadata Resolve(ExcelArchive archive, TemplateSettings? templateSettings) + { + var worksheetPath = ResolveWorksheetPath(archive, templateSettings?.SheetName); + var (tablePath, tableStart) = ResolveTablePath(archive, templateSettings?.TableName); + return new TemplateMetadata(worksheetPath, tablePath, tableStart); + } + + private static string ResolveWorksheetPath(ExcelArchive archive, string? sheetName) + { + if (string.IsNullOrWhiteSpace(sheetName)) + { + if (!archive.EntryExists(WorkbookConstants.DefaultSheetPath)) + { + throw new InvalidOperationException("The workbook template does not contain the default worksheet 'xl/worksheets/sheet1.xml'. Provide FileConfiguration.TemplateSettings.SheetName to indicate the target sheet."); + } + + return WorkbookConstants.DefaultSheetPath; + } + + var workbookXml = archive.ReadText(WorkbookConstants.WorkbookXmlPath); + var workbookDoc = XDocument.Parse(workbookXml, LoadOptions.PreserveWhitespace); + var workbookNs = workbookDoc.Root?.Name.Namespace ?? XNamespace.None; + var sheetElement = workbookDoc + .Descendants(workbookNs + "sheet") + .FirstOrDefault(node => string.Equals(node.Attribute("name")?.Value, sheetName, StringComparison.OrdinalIgnoreCase)) + ?? throw new InvalidOperationException($"Worksheet '{sheetName}' was not found in the workbook template."); + + var relationshipId = sheetElement.Attribute(OfficeRelationshipsNamespace + "id")?.Value + ?? throw new InvalidOperationException($"Worksheet '{sheetName}' is missing the relationship id attribute."); + + var relsXml = archive.ReadText(WorkbookConstants.WorkbookRelsPath); + var relsDoc = XDocument.Parse(relsXml, LoadOptions.PreserveWhitespace); + var relationship = relsDoc + .Descendants(RelationshipsNamespace + XmlNames.Elements.Relationship) + .FirstOrDefault(node => string.Equals(node.Attribute("Id")?.Value, relationshipId, StringComparison.Ordinal)) + ?? throw new InvalidOperationException($"Relationship '{relationshipId}' referenced by worksheet '{sheetName}' was not found."); + + var target = relationship.Attribute("Target")?.Value + ?? throw new InvalidOperationException($"Relationship '{relationshipId}' does not contain a target path."); + + return NormalizePath(target); + } + + private static (string Path, (int Row, int Column) Start) ResolveTablePath(ExcelArchive archive, string? tableName) + { + if (!string.IsNullOrWhiteSpace(tableName)) + { + foreach (var entryPath in archive.EnumerateEntries(WorkbookConstants.TablesFolder)) + { + var metadata = ReadTableMetadata(archive.ReadText(entryPath)); + if (string.Equals(metadata.Name, tableName, StringComparison.OrdinalIgnoreCase)) + { + return (entryPath, metadata.Start); + } + } + + throw new InvalidOperationException($"Table '{tableName}' was not found in the workbook template."); + } + + if (!archive.EntryExists(WorkbookConstants.DefaultTablePath)) + { + throw new InvalidOperationException("The workbook template does not contain the default table 'xl/tables/table1.xml'. Provide FileConfiguration.TemplateSettings.TableName to indicate which table to target."); + } + + var defaultMetadata = ReadTableMetadata(archive.ReadText(WorkbookConstants.DefaultTablePath)); + return (WorkbookConstants.DefaultTablePath, defaultMetadata.Start); + } + + private static (string Name, (int Row, int Column) Start) ReadTableMetadata(string tableXml) + { + var doc = XDocument.Parse(tableXml, LoadOptions.PreserveWhitespace); + var name = doc.Root?.Attribute("name")?.Value + ?? throw new InvalidOperationException("Table definition is missing the 'name' attribute."); + var reference = doc.Root?.Attribute("ref")?.Value ?? "A1"; + var cleanedReference = reference.Replace("$", string.Empty, StringComparison.Ordinal); + var start = CellReferenceHelper.GetStartPosition(cleanedReference); + return (name, start); + } + + private static string NormalizePath(string target) + { + var normalized = target.Replace('\\', '/'); + if (normalized.StartsWith("/", StringComparison.Ordinal)) + { + return $"xl{normalized}"; + } + + return $"xl/{normalized}"; + } +} diff --git a/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs index 597b644..5530e5c 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. using System.Globalization; -using System.Linq; using System.Xml.Linq; using Microsoft.ConnectedWorkbooks.Models; @@ -12,11 +11,17 @@ internal sealed class WorkbookEditor { private readonly ExcelArchive _archive; private readonly DocumentProperties? _documentProperties; + private readonly string _worksheetPath; + private readonly string _tablePath; + private readonly (int Row, int Column) _tableStart; - public WorkbookEditor(ExcelArchive archive, DocumentProperties? documentProperties) + public WorkbookEditor(ExcelArchive archive, DocumentProperties? documentProperties, TemplateMetadata templateMetadata) { _archive = archive; _documentProperties = documentProperties; + _worksheetPath = templateMetadata.WorksheetPath; + _tablePath = templateMetadata.TablePath; + _tableStart = templateMetadata.TableStart; } public void UpdatePowerQueryDocument(string queryName, string mashupDocument) @@ -70,7 +75,7 @@ public int UpdateSharedStrings(string queryName) public void UpdateWorksheet(int sharedStringIndex) { - var xml = _archive.ReadText(WorkbookConstants.DefaultSheetPath); + var xml = _archive.ReadText(_worksheetPath); var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); var ns = doc.Root?.Name.Namespace ?? XNamespace.None; var cellValue = doc.Descendants(ns + XmlNames.Elements.CellValue).FirstOrDefault(); @@ -80,7 +85,7 @@ public void UpdateWorksheet(int sharedStringIndex) } cellValue.Value = sharedStringIndex.ToString(CultureInfo.InvariantCulture); - _archive.WriteText(WorkbookConstants.DefaultSheetPath, doc.ToString(SaveOptions.DisableFormatting)); + _archive.WriteText(_worksheetPath, doc.ToString(SaveOptions.DisableFormatting)); } public void UpdateQueryTable(string connectionId, bool refreshOnOpen) @@ -135,7 +140,7 @@ public void UpdateDocumentProperties() private void UpdateSheetData(TableData tableData) { - var xml = _archive.ReadText(WorkbookConstants.DefaultSheetPath); + var xml = _archive.ReadText(_worksheetPath); var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); var ns = doc.Root?.Name.Namespace ?? XNamespace.None; var x14ac = doc.Root?.GetNamespaceOfPrefix("x14ac") ?? XNamespace.None; @@ -146,8 +151,7 @@ private void UpdateSheetData(TableData tableData) } sheetData.RemoveNodes(); - var startCell = "A1"; - var (startRow, startColumn) = CellReferenceHelper.GetStartPosition(startCell); + var (startRow, startColumn) = _tableStart; var spans = $"{startColumn}:{startColumn + tableData.ColumnNames.Count - 1}"; var headerRow = new XElement(ns + XmlNames.Elements.Row, @@ -180,16 +184,16 @@ private void UpdateSheetData(TableData tableData) sheetData.Add(row); } - var endReference = CellReferenceHelper.BuildReference((startRow, startColumn), tableData.ColumnNames.Count, tableData.Rows.Count + 1); + var endReference = CellReferenceHelper.BuildReference(_tableStart, tableData.ColumnNames.Count, tableData.Rows.Count + 1); doc.Descendants(ns + XmlNames.Elements.Dimension).FirstOrDefault()?.SetAttributeValue(XmlNames.Attributes.Reference, endReference); doc.Descendants(ns + XmlNames.Elements.Selection).FirstOrDefault()?.SetAttributeValue(XmlNames.Attributes.SqRef, endReference); - _archive.WriteText(WorkbookConstants.DefaultSheetPath, doc.ToString(SaveOptions.DisableFormatting)); + _archive.WriteText(_worksheetPath, doc.ToString(SaveOptions.DisableFormatting)); } private void UpdateTableDefinition(TableData tableData) { - var xml = _archive.ReadText(WorkbookConstants.DefaultTablePath); + var xml = _archive.ReadText(_tablePath); var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); var ns = doc.Root?.Name.Namespace ?? XNamespace.None; var tableColumns = doc.Descendants(ns + XmlNames.Elements.TableColumns).FirstOrDefault(); @@ -208,11 +212,11 @@ private void UpdateTableDefinition(TableData tableData) tableColumns.Add(column); } - var reference = $"A1:{CellReferenceHelper.ColumnNumberToName(tableData.ColumnNames.Count - 1)}{tableData.Rows.Count + 1}"; + var reference = CellReferenceHelper.BuildReference(_tableStart, tableData.ColumnNames.Count, tableData.Rows.Count + 1); doc.Root?.SetAttributeValue(XmlNames.Attributes.Reference, reference); doc.Descendants(ns + XmlNames.Elements.AutoFilter).FirstOrDefault()?.SetAttributeValue(XmlNames.Attributes.Reference, reference); - _archive.WriteText(WorkbookConstants.DefaultTablePath, doc.ToString(SaveOptions.DisableFormatting)); + _archive.WriteText(_tablePath, doc.ToString(SaveOptions.DisableFormatting)); } private void UpdateWorkbookDefinedName(TableData tableData) @@ -227,13 +231,18 @@ private void UpdateWorkbookDefinedName(TableData tableData) return; } - var reference = $"!$A$1:${CellReferenceHelper.ColumnNumberToName(tableData.ColumnNames.Count - 1)}${tableData.Rows.Count + 1}"; - definedName.Value = reference; + var range = CellReferenceHelper.BuildReference(_tableStart, tableData.ColumnNames.Count, tableData.Rows.Count + 1); + definedName.Value = CellReferenceHelper.WithAbsolute(range); _archive.WriteText(WorkbookConstants.WorkbookXmlPath, doc.ToString(SaveOptions.DisableFormatting)); } private void UpdateQueryTableColumns(TableData tableData) { + if (!_archive.EntryExists(WorkbookConstants.QueryTablePath)) + { + return; + } + var xml = _archive.ReadText(WorkbookConstants.QueryTablePath); var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); var ns = doc.Root?.Name.Namespace ?? XNamespace.None; diff --git a/dotnet/src/ConnectedWorkbooks/Internal/XmlEncodingHelper.cs b/dotnet/src/ConnectedWorkbooks/Internal/XmlEncodingHelper.cs deleted file mode 100644 index 470e291..0000000 --- a/dotnet/src/ConnectedWorkbooks/Internal/XmlEncodingHelper.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Text; -using System.Linq; - -namespace Microsoft.ConnectedWorkbooks.Internal; - -internal static class XmlEncodingHelper -{ - public static string DecodeToString(byte[] xmlBytes) - { - if (xmlBytes.Length >= 3 && xmlBytes[0] == 0xEF && xmlBytes[1] == 0xBB && xmlBytes[2] == 0xBF) - { - return Encoding.UTF8.GetString(xmlBytes, 3, xmlBytes.Length - 3); - } - - if (xmlBytes.Length >= 2 && xmlBytes[0] == 0xFF && xmlBytes[1] == 0xFE) - { - return Encoding.Unicode.GetString(xmlBytes, 2, xmlBytes.Length - 2); - } - - if (xmlBytes.Length >= 2 && xmlBytes[0] == 0xFE && xmlBytes[1] == 0xFF) - { - return Encoding.BigEndianUnicode.GetString(xmlBytes, 2, xmlBytes.Length - 2); - } - - return Encoding.UTF8.GetString(xmlBytes); - } - - public static byte[] EncodeWithBom(string content, Encoding encoding) - { - if (encoding == Encoding.Unicode) - { - return Encoding.Unicode.GetPreamble().Concat(encoding.GetBytes(content)).ToArray(); - } - - if (encoding == Encoding.BigEndianUnicode) - { - return Encoding.BigEndianUnicode.GetPreamble().Concat(encoding.GetBytes(content)).ToArray(); - } - - if (encoding == Encoding.UTF8) - { - return Encoding.UTF8.GetPreamble().Concat(encoding.GetBytes(content)).ToArray(); - } - - return encoding.GetBytes(content); - } -} - diff --git a/dotnet/src/ConnectedWorkbooks/InternalsVisibleTo.cs b/dotnet/src/ConnectedWorkbooks/InternalsVisibleTo.cs new file mode 100644 index 0000000..f99f6f9 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/InternalsVisibleTo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ConnectedWorkbooks.Tests")] diff --git a/dotnet/src/ConnectedWorkbooks/Models/TableData.cs b/dotnet/src/ConnectedWorkbooks/Models/TableData.cs index 9bb59fd..9652396 100644 --- a/dotnet/src/ConnectedWorkbooks/Models/TableData.cs +++ b/dotnet/src/ConnectedWorkbooks/Models/TableData.cs @@ -8,6 +8,5 @@ namespace Microsoft.ConnectedWorkbooks.Models; /// public sealed record TableData(IReadOnlyList ColumnNames, IReadOnlyList> Rows) { - public static TableData Empty { get; } = new(Array.Empty(), Array.Empty>()); } diff --git a/dotnet/src/ConnectedWorkbooks/WorkbookManager.cs b/dotnet/src/ConnectedWorkbooks/WorkbookManager.cs index 6e6a9ac..afd4f5d 100644 --- a/dotnet/src/ConnectedWorkbooks/WorkbookManager.cs +++ b/dotnet/src/ConnectedWorkbooks/WorkbookManager.cs @@ -24,11 +24,13 @@ public async Task GenerateSingleQueryWorkbookAsync( : query.QueryName!; PqUtilities.ValidateQueryName(queryName); - var templateBytes = fileConfiguration?.TemplateBytes ?? EmbeddedTemplateLoader.LoadSimpleQueryTemplate(); + var templateBytes = fileConfiguration?.TemplateBytes + ?? await EmbeddedTemplateLoader.LoadSimpleQueryTemplateAsync(cancellationToken).ConfigureAwait(false); var tableData = initialDataGrid is null ? null : GridParser.Parse(initialDataGrid); using var archive = ExcelArchive.Load(templateBytes); - var editor = new WorkbookEditor(archive, fileConfiguration?.DocumentProperties); + var templateMetadata = TemplateMetadataResolver.Resolve(archive, fileConfiguration?.TemplateSettings); + var editor = new WorkbookEditor(archive, fileConfiguration?.DocumentProperties, templateMetadata); var mashup = PowerQueryGenerator.GenerateSingleQueryMashup(queryName, query.QueryMashup); editor.UpdatePowerQueryDocument(queryName, mashup); var connectionId = editor.UpdateConnections(queryName, query.RefreshOnOpen); @@ -43,4 +45,24 @@ public async Task GenerateSingleQueryWorkbookAsync( return await Task.FromResult(archive.ToArray()); } + + public async Task GenerateTableWorkbookFromGridAsync( + Grid grid, + FileConfiguration? fileConfiguration = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(grid); + + var templateBytes = fileConfiguration?.TemplateBytes + ?? await EmbeddedTemplateLoader.LoadBlankTableTemplateAsync(cancellationToken).ConfigureAwait(false); + var tableData = GridParser.Parse(grid); + + using var archive = ExcelArchive.Load(templateBytes); + var templateMetadata = TemplateMetadataResolver.Resolve(archive, fileConfiguration?.TemplateSettings); + var editor = new WorkbookEditor(archive, fileConfiguration?.DocumentProperties, templateMetadata); + editor.UpdateTableData(tableData); + editor.UpdateDocumentProperties(); + + return await Task.FromResult(archive.ToArray()); + } } diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/ConnectedWorkbooks.Tests.csproj b/dotnet/tests/ConnectedWorkbooks.Tests/ConnectedWorkbooks.Tests.csproj index 0b01c7b..13f022a 100644 --- a/dotnet/tests/ConnectedWorkbooks.Tests/ConnectedWorkbooks.Tests.csproj +++ b/dotnet/tests/ConnectedWorkbooks.Tests/ConnectedWorkbooks.Tests.csproj @@ -11,6 +11,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/PqUtilitiesTests.cs b/dotnet/tests/ConnectedWorkbooks.Tests/PqUtilitiesTests.cs new file mode 100644 index 0000000..a69ff54 --- /dev/null +++ b/dotnet/tests/ConnectedWorkbooks.Tests/PqUtilitiesTests.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Text; +using Microsoft.ConnectedWorkbooks.Internal; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace ConnectedWorkbooks.Tests; + +[TestClass] +public sealed class PqUtilitiesTests +{ + [TestMethod] + public async Task GetDataMashupHandlesUtf16LittleEndianBom() + { + await AssertDataMashupRoundtripAsync(Encoding.Unicode); + } + + [TestMethod] + public async Task GetDataMashupHandlesUtf16BigEndianBom() + { + await AssertDataMashupRoundtripAsync(Encoding.BigEndianUnicode); + } + + [TestMethod] + public async Task GetDataMashupHandlesUtf8Bom() + { + await AssertDataMashupRoundtripAsync(new UTF8Encoding(encoderShouldEmitUTF8Identifier: true)); + } + + private static async Task AssertDataMashupRoundtripAsync(Encoding encoding) + { + var template = await EmbeddedTemplateLoader.LoadSimpleQueryTemplateAsync(); + using var archive = ExcelArchive.Load(template); + var base64 = Convert.ToBase64String(Guid.NewGuid().ToByteArray()); + WriteDataMashup(archive, base64, encoding); + + var (_, decodedBase64) = PqUtilities.GetDataMashup(archive); + Assert.AreEqual(base64, decodedBase64, $"DataMashup decoding failed for encoding {encoding.WebName}."); + } + + private static void WriteDataMashup(ExcelArchive archive, string base64, Encoding encoding) + { + var path = LocateDataMashupEntry(archive); + var xml = $"{base64}"; + var preamble = encoding.GetPreamble(); + var payload = encoding.GetBytes(xml); + var buffer = new byte[preamble.Length + payload.Length]; + preamble.CopyTo(buffer, 0); + payload.CopyTo(buffer, preamble.Length); + archive.WriteBytes(path, buffer); + } + + private static string LocateDataMashupEntry(ExcelArchive archive) + { + foreach (var entryPath in archive.EnumerateEntries(WorkbookConstants.CustomXmlFolder)) + { + if (!WorkbookConstants.CustomXmlItemRegex.IsMatch(entryPath)) + { + continue; + } + + var xml = archive.ReadText(entryPath); + var doc = System.Xml.Linq.XDocument.Parse(xml); + if (string.Equals(doc.Root?.Name.LocalName, "DataMashup", StringComparison.Ordinal)) + { + return entryPath; + } + } + + throw new AssertFailedException("DataMashup entry was not found in the template."); + } +} diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/TestResults/90a856a6-0001-416c-bed0-a68b7b8eb703/coverage.cobertura.xml b/dotnet/tests/ConnectedWorkbooks.Tests/TestResults/90a856a6-0001-416c-bed0-a68b7b8eb703/coverage.cobertura.xml new file mode 100644 index 0000000..6e0d849 --- /dev/null +++ b/dotnet/tests/ConnectedWorkbooks.Tests/TestResults/90a856a6-0001-416c-bed0-a68b7b8eb703/coverage.cobertura.xml @@ -0,0 +1,3020 @@ + + + + E:\work\connected-workbooks\dotnet\src\ConnectedWorkbooks\ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs b/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs index 7974366..467a781 100644 --- a/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs +++ b/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs @@ -2,9 +2,11 @@ // Licensed under the MIT license. using System.IO.Compression; +using System.Linq; using System.Text; using System.Xml.Linq; using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Internal; using Microsoft.ConnectedWorkbooks.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -14,6 +16,12 @@ namespace ConnectedWorkbooks.Tests; public sealed class WorkbookManagerTests { private readonly WorkbookManager _manager = new(); + private const string CustomSheetName = "DataSheet"; + private const string CustomSheetPath = "xl/worksheets/customSheet.xml"; + private const string CustomSheetRelsPath = "xl/worksheets/_rels/customSheet.xml.rels"; + private const string CustomTableName = "SalesTable"; + private const string CustomTablePath = "xl/tables/customTable.xml"; + private const string CustomTableRange = "C3:D5"; [TestMethod] public async Task GeneratesWorkbookWithMashupAndTable() @@ -57,6 +65,32 @@ public async Task GeneratesWorkbookWithMashupAndTable() StringAssert.Contains(sectionContent, "DataAgentQuery"); } + [TestMethod] + public async Task GeneratesTableWorkbookFromGrid() + { + var grid = new Grid(new List> + { + new List { "Product", "Quantity", "Price" }, + new List { "Apples", 5, 1.25 }, + new List { "Bananas", 8, 0.99 } + }, new GridConfig { PromoteHeaders = true }); + + var bytes = await _manager.GenerateTableWorkbookFromGridAsync(grid); + Assert.IsTrue(bytes.Length > 0, "The generated workbook should not be empty."); + + using var archiveStream = new MemoryStream(bytes); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read); + + var tableXml = ReadEntry(archive, "xl/tables/table1.xml"); + StringAssert.Contains(tableXml, "Product"); + StringAssert.Contains(tableXml, "Quantity"); + StringAssert.Contains(tableXml, "Price"); + + var sheetXml = ReadEntry(archive, "xl/worksheets/sheet1.xml"); + StringAssert.Contains(sheetXml, "Apples"); + StringAssert.Contains(sheetXml, "Bananas"); + } + [TestMethod] public void RejectsInvalidQueryName() { @@ -64,6 +98,58 @@ public void RejectsInvalidQueryName() Assert.ThrowsException(() => _manager.GenerateSingleQueryWorkbookAsync(query).GetAwaiter().GetResult()); } + [TestMethod] + public async Task RequiresTemplateSettingsWhenDefaultsMissing() + { + var template = await CreateCustomTemplateAsync(); + var grid = new Grid(new List> + { + new List { "Col1", "Col2" }, + new List { "A", "B" } + }, new GridConfig { PromoteHeaders = true }); + + var config = new FileConfiguration { TemplateBytes = template }; + + await Assert.ThrowsExceptionAsync(() => _manager.GenerateTableWorkbookFromGridAsync(grid, config)); + } + + [TestMethod] + public async Task GeneratesTableWorkbookWithTemplateSettings() + { + var template = await CreateCustomTemplateAsync(); + var grid = new Grid(new List> + { + new List { "Product", "Qty" }, + new List { "Apples", 5 }, + new List { "Bananas", 3 } + }, new GridConfig { PromoteHeaders = true }); + + var config = new FileConfiguration + { + TemplateBytes = template, + TemplateSettings = new TemplateSettings + { + SheetName = CustomSheetName, + TableName = CustomTableName + } + }; + + var bytes = await _manager.GenerateTableWorkbookFromGridAsync(grid, config); + + using var archiveStream = new MemoryStream(bytes); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read); + + var sheetXml = ReadEntry(archive, CustomSheetPath); + var sheetDoc = XDocument.Parse(sheetXml); + var sheetNs = sheetDoc.Root?.Name.Namespace ?? XNamespace.None; + var firstCell = sheetDoc.Descendants(sheetNs + "c").First(); + Assert.AreEqual("C3", firstCell.Attribute("r")?.Value, "Table data was not written at the expected starting cell."); + + var tableXml = ReadEntry(archive, CustomTablePath); + StringAssert.Contains(tableXml, $"name=\"{CustomTableName}\""); + StringAssert.Contains(tableXml, CustomTableRange, "Table reference was not updated."); + } + private static string ReadEntry(ZipArchive archive, string path) { var entry = archive.GetEntry(path) ?? throw new AssertFailedException($"Entry '{path}' was not found in the workbook."); @@ -72,6 +158,91 @@ private static string ReadEntry(ZipArchive archive, string path) return reader.ReadToEnd(); } + private static async Task CreateCustomTemplateAsync() + { + var template = await EmbeddedTemplateLoader.LoadBlankTableTemplateAsync(); + using var stream = new MemoryStream(); + stream.Write(template, 0, template.Length); + + using (var zip = new ZipArchive(stream, ZipArchiveMode.Update, leaveOpen: true)) + { + RenameEntry(zip, "xl/worksheets/sheet1.xml", CustomSheetPath); + RenameEntry(zip, "xl/worksheets/_rels/sheet1.xml.rels", CustomSheetRelsPath); + RenameEntry(zip, "xl/tables/table1.xml", CustomTablePath); + + MutateXmlEntry(zip, "xl/workbook.xml", doc => + { + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var sheet = doc.Descendants(ns + "sheet").First(); + sheet.SetAttributeValue("name", CustomSheetName); + }); + + MutateXmlEntry(zip, "xl/_rels/workbook.xml.rels", doc => + { + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var relationship = doc.Descendants(ns + "Relationship") + .First(node => node.Attribute("Target")?.Value?.EndsWith("worksheets/sheet1.xml", StringComparison.OrdinalIgnoreCase) == true); + relationship.SetAttributeValue("Target", "worksheets/customSheet.xml"); + }); + + MutateXmlEntry(zip, CustomSheetRelsPath, doc => + { + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + foreach (var relationship in doc.Descendants(ns + "Relationship")) + { + var target = relationship.Attribute("Target")?.Value; + if (target != null && target.EndsWith("../tables/table1.xml", StringComparison.OrdinalIgnoreCase)) + { + relationship.SetAttributeValue("Target", "../tables/customTable.xml"); + } + } + }); + + MutateXmlEntry(zip, CustomTablePath, doc => + { + doc.Root?.SetAttributeValue("name", CustomTableName); + doc.Root?.SetAttributeValue("displayName", CustomTableName); + doc.Root?.SetAttributeValue("ref", CustomTableRange); + }); + } + + return stream.ToArray(); + } + + private static void RenameEntry(ZipArchive zip, string originalName, string newName) + { + var entry = zip.GetEntry(originalName) ?? throw new AssertFailedException($"Entry '{originalName}' not found in the workbook template."); + using var buffer = new MemoryStream(); + using (var source = entry.Open()) + { + source.CopyTo(buffer); + } + + entry.Delete(); + var newEntry = zip.CreateEntry(newName); + using var target = newEntry.Open(); + buffer.Position = 0; + buffer.CopyTo(target); + } + + private static void MutateXmlEntry(ZipArchive zip, string path, Action mutate) + { + var entry = zip.GetEntry(path) ?? throw new AssertFailedException($"Entry '{path}' not found in the workbook template."); + string xml; + using (var reader = new StreamReader(entry.Open(), Encoding.UTF8, detectEncodingFromByteOrderMarks: true)) + { + xml = reader.ReadToEnd(); + } + + var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); + mutate(doc); + + entry.Delete(); + var newEntry = zip.CreateEntry(path); + using var writer = new StreamWriter(newEntry.Open(), new UTF8Encoding(false)); + writer.Write(doc.ToString(SaveOptions.DisableFormatting)); + } + private static string ExtractSection1m(string dataMashupBase64) { var bytes = Convert.FromBase64String(dataMashupBase64); From 1faa1dd949821d0fc24ac34d6c1db775c030f22f Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Wed, 26 Nov 2025 15:42:18 +0100 Subject: [PATCH 3/5] first working version --- .../sample/ConnectedWorkbooks.Sample.csproj | 14 ++ dotnet/sample/Program.cs | 38 ++++ .../ConnectedWorkbooks.csproj | 9 + .../Internal/ArrayReader.cs | 24 +++ .../Internal/CellReferenceHelper.cs | 31 +++- .../Internal/EmbeddedTemplateLoader.cs | 32 +++- .../Internal/ExcelArchive.cs | 42 +++++ .../ConnectedWorkbooks/Internal/GridParser.cs | 8 + .../Internal/MashupDocumentParser.cs | 168 +++++++++++++----- .../Internal/PowerQueryGenerator.cs | 11 +- .../Internal/PqUtilities.cs | 19 ++ .../Internal/QueryNameValidator.cs | 60 +++++++ .../Internal/TemplateMetadataResolver.cs | 15 ++ .../Internal/WorkbookConstants.cs | 4 + .../Internal/WorkbookEditor.cs | 74 +++++++- .../ConnectedWorkbooks/Internal/XmlNames.cs | 9 + .../Models/DocumentProperties.cs | 31 ++++ dotnet/src/ConnectedWorkbooks/Models/Grid.cs | 2 + .../ConnectedWorkbooks/Models/QueryInfo.cs | 15 ++ .../ConnectedWorkbooks/Models/TableData.cs | 2 + .../Models/TemplateSettings.cs | 7 + .../src/ConnectedWorkbooks/WorkbookManager.cs | 45 +++-- .../PqUtilitiesTests.cs | 16 +- .../WorkbookManagerTests.cs | 43 +++-- typescript/package.json | 3 +- typescript/scripts/generateWeatherSample.js | 44 +++++ typescript/scripts/inspectMashup.js | 28 +++ typescript/scripts/validateImplementations.js | 81 +++++++++ typescript/scripts/workbookDetails.js | 159 +++++++++++++++++ 29 files changed, 933 insertions(+), 101 deletions(-) create mode 100644 dotnet/sample/ConnectedWorkbooks.Sample.csproj create mode 100644 dotnet/sample/Program.cs create mode 100644 dotnet/src/ConnectedWorkbooks/Internal/QueryNameValidator.cs create mode 100644 typescript/scripts/generateWeatherSample.js create mode 100644 typescript/scripts/inspectMashup.js create mode 100644 typescript/scripts/validateImplementations.js create mode 100644 typescript/scripts/workbookDetails.js diff --git a/dotnet/sample/ConnectedWorkbooks.Sample.csproj b/dotnet/sample/ConnectedWorkbooks.Sample.csproj new file mode 100644 index 0000000..d1ebd27 --- /dev/null +++ b/dotnet/sample/ConnectedWorkbooks.Sample.csproj @@ -0,0 +1,14 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + diff --git a/dotnet/sample/Program.cs b/dotnet/sample/Program.cs new file mode 100644 index 0000000..b92433a --- /dev/null +++ b/dotnet/sample/Program.cs @@ -0,0 +1,38 @@ +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); + +var mashup = """ +let + Source = #table( + {"City","TempC"}, + { + {"Seattle", 18}, + {"London", 15}, + {"Sydney", 22} + } + ) +in + Source +"""; + +var query = new QueryInfo( + queryMashup: mashup, + queryName: "WeatherSample", + refreshOnOpen: false); + +var grid = new Grid(new[] +{ + new object?[] { "City", "TempC" }, + new object?[] { "Seattle", 0 }, + new object?[] { "London", 0 }, + new object?[] { "Sydney", 0 } +}, new GridConfig { PromoteHeaders = true }); + +var bytes = manager.GenerateSingleQueryWorkbook(query, grid); +var repoRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..")); +var outputPath = Path.Combine(repoRoot, "WeatherSample.xlsx"); +await File.WriteAllBytesAsync(outputPath, bytes); + +Console.WriteLine($"Workbook generated: {outputPath}"); diff --git a/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj b/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj index a00bdbf..5f6964f 100644 --- a/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj +++ b/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj @@ -4,6 +4,7 @@ net8.0 enable enable + true @@ -14,4 +15,12 @@ + + + + diff --git a/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs b/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs index 123a311..673cb20 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/ArrayReader.cs @@ -5,22 +5,38 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Lightweight helper for sequentially reading primitive values from a byte buffer. +/// internal sealed class ArrayReader { private readonly ReadOnlyMemory _buffer; private int _offset; + /// + /// Initializes a new reader that iterates over the supplied byte array. + /// + /// The byte array to consume. public ArrayReader(byte[] buffer) : this(new ReadOnlyMemory(buffer)) { } + /// + /// Initializes a new reader for the provided byte memory segment. + /// + /// The data segment to consume. public ArrayReader(ReadOnlyMemory buffer) { _buffer = buffer; _offset = 0; } + /// + /// Reads the specified number of bytes, advancing the reader past them. + /// + /// Number of bytes to read. + /// The requested slice of the underlying buffer. public ReadOnlyMemory ReadMemory(int count) { EnsureAvailable(count); @@ -29,6 +45,10 @@ public ReadOnlyMemory ReadMemory(int count) return slice; } + /// + /// Reads a 32-bit little-endian integer from the buffer. + /// + /// The parsed integer. public int ReadInt32() { EnsureAvailable(sizeof(int)); @@ -37,6 +57,10 @@ public int ReadInt32() return value; } + /// + /// Returns the remaining bytes from the current position and advances to the end. + /// + /// A slice containing all remaining bytes. public ReadOnlyMemory ReadToEnd() { var slice = _buffer.Slice(_offset); diff --git a/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs b/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs index 5b11897..7544b50 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/CellReferenceHelper.cs @@ -3,8 +3,16 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Utility methods for converting between Excel-style cell references and numeric coordinates. +/// internal static class CellReferenceHelper { + /// + /// Returns the zero-based row/column tuple that represents the starting cell in the reference. + /// + /// A cell reference such as "A1" or "A1:B5". + /// A tuple containing one-based row and column coordinates. public static (int Row, int Column) GetStartPosition(string reference) { // Reference format "A1" or "A1:B5"; we only care about the first cell @@ -16,6 +24,11 @@ public static (int Row, int Column) GetStartPosition(string reference) return (row, column); } + /// + /// Converts a zero-based column index into its Excel column name. + /// + /// Zero-based column index. + /// The Excel column label (e.g. 0 -> "A"). public static string ColumnNumberToName(int columnIndex) { columnIndex++; // zero-based to one-based @@ -30,6 +43,13 @@ public static string ColumnNumberToName(int columnIndex) return columnName; } + /// + /// Builds a rectangular range reference given a starting coordinate and bounds. + /// + /// One-based starting position. + /// Number of columns in the range. + /// Number of rows in the range. + /// The Excel range reference spanning the requested area. public static string BuildReference((int Row, int Column) start, int columnCount, int rowCount) { var endColumnIndex = start.Column - 1 + columnCount; @@ -39,11 +59,18 @@ public static string BuildReference((int Row, int Column) start, int columnCount return $"{startRef}:{endRef}"; } - public static string WithAbsolute(string reference) + /// + /// Converts the provided range into an absolute reference with an optional sheet prefix. + /// + /// The relative range reference. + /// Optional sheet prefix (for example 'Sheet1'). + /// The absolute equivalent (e.g. 'Sheet1'!$A$1:$B$2). + public static string WithAbsolute(string reference, string? sheetName = null) { var (row, column) = GetStartPosition(reference); var (endRow, endColumn) = GetEndPosition(reference); - return $"!${ColumnNumberToName(column - 1)}${row}:${ColumnNumberToName(endColumn - 1)}${endRow}"; + var prefix = string.IsNullOrEmpty(sheetName) ? string.Empty : $"{sheetName}!"; + return $"{prefix}${ColumnNumberToName(column - 1)}${row}:${ColumnNumberToName(endColumn - 1)}${endRow}"; } private static (int Row, int Column) GetEndPosition(string reference) diff --git a/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs b/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs index 320da39..38e7f93 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/EmbeddedTemplateLoader.cs @@ -5,25 +5,41 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Provides access to the workbook templates embedded in the assembly resources. +/// internal static class EmbeddedTemplateLoader { private const string ResourcePrefix = "ConnectedWorkbooks.Templates."; private const string SimpleQueryTemplateResource = ResourcePrefix + "SIMPLE_QUERY_WORKBOOK_TEMPLATE.xlsx"; private const string BlankTableTemplateResource = ResourcePrefix + "SIMPLE_BLANK_TABLE_TEMPLATE.xlsx"; + private static readonly Lazy SimpleQueryTemplate = new(() => LoadTemplateFromManifest(SimpleQueryTemplateResource)); + private static readonly Lazy BlankTableTemplate = new(() => LoadTemplateFromManifest(BlankTableTemplateResource)); - public static Task LoadSimpleQueryTemplateAsync(CancellationToken cancellationToken = default) => - LoadTemplateAsync(SimpleQueryTemplateResource, cancellationToken); + /// + /// Returns the cached bytes for the single-query workbook template. + /// + public static byte[] LoadSimpleQueryTemplate() => + SimpleQueryTemplate.Value; - public static Task LoadBlankTableTemplateAsync(CancellationToken cancellationToken = default) => - LoadTemplateAsync(BlankTableTemplateResource, cancellationToken); + /// + /// Returns the cached bytes for the blank table workbook template. + /// + public static byte[] LoadBlankTableTemplate() => + BlankTableTemplate.Value; - private static async Task LoadTemplateAsync(string resourceName, CancellationToken cancellationToken) + /// + /// Reads an embedded resource stream and returns its contents as a byte array. + /// + /// Fully qualified manifest resource name. + /// The resource contents. + private static byte[] LoadTemplateFromManifest(string resourceName) { var assembly = Assembly.GetExecutingAssembly(); - await using var stream = assembly.GetManifestResourceStream(resourceName) + using var stream = assembly.GetManifestResourceStream(resourceName) ?? throw new InvalidOperationException($"Unable to locate embedded template '{resourceName}'."); - await using var memory = new MemoryStream(); - await stream.CopyToAsync(memory, cancellationToken); + using var memory = new MemoryStream(); + stream.CopyTo(memory); return memory.ToArray(); } } diff --git a/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs b/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs index c0d8b04..c4926b7 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/ExcelArchive.cs @@ -6,6 +6,9 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Thin wrapper around that simplifies editing workbook parts in memory. +/// internal sealed class ExcelArchive : IDisposable { private readonly MemoryStream _stream; @@ -20,8 +23,17 @@ private ExcelArchive(byte[] template) _zipArchive = new ZipArchive(_stream, ZipArchiveMode.Update, leaveOpen: true); } + /// + /// Loads the supplied workbook template bytes into an editable archive. + /// + /// The XLSX template to load. + /// An ready for manipulation. public static ExcelArchive Load(byte[] template) => new(template); + /// + /// Serializes the in-memory workbook back into a byte array. + /// + /// The workbook bytes. public byte[] ToArray() { _zipArchive?.Dispose(); @@ -29,6 +41,11 @@ public byte[] ToArray() return _stream.ToArray(); } + /// + /// Reads the contents of the specified part as UTF-8 text. + /// + /// Part path inside the archive. + /// File contents. public string ReadText(string path) { var entry = GetEntry(path); @@ -37,6 +54,12 @@ public string ReadText(string path) return reader.ReadToEnd(); } + /// + /// Writes text into the specified part, truncating existing data. + /// + /// Part path inside the archive. + /// Text to persist. + /// Optional encoding (defaults to UTF-8 without BOM). public void WriteText(string path, string content, Encoding? encoding = null) { var entry = GetOrCreateEntry(path); @@ -47,6 +70,11 @@ public void WriteText(string path, string content, Encoding? encoding = null) writer.Flush(); } + /// + /// Writes raw bytes into the specified part, truncating existing data. + /// + /// Part path inside the archive. + /// Data to persist. public void WriteBytes(string path, byte[] content) { var entry = GetOrCreateEntry(path); @@ -55,6 +83,11 @@ public void WriteBytes(string path, byte[] content) stream.Write(content, 0, content.Length); } + /// + /// Enumerates entries that reside under the provided folder prefix. + /// + /// Folder prefix, e.g. xl/tables/. + /// Paths of matching entries. public IEnumerable EnumerateEntries(string folderPrefix) { EnsureNotDisposed(); @@ -68,12 +101,20 @@ public IEnumerable EnumerateEntries(string folderPrefix) } } + /// + /// Indicates whether the specified entry exists within the archive. + /// + /// Part path inside the archive. public bool EntryExists(string path) { EnsureNotDisposed(); return _zipArchive!.GetEntry(path) is not null; } + /// + /// Removes the specified entry if present. + /// + /// Part path inside the archive. public void Remove(string path) { EnsureNotDisposed(); @@ -92,6 +133,7 @@ private ZipArchiveEntry GetOrCreateEntry(string path) return _zipArchive?.GetEntry(path) ?? _zipArchive!.CreateEntry(path, CompressionLevel.Optimal); } + /// public void Dispose() { if (_disposed) diff --git a/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs b/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs index fe8e80f..de5223b 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/GridParser.cs @@ -5,8 +5,16 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Converts the public abstraction into normalized instances. +/// internal static class GridParser { + /// + /// Normalizes the supplied grid into a form that can be written to Excel. + /// + /// Grid data supplied by the caller. + /// A normalized instance. public static TableData Parse(Grid grid) { grid ??= new Grid(Array.Empty>()); diff --git a/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs b/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs index a35ced1..e985846 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/MashupDocumentParser.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Buffers.Binary; using System.IO.Compression; using System.Text; @@ -8,8 +9,18 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Handles low-level editing of the Power Query (DataMashup) payload inside workbook templates. +/// internal static class MashupDocumentParser { + /// + /// Replaces the contents of the Section1.m formula with the provided mashup and updates metadata accordingly. + /// + /// Original DataMashup payload encoded as Base64. + /// Friendly query name that should be referenced throughout metadata. + /// New M document to write into Section1.m. + /// A base64 string with the updated mashup payload. public static string ReplaceSingleQuery(string base64, string queryName, string queryMashupDocument) { var buffer = Convert.FromBase64String(base64); @@ -36,24 +47,45 @@ public static string ReplaceSingleQuery(string base64, string queryName, string + endBuffer.Length; var finalBytes = new byte[totalLength]; - var destination = finalBytes.AsSpan(); - var offset = 0; - - offset += Copy(versionBytes.Span, destination[offset..]); - BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset, sizeof(int)), newPackage.Length); - offset += sizeof(int); - offset += Copy(newPackage, destination[offset..]); - BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset, sizeof(int)), permissionsSize); - offset += sizeof(int); - offset += Copy(permissions.Span, destination[offset..]); - BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset, sizeof(int)), newMetadata.Length); - offset += sizeof(int); - offset += Copy(newMetadata, destination[offset..]); - _ = Copy(endBuffer.Span, destination[offset..]); + var writer = new SpanWriter(finalBytes); + writer.WriteBytes(versionBytes.Span); + writer.WriteLength(newPackage.Length); + writer.WriteBytes(newPackage); + writer.WriteLength(permissionsSize); + writer.WriteBytes(permissions.Span); + writer.WriteLength(newMetadata.Length); + writer.WriteBytes(newMetadata); + writer.WriteBytes(endBuffer.Span); return Convert.ToBase64String(finalBytes); } + /// + /// Extracts the first query name referenced in the template's metadata. + /// + /// Base64-encoded DataMashup payload. + /// The query name referenced by Section1 (defaults to Query1 when missing). + public static string GetPrimaryQueryName(string base64) + { + var buffer = Convert.FromBase64String(base64); + var reader = new ArrayReader(buffer); + reader.ReadMemory(4); // version + var packageSize = reader.ReadInt32(); + reader.ReadMemory(packageSize); + var permissionsSize = reader.ReadInt32(); + reader.ReadMemory(permissionsSize); + var metadataSize = reader.ReadInt32(); + var metadataBytes = reader.ReadMemory(metadataSize); + + var metadataReader = new ArrayReader(metadataBytes); + metadataReader.ReadMemory(4); // metadata version + var metadataXmlSize = metadataReader.ReadInt32(); + var metadataXmlBytes = metadataReader.ReadMemory(metadataXmlSize); + + var doc = ParseMetadataDocument(metadataXmlBytes); + return ExtractQueryName(doc) ?? WorkbookConstants.DefaultQueryName; + } + private static byte[] EditSingleQueryPackage(ReadOnlySpan packageOpc, string queryMashupDocument) { using var packageStream = new MemoryStream(); @@ -83,41 +115,41 @@ private static byte[] EditSingleQueryMetadata(ReadOnlyMemory metadataBytes var metadataXmlBytes = reader.ReadMemory(metadataXmlSize); var endBuffer = reader.ReadToEnd(); + var metadataDoc = ParseMetadataDocument(metadataXmlBytes); + UpdateMetadataDocument(metadataDoc, queryName); + + var newMetadataXml = Encoding.UTF8.GetBytes(metadataDoc.ToString(SaveOptions.DisableFormatting)); + + var totalLength = metadataVersion.Length + sizeof(int) + newMetadataXml.Length + endBuffer.Length; + var buffer = new byte[totalLength]; + var writer = new SpanWriter(buffer); + writer.WriteBytes(metadataVersion.Span); + writer.WriteLength(newMetadataXml.Length); + writer.WriteBytes(newMetadataXml); + writer.WriteBytes(endBuffer.Span); + + return buffer; + } + + private static XDocument ParseMetadataDocument(ReadOnlyMemory metadataXmlBytes) + { var metadataXmlString = Encoding.UTF8.GetString(metadataXmlBytes.Span).TrimStart('\uFEFF'); - XDocument metadataDoc; try { - metadataDoc = XDocument.Parse(metadataXmlString, LoadOptions.PreserveWhitespace); + return XDocument.Parse(metadataXmlString, LoadOptions.PreserveWhitespace); } catch (Exception ex) { var preview = Convert.ToHexString(metadataXmlBytes.Span[..Math.Min(metadataXmlBytes.Length, 64)]); throw new InvalidOperationException($"Failed to parse metadata XML. Hex preview: {preview}", ex); } - UpdateItemPaths(metadataDoc, queryName); - UpdateEntries(metadataDoc); - - var newMetadataXml = Encoding.UTF8.GetBytes(metadataDoc.ToString(SaveOptions.DisableFormatting)); - - var totalLength = metadataVersion.Length + sizeof(int) + newMetadataXml.Length + endBuffer.Length; - var buffer = new byte[totalLength]; - var destination = buffer.AsSpan(); - var offset = 0; - - offset += Copy(metadataVersion.Span, destination[offset..]); - BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(offset, sizeof(int)), newMetadataXml.Length); - offset += sizeof(int); - offset += Copy(newMetadataXml, destination[offset..]); - _ = Copy(endBuffer.Span, destination[offset..]); - - return buffer; } - private static void UpdateItemPaths(XDocument doc, string queryName) + private static string? ExtractQueryName(XDocument doc) { if (doc.Root is null) { - return; + return null; } foreach (var itemPathElement in doc.Descendants().Where(e => e.Name.LocalName == XmlNames.Elements.ItemPath)) @@ -134,15 +166,21 @@ private static void UpdateItemPaths(XDocument doc, string queryName) continue; } - parts[1] = Uri.EscapeDataString(queryName); - itemPathElement.Value = string.Join('/', parts); + return Uri.UnescapeDataString(parts[1]); } + + return null; } - private static void UpdateEntries(XDocument doc) + private static void UpdateMetadataDocument(XDocument doc, string queryName) { - var now = DateTime.UtcNow.ToString("o", System.Globalization.CultureInfo.InvariantCulture); - var lastUpdatedValue = $"d{now}".Replace("Z", "0000Z", StringComparison.Ordinal); + if (!string.IsNullOrWhiteSpace(queryName)) + { + RenameItemPaths(doc, queryName); + } + + var now = DateTime.UtcNow.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff", System.Globalization.CultureInfo.InvariantCulture); + var lastUpdatedValue = $"d{now}0000Z"; foreach (var entry in doc.Descendants().Where(e => e.Name.LocalName == XmlNames.Elements.Entry)) { @@ -158,10 +196,54 @@ private static void UpdateEntries(XDocument doc) } } - private static int Copy(ReadOnlySpan source, Span destination) + private static void RenameItemPaths(XDocument doc, string queryName) { - source.CopyTo(destination); - return source.Length; + foreach (var itemPathElement in doc.Descendants().Where(e => e.Name.LocalName == XmlNames.Elements.ItemPath)) + { + var content = itemPathElement.Value; + if (!content.Contains("Section1/", StringComparison.Ordinal)) + { + continue; + } + + var parts = content.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2) + { + continue; + } + + parts[1] = Uri.EscapeDataString(queryName); + itemPathElement.Value = string.Join('/', parts); + } + } + + private ref struct SpanWriter + { + private readonly Span _destination; + private int _offset; + + public SpanWriter(Span destination) + { + _destination = destination; + _offset = 0; + } + + public void WriteBytes(ReadOnlySpan source) + { + if (source.Length == 0) + { + return; + } + + source.CopyTo(_destination[_offset..]); + _offset += source.Length; + } + + public void WriteLength(int value) + { + BinaryPrimitives.WriteInt32LittleEndian(_destination.Slice(_offset, sizeof(int)), value); + _offset += sizeof(int); + } } } diff --git a/dotnet/src/ConnectedWorkbooks/Internal/PowerQueryGenerator.cs b/dotnet/src/ConnectedWorkbooks/Internal/PowerQueryGenerator.cs index 50d7742..88f1069 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/PowerQueryGenerator.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/PowerQueryGenerator.cs @@ -3,11 +3,20 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Generates Power Query (M) documents used by the workbook templates. +/// internal static class PowerQueryGenerator { + /// + /// Creates a Section1.m document that exposes a single shared query with the supplied body. + /// + /// Name of the query to generate. + /// M script that defines the query. + /// The complete M document ready to embed. public static string GenerateSingleQueryMashup(string queryName, string queryBody) { - return $"section Section1;\n\nshared \"{queryName}\" = \n{queryBody};"; + return $"section Section1;\n\nshared #\"{queryName}\" = \n{queryBody};"; } } diff --git a/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs b/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs index de3c7b4..4bc8d92 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/PqUtilities.cs @@ -6,8 +6,16 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Helpers for interacting with the Power Query (DataMashup) parts inside workbook templates. +/// internal static class PqUtilities { + /// + /// Locates the DataMashup XML inside the workbook and returns its location and payload. + /// + /// Workbook archive to inspect. + /// The entry path and base64 payload. public static (string Path, string Base64) GetDataMashup(ExcelArchive archive) { foreach (var entryPath in archive.EnumerateEntries(WorkbookConstants.CustomXmlFolder)) @@ -32,6 +40,12 @@ public static (string Path, string Base64) GetDataMashup(ExcelArchive archive) throw new InvalidOperationException("DataMashup XML was not found in the workbook template."); } + /// + /// Writes the provided DataMashup payload back into the workbook. + /// + /// Workbook archive to mutate. + /// Entry path returned by . + /// Base64-encoded payload that should replace the current content. public static void SetDataMashup(ExcelArchive archive, string path, string base64) { var xml = $"{base64}"; @@ -39,6 +53,11 @@ public static void SetDataMashup(ExcelArchive archive, string path, string base6 archive.WriteBytes(path, encoded); } + /// + /// Validates that a query name conforms to Excel's constraints. + /// + /// The user supplied query name. + /// Thrown when the name violates naming rules. public static void ValidateQueryName(string queryName) { if (string.IsNullOrWhiteSpace(queryName)) diff --git a/dotnet/src/ConnectedWorkbooks/Internal/QueryNameValidator.cs b/dotnet/src/ConnectedWorkbooks/Internal/QueryNameValidator.cs new file mode 100644 index 0000000..2f99c63 --- /dev/null +++ b/dotnet/src/ConnectedWorkbooks/Internal/QueryNameValidator.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Microsoft.ConnectedWorkbooks.Internal; + +/// +/// Provides validation and normalization helpers for user-supplied query names. +/// +internal static class QueryNameValidator +{ + /// + /// Normalizes a user-supplied query name, applying defaults and validation. + /// + /// The original query name provided by the caller. + /// A validated query name. + /// Thrown when the name does not meet requirements. + public static string Resolve(string? candidate) + { + var effectiveName = string.IsNullOrWhiteSpace(candidate) + ? WorkbookConstants.DefaultQueryName + : candidate.Trim(); + + Validate(effectiveName); + return effectiveName; + } + + /// + /// Validates a query name using the same constraints as the TypeScript implementation. + /// + /// Name to validate. + /// Thrown when the name does not meet requirements. + public static void Validate(string queryName) + { + if (string.IsNullOrWhiteSpace(queryName)) + { + throw new ArgumentException("Query name cannot be empty.", nameof(queryName)); + } + + if (queryName.Length > WorkbookConstants.MaxQueryLength) + { + throw new ArgumentException($"Query names are limited to {WorkbookConstants.MaxQueryLength} characters.", nameof(queryName)); + } + + foreach (var ch in queryName) + { + if (ch == '"' || ch == '.' || IsControlCharacter(ch)) + { + throw new ArgumentException("Query name contains invalid characters.", nameof(queryName)); + } + } + } + + private static bool IsControlCharacter(char value) + { + return (value >= '\u0000' && value <= '\u001F') + || (value >= '\u007F' && value <= '\u009F'); + } +} diff --git a/dotnet/src/ConnectedWorkbooks/Internal/TemplateMetadataResolver.cs b/dotnet/src/ConnectedWorkbooks/Internal/TemplateMetadataResolver.cs index 78a39cc..bec5958 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/TemplateMetadataResolver.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/TemplateMetadataResolver.cs @@ -7,13 +7,28 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Represents the worksheet/table components extracted from a workbook template. +/// +/// Path to the worksheet part that should receive data. +/// Path to the table definition part. +/// One-based coordinates describing where the table begins. internal sealed record TemplateMetadata(string WorksheetPath, string TablePath, (int Row, int Column) TableStart); +/// +/// Resolves template metadata (worksheet/table paths and coordinates) for built-in or custom templates. +/// internal static class TemplateMetadataResolver { private static readonly XNamespace RelationshipsNamespace = "http://schemas.openxmlformats.org/package/2006/relationships"; private static readonly XNamespace OfficeRelationshipsNamespace = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + /// + /// Resolves worksheet/table information for the provided template, honoring optional overrides. + /// + /// Workbook archive to inspect. + /// Optional overrides that specify sheet/table names. + /// A instance describing the target sheet/table. public static TemplateMetadata Resolve(ExcelArchive archive, TemplateSettings? templateSettings) { var worksheetPath = ResolveWorksheetPath(archive, templateSettings?.SheetName); diff --git a/dotnet/src/ConnectedWorkbooks/Internal/WorkbookConstants.cs b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookConstants.cs index 40d0112..9a2c519 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/WorkbookConstants.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookConstants.cs @@ -5,6 +5,9 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Central location for common workbook part paths, namespaces, and other constants. +/// internal static class WorkbookConstants { public const string ConnectionsXmlPath = "xl/connections.xml"; @@ -25,6 +28,7 @@ internal static class WorkbookConstants public const string CustomXmlFolder = "customXml"; public const string LabelInfoPath = "docMetadata/LabelInfo.xml"; public const string TablesFolder = "xl/tables/"; + public const string DefaultQueryName = "Query1"; public const string ConnectedWorkbookNamespace = "http://schemas.microsoft.com/ConnectedWorkbook"; public const string DataMashupNamespace = "http://schemas.microsoft.com/DataMashup"; diff --git a/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs index 5530e5c..43b5456 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/WorkbookEditor.cs @@ -7,6 +7,9 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Coordinates the various OpenXML edits required to transform the template into a Connected Workbook. +/// internal sealed class WorkbookEditor { private readonly ExcelArchive _archive; @@ -15,6 +18,12 @@ internal sealed class WorkbookEditor private readonly string _tablePath; private readonly (int Row, int Column) _tableStart; + /// + /// Initializes a new instance bound to the supplied archive and template metadata. + /// + /// Underlying workbook archive to mutate. + /// Optional document metadata to stamp. + /// Pre-resolved worksheet/table information. public WorkbookEditor(ExcelArchive archive, DocumentProperties? documentProperties, TemplateMetadata templateMetadata) { _archive = archive; @@ -24,13 +33,25 @@ public WorkbookEditor(ExcelArchive archive, DocumentProperties? documentProperti _tableStart = templateMetadata.TableStart; } - public void UpdatePowerQueryDocument(string queryName, string mashupDocument) + /// + /// Rewrites the DataMashup payload with the supplied query definition. + /// + /// M script that should populate the template query. + /// Friendly query name to embed. + public void UpdatePowerQueryDocument(string queryBody, string queryName) { var (path, base64) = PqUtilities.GetDataMashup(_archive); + var mashupDocument = PowerQueryGenerator.GenerateSingleQueryMashup(queryName, queryBody); var nextBase64 = MashupDocumentParser.ReplaceSingleQuery(base64, queryName, mashupDocument); PqUtilities.SetDataMashup(_archive, path, nextBase64); } + /// + /// Updates the workbook connection entry to point at the new Power Query and returns its ID. + /// + /// Friendly name shown in Excel's connection UI. + /// Whether Excel should refresh on open. + /// The connection ID used by the workbook. public string UpdateConnections(string queryName, bool refreshOnOpen) { var xml = _archive.ReadText(WorkbookConstants.ConnectionsXmlPath); @@ -49,6 +70,11 @@ public string UpdateConnections(string queryName, bool refreshOnOpen) return connection.Attribute("id")?.Value ?? "1"; } + /// + /// Ensures the query name exists in sharedStrings.xml and returns its index. + /// + /// Name to persist. + /// The shared string index (1-based) that contains the query name. public int UpdateSharedStrings(string queryName) { var xml = _archive.ReadText(WorkbookConstants.SharedStringsXmlPath); @@ -73,6 +99,10 @@ public int UpdateSharedStrings(string queryName) return sharedStringIndex; } + /// + /// Points the worksheet's single cell at the provided shared string index. + /// + /// Index returned by . public void UpdateWorksheet(int sharedStringIndex) { var xml = _archive.ReadText(_worksheetPath); @@ -88,6 +118,11 @@ public void UpdateWorksheet(int sharedStringIndex) _archive.WriteText(_worksheetPath, doc.ToString(SaveOptions.DisableFormatting)); } + /// + /// Updates the legacy query table XML so it references the new connection and refresh behavior. + /// + /// Connection ID from . + /// Whether Excel should refresh on open. public void UpdateQueryTable(string connectionId, bool refreshOnOpen) { var xml = _archive.ReadText(WorkbookConstants.QueryTablePath); @@ -98,6 +133,10 @@ public void UpdateQueryTable(string connectionId, bool refreshOnOpen) _archive.WriteText(WorkbookConstants.QueryTablePath, doc.ToString(SaveOptions.DisableFormatting)); } + /// + /// Writes the supplied tabular data into the worksheet/table/query table definitions. + /// + /// Normalized table data. public void UpdateTableData(TableData tableData) { if (tableData.ColumnNames.Count == 0) @@ -111,9 +150,12 @@ public void UpdateTableData(TableData tableData) UpdateQueryTableColumns(tableData); } + /// + /// Stamps the workbook's core properties file with timestamps and optional metadata. + /// public void UpdateDocumentProperties() { - var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); + var now = FormatW3CDate(DateTime.UtcNow); var xml = _archive.ReadText(WorkbookConstants.DocPropsCoreXmlPath); var doc = XDocument.Parse(xml, LoadOptions.PreserveWhitespace); var cp = (XNamespace)"http://schemas.openxmlformats.org/package/2006/metadata/core-properties"; @@ -129,7 +171,7 @@ public void UpdateDocumentProperties() SetElement(doc, cp + "coreProperties", dc + "subject", _documentProperties.Subject); SetElement(doc, cp + "coreProperties", dc + "creator", _documentProperties.CreatedBy); SetElement(doc, cp + "coreProperties", dc + "description", _documentProperties.Description); - SetElement(doc, cp + "coreProperties", (XNamespace)"http://schemas.openxmlformats.org/package/2006/metadata/core-properties" + "keywords", _documentProperties.Keywords); + SetElement(doc, cp + "coreProperties", cp + "keywords", _documentProperties.Keywords); SetElement(doc, cp + "coreProperties", cp + "lastModifiedBy", _documentProperties.LastModifiedBy); SetElement(doc, cp + "coreProperties", cp + "category", _documentProperties.Category); SetElement(doc, cp + "coreProperties", cp + "revision", _documentProperties.Revision); @@ -138,6 +180,11 @@ public void UpdateDocumentProperties() _archive.WriteText(WorkbookConstants.DocPropsCoreXmlPath, doc.ToString(SaveOptions.DisableFormatting)); } + private static string FormatW3CDate(DateTime utcDateTime) + { + return utcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'Z'", CultureInfo.InvariantCulture); + } + private void UpdateSheetData(TableData tableData) { var xml = _archive.ReadText(_worksheetPath); @@ -209,6 +256,8 @@ private void UpdateTableDefinition(TableData tableData) var column = new XElement(ns + XmlNames.Elements.TableColumn); column.SetAttributeValue(XmlNames.Attributes.Id, index + 1); column.SetAttributeValue(XmlNames.Attributes.Name, tableData.ColumnNames[index]); + column.SetAttributeValue(XmlNames.Attributes.UniqueName, (index + 1).ToString(CultureInfo.InvariantCulture)); + column.SetAttributeValue(XmlNames.Attributes.QueryTableFieldId, index + 1); tableColumns.Add(column); } @@ -232,7 +281,8 @@ private void UpdateWorkbookDefinedName(TableData tableData) } var range = CellReferenceHelper.BuildReference(_tableStart, tableData.ColumnNames.Count, tableData.Rows.Count + 1); - definedName.Value = CellReferenceHelper.WithAbsolute(range); + var sheetPrefix = ExtractDefinedNameSheetPrefix(definedName.Value); + definedName.Value = CellReferenceHelper.WithAbsolute(range, sheetPrefix); _archive.WriteText(WorkbookConstants.WorkbookXmlPath, doc.ToString(SaveOptions.DisableFormatting)); } @@ -334,5 +384,21 @@ private static void SetElement(XDocument doc, XName parentName, XName elementNam element.Value = value; } + + private static string ExtractDefinedNameSheetPrefix(string? definedNameValue) + { + if (string.IsNullOrWhiteSpace(definedNameValue)) + { + return string.Empty; + } + + var separatorIndex = definedNameValue.IndexOf('!'); + if (separatorIndex < 0) + { + return string.Empty; + } + + return definedNameValue[..separatorIndex]; + } } diff --git a/dotnet/src/ConnectedWorkbooks/Internal/XmlNames.cs b/dotnet/src/ConnectedWorkbooks/Internal/XmlNames.cs index f8ab742..e44fd2b 100644 --- a/dotnet/src/ConnectedWorkbooks/Internal/XmlNames.cs +++ b/dotnet/src/ConnectedWorkbooks/Internal/XmlNames.cs @@ -3,8 +3,14 @@ namespace Microsoft.ConnectedWorkbooks.Internal; +/// +/// Centralizes frequently used OpenXML element/attribute names. +/// internal static class XmlNames { + /// + /// Element names used throughout the workbook package. + /// internal static class Elements { public const string SharedStringTable = "sst"; @@ -36,6 +42,9 @@ internal static class Elements public const string Selection = "selection"; } + /// + /// Attribute names used throughout the workbook package. + /// internal static class Attributes { public const string Count = "count"; diff --git a/dotnet/src/ConnectedWorkbooks/Models/DocumentProperties.cs b/dotnet/src/ConnectedWorkbooks/Models/DocumentProperties.cs index ffcb9e6..1449347 100644 --- a/dotnet/src/ConnectedWorkbooks/Models/DocumentProperties.cs +++ b/dotnet/src/ConnectedWorkbooks/Models/DocumentProperties.cs @@ -8,13 +8,44 @@ namespace Microsoft.ConnectedWorkbooks.Models; /// public sealed record DocumentProperties { + /// + /// Value written to dc:title. + /// public string? Title { get; init; } + + /// + /// Value written to dc:subject. + /// public string? Subject { get; init; } + + /// + /// Optional keywords associated with the document. + /// public string? Keywords { get; init; } + + /// + /// Author recorded in dc:creator. + /// public string? CreatedBy { get; init; } + + /// + /// Long-form description of the workbook. + /// public string? Description { get; init; } + + /// + /// Value written to cp:lastModifiedBy. + /// public string? LastModifiedBy { get; init; } + + /// + /// Optional category classification. + /// public string? Category { get; init; } + + /// + /// Revision string assigned to cp:revision. + /// public string? Revision { get; init; } } diff --git a/dotnet/src/ConnectedWorkbooks/Models/Grid.cs b/dotnet/src/ConnectedWorkbooks/Models/Grid.cs index 9660a77..67ce52b 100644 --- a/dotnet/src/ConnectedWorkbooks/Models/Grid.cs +++ b/dotnet/src/ConnectedWorkbooks/Models/Grid.cs @@ -6,5 +6,7 @@ namespace Microsoft.ConnectedWorkbooks.Models; /// /// Simple 2D grid abstraction used for data ingestion. /// +/// Rows and columns that represent the dataset. +/// Optional configuration that influences how the grid is interpreted. public sealed record Grid(IReadOnlyList> Data, GridConfig? Config = null); diff --git a/dotnet/src/ConnectedWorkbooks/Models/QueryInfo.cs b/dotnet/src/ConnectedWorkbooks/Models/QueryInfo.cs index 151b41b..c60e27a 100644 --- a/dotnet/src/ConnectedWorkbooks/Models/QueryInfo.cs +++ b/dotnet/src/ConnectedWorkbooks/Models/QueryInfo.cs @@ -8,6 +8,12 @@ namespace Microsoft.ConnectedWorkbooks.Models; /// public sealed record QueryInfo { + /// + /// Creates a new query description. + /// + /// The Power Query (M) text that defines the query. + /// Optional friendly name; defaults to Query1 if omitted. + /// Whether Excel should refresh the query automatically when the workbook opens. public QueryInfo(string queryMashup, string? queryName = null, bool refreshOnOpen = true) { QueryMashup = string.IsNullOrWhiteSpace(queryMashup) @@ -18,10 +24,19 @@ public QueryInfo(string queryMashup, string? queryName = null, bool refreshOnOpe RefreshOnOpen = refreshOnOpen; } + /// + /// Gets the Power Query (M) script. + /// public string QueryMashup { get; } + /// + /// Gets the friendly query name (or null to fall back to the default). + /// public string? QueryName { get; } + /// + /// Gets a value indicating whether the query should refresh when the workbook opens. + /// public bool RefreshOnOpen { get; } } diff --git a/dotnet/src/ConnectedWorkbooks/Models/TableData.cs b/dotnet/src/ConnectedWorkbooks/Models/TableData.cs index 9652396..b0f0563 100644 --- a/dotnet/src/ConnectedWorkbooks/Models/TableData.cs +++ b/dotnet/src/ConnectedWorkbooks/Models/TableData.cs @@ -6,6 +6,8 @@ namespace Microsoft.ConnectedWorkbooks.Models; /// /// Normalized representation of tabular data that can be written into the workbook. /// +/// Column headers that appear in the table. +/// Table rows aligned with . public sealed record TableData(IReadOnlyList ColumnNames, IReadOnlyList> Rows) { } diff --git a/dotnet/src/ConnectedWorkbooks/Models/TemplateSettings.cs b/dotnet/src/ConnectedWorkbooks/Models/TemplateSettings.cs index aa9a32b..d2fb26b 100644 --- a/dotnet/src/ConnectedWorkbooks/Models/TemplateSettings.cs +++ b/dotnet/src/ConnectedWorkbooks/Models/TemplateSettings.cs @@ -8,7 +8,14 @@ namespace Microsoft.ConnectedWorkbooks.Models; /// public sealed record TemplateSettings { + /// + /// Optional table name override inside the custom template. + /// public string? TableName { get; init; } + + /// + /// Optional worksheet name override inside the custom template. + /// public string? SheetName { get; init; } } diff --git a/dotnet/src/ConnectedWorkbooks/WorkbookManager.cs b/dotnet/src/ConnectedWorkbooks/WorkbookManager.cs index afd4f5d..fba143e 100644 --- a/dotnet/src/ConnectedWorkbooks/WorkbookManager.cs +++ b/dotnet/src/ConnectedWorkbooks/WorkbookManager.cs @@ -11,30 +11,32 @@ namespace Microsoft.ConnectedWorkbooks; /// public sealed class WorkbookManager { - public async Task GenerateSingleQueryWorkbookAsync( + /// + /// Generates a workbook that contains a single Power Query backed by optional initial grid data. + /// + /// Information about the query to embed. + /// Optional grid whose data should seed the query table. + /// Optional template and document configuration overrides. + /// The generated workbook bytes. + public byte[] GenerateSingleQueryWorkbook( QueryInfo query, Grid? initialDataGrid = null, - FileConfiguration? fileConfiguration = null, - CancellationToken cancellationToken = default) + FileConfiguration? fileConfiguration = null) { ArgumentNullException.ThrowIfNull(query); - var queryName = string.IsNullOrWhiteSpace(query.QueryName) - ? "Query1" - : query.QueryName!; - - PqUtilities.ValidateQueryName(queryName); var templateBytes = fileConfiguration?.TemplateBytes - ?? await EmbeddedTemplateLoader.LoadSimpleQueryTemplateAsync(cancellationToken).ConfigureAwait(false); + ?? EmbeddedTemplateLoader.LoadSimpleQueryTemplate(); var tableData = initialDataGrid is null ? null : GridParser.Parse(initialDataGrid); + var effectiveQueryName = QueryNameValidator.Resolve(query.QueryName); using var archive = ExcelArchive.Load(templateBytes); var templateMetadata = TemplateMetadataResolver.Resolve(archive, fileConfiguration?.TemplateSettings); var editor = new WorkbookEditor(archive, fileConfiguration?.DocumentProperties, templateMetadata); - var mashup = PowerQueryGenerator.GenerateSingleQueryMashup(queryName, query.QueryMashup); - editor.UpdatePowerQueryDocument(queryName, mashup); - var connectionId = editor.UpdateConnections(queryName, query.RefreshOnOpen); - var sharedStringIndex = editor.UpdateSharedStrings(queryName); + editor.UpdatePowerQueryDocument(query.QueryMashup, effectiveQueryName); + + var connectionId = editor.UpdateConnections(effectiveQueryName, query.RefreshOnOpen); + var sharedStringIndex = editor.UpdateSharedStrings(effectiveQueryName); editor.UpdateWorksheet(sharedStringIndex); editor.UpdateQueryTable(connectionId, query.RefreshOnOpen); if (tableData is not null) @@ -43,18 +45,23 @@ public async Task GenerateSingleQueryWorkbookAsync( } editor.UpdateDocumentProperties(); - return await Task.FromResult(archive.ToArray()); + return archive.ToArray(); } - public async Task GenerateTableWorkbookFromGridAsync( + /// + /// Generates a workbook that contains only a table populated from the supplied grid. + /// + /// The source grid data. + /// Optional template/document overrides. + /// The generated workbook bytes. + public byte[] GenerateTableWorkbookFromGrid( Grid grid, - FileConfiguration? fileConfiguration = null, - CancellationToken cancellationToken = default) + FileConfiguration? fileConfiguration = null) { ArgumentNullException.ThrowIfNull(grid); var templateBytes = fileConfiguration?.TemplateBytes - ?? await EmbeddedTemplateLoader.LoadBlankTableTemplateAsync(cancellationToken).ConfigureAwait(false); + ?? EmbeddedTemplateLoader.LoadBlankTableTemplate(); var tableData = GridParser.Parse(grid); using var archive = ExcelArchive.Load(templateBytes); @@ -63,6 +70,6 @@ public async Task GenerateTableWorkbookFromGridAsync( editor.UpdateTableData(tableData); editor.UpdateDocumentProperties(); - return await Task.FromResult(archive.ToArray()); + return archive.ToArray(); } } diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/PqUtilitiesTests.cs b/dotnet/tests/ConnectedWorkbooks.Tests/PqUtilitiesTests.cs index a69ff54..6bd3579 100644 --- a/dotnet/tests/ConnectedWorkbooks.Tests/PqUtilitiesTests.cs +++ b/dotnet/tests/ConnectedWorkbooks.Tests/PqUtilitiesTests.cs @@ -11,26 +11,26 @@ namespace ConnectedWorkbooks.Tests; public sealed class PqUtilitiesTests { [TestMethod] - public async Task GetDataMashupHandlesUtf16LittleEndianBom() + public void GetDataMashupHandlesUtf16LittleEndianBom() { - await AssertDataMashupRoundtripAsync(Encoding.Unicode); + AssertDataMashupRoundtrip(Encoding.Unicode); } [TestMethod] - public async Task GetDataMashupHandlesUtf16BigEndianBom() + public void GetDataMashupHandlesUtf16BigEndianBom() { - await AssertDataMashupRoundtripAsync(Encoding.BigEndianUnicode); + AssertDataMashupRoundtrip(Encoding.BigEndianUnicode); } [TestMethod] - public async Task GetDataMashupHandlesUtf8Bom() + public void GetDataMashupHandlesUtf8Bom() { - await AssertDataMashupRoundtripAsync(new UTF8Encoding(encoderShouldEmitUTF8Identifier: true)); + AssertDataMashupRoundtrip(new UTF8Encoding(encoderShouldEmitUTF8Identifier: true)); } - private static async Task AssertDataMashupRoundtripAsync(Encoding encoding) + private static void AssertDataMashupRoundtrip(Encoding encoding) { - var template = await EmbeddedTemplateLoader.LoadSimpleQueryTemplateAsync(); + var template = EmbeddedTemplateLoader.LoadSimpleQueryTemplate(); using var archive = ExcelArchive.Load(template); var base64 = Convert.ToBase64String(Guid.NewGuid().ToByteArray()); WriteDataMashup(archive, base64, encoding); diff --git a/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs b/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs index 467a781..64c7e94 100644 --- a/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs +++ b/dotnet/tests/ConnectedWorkbooks.Tests/WorkbookManagerTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.IO.Compression; using System.Linq; using System.Text; @@ -24,7 +25,7 @@ public sealed class WorkbookManagerTests private const string CustomTableRange = "C3:D5"; [TestMethod] - public async Task GeneratesWorkbookWithMashupAndTable() + public void GeneratesWorkbookWithMashupAndTable() { var queryBody = @"let Source = Kusto.Contents(""https://help.kusto.windows.net"", ""Samples"", ""StormEvents"") @@ -38,14 +39,14 @@ public async Task GeneratesWorkbookWithMashupAndTable() new List { "London", 12 } }, new GridConfig { PromoteHeaders = true, AdjustColumnNames = true }); - var bytes = await _manager.GenerateSingleQueryWorkbookAsync(query, grid); + var bytes = _manager.GenerateSingleQueryWorkbook(query, grid); Assert.IsTrue(bytes.Length > 0, "The generated workbook should not be empty."); using var archiveStream = new MemoryStream(bytes); using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read); var connections = ReadEntry(archive, "xl/connections.xml"); - StringAssert.Contains(connections, "DataAgentQuery"); + StringAssert.Contains(connections, "Query - DataAgentQuery"); var sharedStrings = ReadEntry(archive, "xl/sharedStrings.xml"); StringAssert.Contains(sharedStrings, "DataAgentQuery"); @@ -62,11 +63,12 @@ public async Task GeneratesWorkbookWithMashupAndTable() StringAssert.Contains(mashupXml, "DataMashup"); var root = XDocument.Parse(mashupXml).Root ?? throw new AssertFailedException("DataMashup XML root was missing."); var sectionContent = ExtractSection1m(root.Value.Trim()); - StringAssert.Contains(sectionContent, "DataAgentQuery"); + StringAssert.Contains(sectionContent, "shared #\"DataAgentQuery\""); + StringAssert.Contains(sectionContent, "Kusto.Contents(\"https://help.kusto.windows.net\""); } [TestMethod] - public async Task GeneratesTableWorkbookFromGrid() + public void GeneratesTableWorkbookFromGrid() { var grid = new Grid(new List> { @@ -75,7 +77,7 @@ public async Task GeneratesTableWorkbookFromGrid() new List { "Bananas", 8, 0.99 } }, new GridConfig { PromoteHeaders = true }); - var bytes = await _manager.GenerateTableWorkbookFromGridAsync(grid); + var bytes = _manager.GenerateTableWorkbookFromGrid(grid); Assert.IsTrue(bytes.Length > 0, "The generated workbook should not be empty."); using var archiveStream = new MemoryStream(bytes); @@ -95,13 +97,24 @@ public async Task GeneratesTableWorkbookFromGrid() public void RejectsInvalidQueryName() { var query = new QueryInfo("let Source = 1 in Source", "Invalid.Name", refreshOnOpen: false); - Assert.ThrowsException(() => _manager.GenerateSingleQueryWorkbookAsync(query).GetAwaiter().GetResult()); + Assert.ThrowsException(() => _manager.GenerateSingleQueryWorkbook(query)); } [TestMethod] - public async Task RequiresTemplateSettingsWhenDefaultsMissing() + public void DefaultsQueryNameWhenMissing() { - var template = await CreateCustomTemplateAsync(); + var query = new QueryInfo("let Source = 1 in Source"); + var bytes = _manager.GenerateSingleQueryWorkbook(query); + using var archiveStream = new MemoryStream(bytes); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read); + var connections = ReadEntry(archive, "xl/connections.xml"); + StringAssert.Contains(connections, "Query - Query1"); + } + + [TestMethod] + public void RequiresTemplateSettingsWhenDefaultsMissing() + { + var template = CreateCustomTemplate(); var grid = new Grid(new List> { new List { "Col1", "Col2" }, @@ -110,13 +123,13 @@ public async Task RequiresTemplateSettingsWhenDefaultsMissing() var config = new FileConfiguration { TemplateBytes = template }; - await Assert.ThrowsExceptionAsync(() => _manager.GenerateTableWorkbookFromGridAsync(grid, config)); + Assert.ThrowsException(() => _manager.GenerateTableWorkbookFromGrid(grid, config)); } [TestMethod] - public async Task GeneratesTableWorkbookWithTemplateSettings() + public void GeneratesTableWorkbookWithTemplateSettings() { - var template = await CreateCustomTemplateAsync(); + var template = CreateCustomTemplate(); var grid = new Grid(new List> { new List { "Product", "Qty" }, @@ -134,7 +147,7 @@ public async Task GeneratesTableWorkbookWithTemplateSettings() } }; - var bytes = await _manager.GenerateTableWorkbookFromGridAsync(grid, config); + var bytes = _manager.GenerateTableWorkbookFromGrid(grid, config); using var archiveStream = new MemoryStream(bytes); using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read); @@ -158,9 +171,9 @@ private static string ReadEntry(ZipArchive archive, string path) return reader.ReadToEnd(); } - private static async Task CreateCustomTemplateAsync() + private static byte[] CreateCustomTemplate() { - var template = await EmbeddedTemplateLoader.LoadBlankTableTemplateAsync(); + var template = EmbeddedTemplateLoader.LoadBlankTableTemplate(); using var stream = new MemoryStream(); stream.Write(template, 0, template.Length); diff --git a/typescript/package.json b/typescript/package.json index 2062add..f7ada26 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -19,7 +19,8 @@ "test:clean": "npm run clean && npm run test", "test:node": "tsc && jest --config jest.config.node.js", "test:jsdom": "tsc && jest --config jest.config.jsdom.js", - "test": "npm run test:jsdom && npm run test:node" + "test": "npm run test:jsdom && npm run test:node", + "validate:implementations": "node scripts/validateImplementations.js" }, "repository": { "type": "git", diff --git a/typescript/scripts/generateWeatherSample.js b/typescript/scripts/generateWeatherSample.js new file mode 100644 index 0000000..50a55ba --- /dev/null +++ b/typescript/scripts/generateWeatherSample.js @@ -0,0 +1,44 @@ +const path = require("path"); +const fs = require("fs/promises"); +const { workbookManager } = require("../dist"); + +const mashup = `let + Source = #table( + {"City","TempC"}, + { + {"Seattle", 18}, + {"London", 15}, + {"Sydney", 22} + } + ) +in + Source`; + +const query = { + queryMashup: mashup, + queryName: "WeatherSample", + refreshOnOpen: false, +}; + +const grid = { + data: [ + ["City", "TempC"], + ["Seattle", 0], + ["London", 0], + ["Sydney", 0], + ], + config: { promoteHeaders: true }, +}; + +async function main() { + const blob = await workbookManager.generateSingleQueryWorkbook(query, grid); + const buffer = Buffer.from(await blob.arrayBuffer()); + const outputPath = path.resolve(__dirname, "..", "..", "WeatherSample.ts.xlsx"); + await fs.writeFile(outputPath, buffer); + console.log(`TypeScript workbook generated: ${outputPath}`); +} + +main().catch((error) => { + console.error("Failed to generate TypeScript workbook", error); + process.exitCode = 1; +}); diff --git a/typescript/scripts/inspectMashup.js b/typescript/scripts/inspectMashup.js new file mode 100644 index 0000000..b367dde --- /dev/null +++ b/typescript/scripts/inspectMashup.js @@ -0,0 +1,28 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +const path = require("path"); +const { extractWorkbookDetails } = require("./workbookDetails"); + +async function main() { + const targets = process.argv.slice(2); + if (targets.length === 0) { + console.error("Usage: node inspectMashup.js [workbook...]"); + process.exit(1); + } + + for (const target of targets) { + const fullPath = path.resolve(target); + const details = await extractWorkbookDetails(fullPath); + const sectionPreview = details.sectionContent.split("\n").slice(0, 5).join("\n"); + const metadataPreview = details.metadataXml.slice(0, 400); + console.log(`\n${fullPath}`); + console.log(` Query name : ${details.queryName}`); + console.log(` Metadata : ${details.metadataBytesLength ?? details.metadataXml.length} bytes`); + console.log(" Section1.m preview:\n" + sectionPreview.split("\n").map((line) => ` ${line}`).join("\n")); + console.log(" Metadata preview:\n" + metadataPreview.split("\n").map((line) => ` ${line}`).join("\n")); + } +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/typescript/scripts/validateImplementations.js b/typescript/scripts/validateImplementations.js new file mode 100644 index 0000000..ed299bd --- /dev/null +++ b/typescript/scripts/validateImplementations.js @@ -0,0 +1,81 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +const path = require("path"); +const { execSync } = require("child_process"); +const { extractWorkbookDetails } = require("./workbookDetails"); + +const repoRoot = path.resolve(__dirname, "..", ".."); +const tsRoot = path.resolve(__dirname, ".."); + +function run(command, cwd) { + execSync(command, { cwd, stdio: "inherit" }); +} + +async function ensureSamples() { + run("dotnet run --project dotnet/sample/ConnectedWorkbooks.Sample.csproj", repoRoot); + run("npm run build", tsRoot); + run("node scripts/generateWeatherSample.js", tsRoot); +} + +function compare(detailsA, detailsB) { + const mismatches = []; + const checks = [ + ["Query name", detailsA.queryName, detailsB.queryName], + ["Connection name", detailsA.connection.name, detailsB.connection.name], + ["Connection description", detailsA.connection.description, detailsB.connection.description], + ["Connection location", detailsA.connection.location, detailsB.connection.location], + ["Connection command", detailsA.connection.command, detailsB.connection.command], + ["Refresh flag", detailsA.connection.refreshOnLoad, detailsB.connection.refreshOnLoad], + ]; + + for (const [label, left, right] of checks) { + if (left !== right) { + mismatches.push(`${label} mismatch: '${left}' vs '${right}'`); + } + } + + const leftPaths = detailsA.metadataItemPaths.filter((entry) => entry.includes("Section1/")); + const rightPaths = detailsB.metadataItemPaths.filter((entry) => entry.includes("Section1/")); + if (JSON.stringify(leftPaths) !== JSON.stringify(rightPaths)) { + mismatches.push("Metadata ItemPath entries differ"); + } + + if (!detailsA.sharedStrings.includes(detailsA.queryName)) { + mismatches.push(".NET sharedStrings missing query name"); + } + + if (!detailsB.sharedStrings.includes(detailsB.queryName)) { + mismatches.push("TypeScript sharedStrings missing query name"); + } + + if (mismatches.length > 0) { + const error = mismatches.join("\n - "); + throw new Error(`Workbooks are not aligned:\n - ${error}`); + } +} + +async function main() { + const args = process.argv.slice(2); + let dotnetWorkbook; + let tsWorkbook; + + if (args.length === 2) { + [dotnetWorkbook, tsWorkbook] = args.map((arg) => path.resolve(arg)); + } else if (args.length === 0) { + await ensureSamples(); + dotnetWorkbook = path.resolve(repoRoot, "dotnet", "WeatherSample.xlsx"); + tsWorkbook = path.resolve(repoRoot, "WeatherSample.ts.xlsx"); + } else { + console.error("Usage: node scripts/validateImplementations.js [dotnetWorkbook tsWorkbook]"); + process.exit(1); + } + + const dotnetDetails = await extractWorkbookDetails(dotnetWorkbook); + const tsDetails = await extractWorkbookDetails(tsWorkbook); + compare(dotnetDetails, tsDetails); + console.log("\nβœ… Validation succeeded. The .NET and TypeScript outputs match for the inspected fields."); +} + +main().catch((error) => { + console.error(error.message || error); + process.exit(1); +}); diff --git a/typescript/scripts/workbookDetails.js b/typescript/scripts/workbookDetails.js new file mode 100644 index 0000000..fea203b --- /dev/null +++ b/typescript/scripts/workbookDetails.js @@ -0,0 +1,159 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +const fs = require("fs/promises"); +const path = require("path"); +const JSZip = require("jszip"); +const { DOMParser } = require("@xmldom/xmldom"); + +const DATA_MASHUP_REGEX = /]*>([\s\S]+?)<\/DataMashup>/i; + +function decodeXmlBuffer(buffer) { + if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) { + return buffer.toString("utf16le").replace(/^\ufeff/, ""); + } + + return buffer.toString("utf8").replace(/^\ufeff/, ""); +} + +async function loadWorkbookZip(workbookPath) { + const absolutePath = path.resolve(workbookPath); + const buffer = await fs.readFile(absolutePath); + return JSZip.loadAsync(buffer); +} + +function extractQueryName(sectionContent) { + const normalized = sectionContent.replace(/\r/g, ""); + const match = normalized.match(/shared\s+(?:#")?([^"\n]+)"?\s*=/i); + return match ? match[1].trim() : ""; +} + +async function readMashup(zip) { + const entry = zip.file("customXml/item1.xml"); + if (!entry) { + throw new Error("customXml/item1.xml not found"); + } + + const xmlBuffer = await entry.async("nodebuffer"); + const mashupXml = decodeXmlBuffer(xmlBuffer); + const match = mashupXml.match(DATA_MASHUP_REGEX); + if (!match) { + throw new Error("DataMashup payload missing"); + } + + const payloadBuffer = Buffer.from(match[1].trim(), "base64"); + let offset = 0; + const readInt32 = () => { + const value = payloadBuffer.readInt32LE(offset); + offset += 4; + return value; + }; + + offset += 4; // version + const packageSize = readInt32(); + const packageBytes = payloadBuffer.subarray(offset, offset + packageSize); + offset += packageSize; + const permissionsSize = readInt32(); + offset += permissionsSize; + const metadataSize = readInt32(); + const metadataBytes = payloadBuffer.subarray(offset, offset + metadataSize); + + const packageZip = await JSZip.loadAsync(packageBytes); + const sectionEntry = packageZip.file("Formulas/Section1.m"); + if (!sectionEntry) { + throw new Error("Formulas/Section1.m missing"); + } + + const sectionContent = (await sectionEntry.async("text")).trim(); + const queryName = extractQueryName(sectionContent); + + let metadataOffset = 4; // metadata version + const metadataXmlLength = metadataBytes.readInt32LE(metadataOffset); + metadataOffset += 4; + const metadataXml = metadataBytes.subarray(metadataOffset, metadataOffset + metadataXmlLength).toString("utf8"); + + return { + queryName, + sectionContent, + metadataBytesLength: metadataBytes.length, + metadataXml, + }; +} + +function parseSharedStrings(xml) { + if (!xml) { + return []; + } + + const doc = new DOMParser().parseFromString(xml, "text/xml"); + const nodes = doc.getElementsByTagName("t"); + const values = []; + for (let i = 0; i < nodes.length; i++) { + values.push((nodes[i].textContent || "").trim()); + } + + return values; +} + +function parseMetadataPaths(metadataXml) { + const doc = new DOMParser().parseFromString(metadataXml, "text/xml"); + const nodes = doc.getElementsByTagName("ItemPath"); + const paths = []; + for (let i = 0; i < nodes.length; i++) { + paths.push(nodes[i].textContent || ""); + } + + return paths; +} + +async function extractWorkbookDetails(workbookPath) { + const zip = await loadWorkbookZip(workbookPath); + const mashup = await readMashup(zip); + + const connectionsEntry = zip.file("xl/connections.xml"); + if (!connectionsEntry) { + throw new Error("xl/connections.xml not found"); + } + + const connectionsXml = await connectionsEntry.async("text"); + const connectionDoc = new DOMParser().parseFromString(connectionsXml, "text/xml"); + const connectionNode = connectionDoc.getElementsByTagName("connection")[0]; + const dbPrNode = connectionDoc.getElementsByTagName("dbPr")[0]; + + const connection = { + id: connectionNode?.getAttribute("id") || "", + name: connectionNode?.getAttribute("name") || "", + description: connectionNode?.getAttribute("description") || "", + refreshOnLoad: dbPrNode?.getAttribute("refreshOnLoad") || "", + location: dbPrNode?.getAttribute("connection") || "", + command: dbPrNode?.getAttribute("command") || "", + }; + + const sharedStringsEntry = zip.file("xl/sharedStrings.xml"); + const sharedStringsXml = sharedStringsEntry ? await sharedStringsEntry.async("text") : ""; + + return { + workbookPath: path.resolve(workbookPath), + queryName: mashup.queryName, + metadataXml: mashup.metadataXml, + metadataBytesLength: mashup.metadataBytesLength, + sectionContent: mashup.sectionContent, + metadataItemPaths: parseMetadataPaths(mashup.metadataXml), + connection, + sharedStrings: parseSharedStrings(sharedStringsXml), + }; +} + +async function extractMashupInfo(workbookPath) { + const details = await extractWorkbookDetails(workbookPath); + return { + workbookPath: details.workbookPath, + queryName: details.queryName, + sectionContent: details.sectionContent, + metadataBytesLength: details.metadataBytesLength, + metadataXml: details.metadataXml, + }; +} + +module.exports = { + extractMashupInfo, + extractWorkbookDetails, +}; From c55250a37c9b3531a1f302186d78d2df3204685f Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Wed, 26 Nov 2025 16:02:01 +0100 Subject: [PATCH 4/5] update docs and pipeline --- .gitignore | 6 +- README.md | 761 ++++++++++++++++++ azure-pipelines-1.yml | 46 +- azure-pipelines.yml | 18 +- .../ConnectedWorkbooks.csproj | 20 + typescript/README.md | 442 +++------- typescript/scripts/inspectMashup.js | 102 ++- typescript/scripts/validateImplementations.js | 44 +- 8 files changed, 1025 insertions(+), 414 deletions(-) create mode 100644 README.md diff --git a/.gitignore b/.gitignore index 150df22..11ccbe1 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,8 @@ dotnet/**/obj/ # Python artifacts python/.venv/ -python/**/__pycache__/ \ No newline at end of file +python/**/__pycache__/ + +# Test files +**/WeatherSample.xlsx +**/WeatherSample.ts.xlsx \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..371b0cd --- /dev/null +++ b/README.md @@ -0,0 +1,761 @@ +
+ +# Open In Excel + +[![TypeScript](https://img.shields.io/badge/TypeScript-007ACC?style=flat&logo=typescript&logoColor=white)](https://www.typescriptlang.org/) +[![License](https://img.shields.io/github/license/microsoft/connected-workbooks)](https://github.com/microsoft/connected-workbooks/blob/master/LICENSE) +[![npm version](https://img.shields.io/npm/v/@microsoft/connected-workbooks)](https://www.npmjs.com/package/@microsoft/connected-workbooks) +[![Build Status](https://img.shields.io/github/workflow/status/microsoft/connected-workbooks/CI)](https://github.com/microsoft/connected-workbooks/actions) + +**Open your data directly in Excel for the Web with zero installation** - A JavaScript library that converts web tables and data into interactive Excel workbooks with Power Query integration and custom branded templates + +
+Connected Workbooks Demo +
+ + +
+ +--- + +## ✨ Key Features & Benefits + +Transform your web applications with enterprise-grade Excel integration that goes far beyond simple CSV exports. + +### 🎯 **Interactive Excel Workbooks, Not Static Files** +Convert raw data or HTML tables arrays to Excel tables while preserving data types, ensuring your data maintains its structure and formatting. instead of basic CSV exports that lose all structure and functionality. + +### 🌐 **Zero-Installation Excel Experience** +Launch workbooks directly in Excel for the Web through any browser without requiring Excel desktop installation, making your data accessible to any user anywhere. No installation required, works on any device. + +### 🎨 **Corporate Branding & Custom Dashboards** +Inject your data into pre-built Excel templates containing your company branding, PivotTables, charts, and business logic while preserving all formatting and calculations. Use your own branded Excel templates with PivotTables and charts to maintain corporate identity and pre-built analytics. + +### πŸ”„ **Live Data Connections with Power Query** +Create workbooks that automatically refresh from your web APIs, databases, or data sources using Microsoft's Power Query technology, eliminating manual data updates. Create workbooks that refresh data on-demand using Power Query for real-time data updates and automated reporting. + +### βš™οΈ **Advanced Configuration** +Full control over document properties including title and description for professional document management, allowing you to customize metadata and maintain enterprise standards. + +--- + +## 🏒 Where is this library used? + +Open In Excel powers data export functionality across Microsoft's enterprise platforms: + +
+ +|Azure Data Explorer|Log Analytics|Datamart|Viva Sales| +|:---:|:---:|:---:|:---:| +|**Azure Data Explorer**|**Log Analytics**|**Datamart**|**Viva Sales**| + +
+ + + +--- + +## πŸš€ Quick Start + +### Installation + +```bash +npm install @microsoft/connected-workbooks +``` + +#### .NET Consumers + +```bash +dotnet add package Microsoft.ConnectedWorkbooks --prerelease +``` + +```csharp +using System.Collections.Generic; +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); +var grid = new Grid(new List> +{ + new List { "Product", "Revenue" }, + new List { "Surface Laptop", 1299.99 }, + new List { "Office 365", 99.99 } +}); + +var workbook = manager.GenerateTableWorkbookFromGrid(grid); +await File.WriteAllBytesAsync("Workbook.xlsx", workbook); +``` + +--- + +## πŸ’‘ Usage Examples + +Each scenario lists both TypeScript and .NET code. GitHub Markdown does not support interactive tabs, so we use collapsible `
` sectionsβ€”expand the platform you care about and keep scrolling. + +### πŸ“‹ **HTML Table Export** + +Perfect for quick data exports from existing web tables. + +
+TypeScript + +```typescript +import { workbookManager } from '@microsoft/connected-workbooks'; + +// One line of code to convert any table +const blob = await workbookManager.generateTableWorkbookFromHtml( + document.querySelector('table') as HTMLTableElement +); + +// Open in Excel for the Web with editing enabled +workbookManager.openInExcelWeb(blob, "QuickExport.xlsx", true); +``` + +
+ +
+.NET + +```csharp +using System.Collections.Generic; +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); +var grid = new Grid(new List> +{ + new List { "Product", "Revenue", "InStock", "Category" }, + new List { "Surface Laptop", 1299.99, true, "Hardware" }, + new List { "Azure Credits", 500.00, false, "Cloud" } +}); + +var workbook = manager.GenerateTableWorkbookFromGrid(grid); +await File.WriteAllBytesAsync("QuickExport.xlsx", workbook); +``` + +
+ +### πŸ“Š **Smart Data Formatting** + +Transform raw data arrays into professionally formatted Excel tables. + +
+TypeScript + +```typescript +import { workbookManager } from '@microsoft/connected-workbooks'; + +const salesData = { + config: { + promoteHeaders: true, // First row becomes headers + adjustColumnNames: true // Clean up column names + }, + data: [ + ["Product", "Revenue", "InStock", "Category", "LastUpdated"], + ["Surface Laptop", 1299.99, true, "Hardware", "2024-10-26"], + ["Office 365", 99.99, true, "Software", "2024-10-26"], + ["Azure Credits", 500.00, false, "Cloud", "2024-10-25"], + ["Teams Premium", 149.99, true, "Software", "2024-10-24"] + ] +}; + +const blob = await workbookManager.generateTableWorkbookFromGrid(salesData); +workbookManager.openInExcelWeb(blob, "SalesReport.xlsx", true); +``` + +
+ +
+.NET + +```csharp +using System.Collections.Generic; +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); +var grid = new Grid( + new List> + { + new List { "Product", "Revenue", "InStock", "Category" }, + new List { "Surface Laptop", 1299.99, true, "Hardware" }, + new List { "Office 365", 99.99, true, "Software" }, + new List { "Azure Credits", 500.00, false, "Cloud" } + }, + new GridConfig + { + PromoteHeaders = true, + AdjustColumnNames = true + }); + +var workbook = manager.GenerateTableWorkbookFromGrid(grid); +await File.WriteAllBytesAsync("SalesReport.xlsx", workbook); +``` + +
+ +
+Smart Formatted Excel Table +
+ + + +### 🎨 **Custom Branded Templates** + +Transform your data using pre-built Excel templates with your corporate branding. + + + +**Steps:** + +1. **Prepare Your Template File** + + Open Excel and create (or open) your branded file. +2. **Pick one sheet that will hold your data.** + + The default "Sheet1"(3) +3. **Inside that sheet, choose were you want your data to be populated(1) and create a table (Insert β†’ Table).** + + The default table name is Table1(2) + The table need to have the same column structure as your incoming data. +4. **Add any charts, formulas, or formatting that reference this table.** + + Example: Pie chart using Gross column(4). +5. **Save the Excel file (e.g., my-template.xlsx).** +6. **Use the saved file as the template for your incoming data** + +The library will then populate the designated table with your data. Any functions, figures, or references linked to this table within the Excel template will automatically reflect the newly exported data. + +
+Custom Branded Excel Dashboard +
+ + + +
+TypeScript + +#### πŸ“ **Loading Template Files** + +```typescript +// Method 1: File upload from user +const templateInput = document.querySelector('#template-upload') as HTMLInputElement; +const templateFile = templateInput.files[0]; + +// Method 2: Fetch from your server +const templateResponse = await fetch('/assets/templates/sales-dashboard.xlsx'); +const templateFile = await templateResponse.blob(); + +// Method 3: Drag and drop +function handleTemplateDrop(event: DragEvent) { + const templateFile = event.dataTransfer.files[0]; + // Use templateFile with the library +} +``` + +#### πŸ“Š **Generate Branded Workbook** + +```typescript +const quarterlyData = { + config: { promoteHeaders: true, adjustColumnNames: true }, + data: [ + ["Region", "Q3_Revenue", "Q4_Revenue", "Growth", "Target_Met"], + ["North America", 2500000, 2750000, "10%", true], + ["Europe", 1800000, 2100000, "17%", true], + ["Asia Pacific", 1200000, 1400000, "17%", true], + ["Latin America", 800000, 950000, "19%", true] + ] +}; + +// Inject data into your branded template +const blob = await workbookManager.generateTableWorkbookFromGrid( + quarterlyData, + undefined, // Use template's existing data structure + { + templateFile: templateFile, + TempleteSettings: { + sheetName: "Dashboard", // Target worksheet + tableName: "QuarterlyData" // Target table name + } + } +); + +// Users get a fully branded report +workbookManager.openInExcelWeb(blob, "Q4_Executive_Dashboard.xlsx", true); +``` + +
+ +
+.NET + +```csharp +using System.Collections.Generic; +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); +var grid = new Grid(new List> +{ + new List { "Region", "Q3_Revenue", "Q4_Revenue", "Growth", "Target_Met" }, + new List { "North America", 2_500_000, 2_750_000, "10%", true }, + new List { "Europe", 1_800_000, 2_100_000, "17%", true } +}); + +var templateBytes = await File.ReadAllBytesAsync("Templates/sales-dashboard.xlsx"); +var configuration = new FileConfiguration +{ + TemplateBytes = templateBytes, + TemplateSettings = new TemplateSettings + { + SheetName = "Dashboard", + TableName = "QuarterlyData" + } +}; + +var workbook = manager.GenerateTableWorkbookFromGrid(grid, configuration); +await File.WriteAllBytesAsync("Q4_Executive_Dashboard.xlsx", workbook); +``` + +
+ +
+Custom Branded Excel Dashboard +
+ +> πŸ’‘ **Template Requirements**: Include a query named **"Query1"** connected to a **Table**. + +### πŸ”„ **Live Data Connections with Power Query** + +Create workbooks that automatically refresh from your data sources. + +
+TypeScript + +```typescript +import { workbookManager } from '@microsoft/connected-workbooks'; + +// Create a workbook that connects to your API +const blob = await workbookManager.generateSingleQueryWorkbook({ + queryMashup: `let + Source = {1..10} + in + Source`, + refreshOnOpen: true +}); + +workbookManager.openInExcelWeb(blob, "MyData.xlsx", true); +``` + +
+ +
+.NET + +```csharp +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); +var query = new QueryInfo +{ + QueryMashup = """ +let + Source = {1..10} +in + Source +""", + RefreshOnOpen = true, + QueryName = "Query1" +}; + +var workbook = manager.GenerateSingleQueryWorkbook(query); +await File.WriteAllBytesAsync("MyData.xlsx", workbook); +``` + +
+ +> πŸ“š **Learn Power Query**: New to Power Query? Check out the [official documentation](https://docs.microsoft.com/en-us/power-query/) to unlock the full potential of live data connections. + +
+Live Data Workbook +
+### πŸ“„ **Professional Document Properties** + +Add metadata and professional document properties for enterprise use. + +
+TypeScript + +```typescript +const blob = await workbookManager.generateTableWorkbookFromHtml( + document.querySelector('table') as HTMLTableElement, + { + docProps: { + createdBy: 'John Doe', + lastModifiedBy: 'Jane Doe', + description: 'Sales Report Q4 2024', + title: 'Quarterly Sales Data' + } + } +); + +// Download for offline use +workbookManager.downloadWorkbook(blob, "MyTable.xlsx"); +``` + +
+ +
+.NET + +```csharp +using System.Collections.Generic; +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); +var grid = new Grid(new List> +{ + new List { "Product", "Revenue" }, + new List { "Surface Laptop", 1299.99 } +}); + +var configuration = new FileConfiguration +{ + DocumentProperties = new DocumentProperties + { + CreatedBy = "John Doe", + LastModifiedBy = "Jane Doe", + Description = "Sales Report Q4 2024", + Title = "Quarterly Sales Data" + } +}; + +var workbook = manager.GenerateTableWorkbookFromGrid(grid, configuration); +await File.WriteAllBytesAsync("MyTable.xlsx", workbook); +``` + +
+ +
+Professional Document Properties +
+ +## πŸ“š Complete API Reference + +### Core Functions + +#### πŸ”— `generateSingleQueryWorkbook()` +Create Power Query connected workbooks with live data refresh capabilities. + +
+TypeScript + +```typescript +async function generateSingleQueryWorkbook( + query: QueryInfo, + grid?: Grid, + fileConfigs?: FileConfigs +): Promise +``` + +
+ +
+.NET + +```csharp +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); +var workbook = manager.GenerateSingleQueryWorkbook( + query: new QueryInfo { QueryMashup = "...", RefreshOnOpen = true }, + initialDataGrid: grid, + fileConfiguration: new FileConfiguration()); +``` + +
+ +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `query` | [`QueryInfo`](#queryinfo) | βœ… **Required** | Power Query configuration | +| `grid` | [`Grid`](#grid) | Optional | Pre-populate with data | +| `fileConfigs` | [`FileConfigs`](#fileconfigs) | Optional | Customization options | + +#### πŸ“‹ `generateTableWorkbookFromHtml()` +Convert HTML tables to Excel workbooks instantly. + +
+TypeScript + +```typescript +async function generateTableWorkbookFromHtml( + htmlTable: HTMLTableElement, + fileConfigs?: FileConfigs +): Promise +``` + +
+ +
+.NET + +> The .NET SDK does not parse HTML tables directly. Convert the incoming data to a `Grid` and call `WorkbookManager.GenerateTableWorkbookFromGrid` as shown above. + +```csharp +var workbook = manager.GenerateTableWorkbookFromGrid(grid); +``` + +
+ +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `htmlTable` | `HTMLTableElement` | βœ… **Required** | Source HTML table | +| `fileConfigs` | [`FileConfigs`](#fileconfigs) | Optional | Customization options | + +#### πŸ“Š `generateTableWorkbookFromGrid()` +Transform raw data arrays into formatted Excel tables. + +
+TypeScript + +```typescript +async function generateTableWorkbookFromGrid( + grid: Grid, + fileConfigs?: FileConfigs +): Promise +``` + +
+ +
+.NET + +```csharp +using Microsoft.ConnectedWorkbooks; +using Microsoft.ConnectedWorkbooks.Models; + +var manager = new WorkbookManager(); +var workbook = manager.GenerateTableWorkbookFromGrid(grid, fileConfiguration); +``` + +
+ +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `grid` | [`Grid`](#grid) | βœ… **Required** | Data and configuration | +| `fileConfigs` | [`FileConfigs`](#fileconfigs) | Optional | Customization options | + +#### 🌐 `openInExcelWeb()` +Open workbooks directly in Excel for the Web. + +
+TypeScript + +```typescript +async function openInExcelWeb( + blob: Blob, + filename?: string, + allowTyping?: boolean +): Promise +``` + +
+ +
+.NET + +> Server-side apps typically return the generated bytes as a download instead of opening Excel for the Web directly. + +```csharp +var workbook = manager.GenerateTableWorkbookFromGrid(grid); +return Results.File( + workbook, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "MyData.xlsx"); +``` + +
+ +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `blob` | `Blob` | βœ… **Required** | Generated workbook | +| `filename` | `string` | Optional | Custom filename | +| `allowTyping` | `boolean` | Optional | Enable editing (default: false) | + +#### πŸ’Ύ `downloadWorkbook()` +Trigger browser download of the workbook. + +
+TypeScript + +```typescript +function downloadWorkbook(file: Blob, filename: string): void +``` + +
+ +
+.NET + +```csharp +var workbook = manager.GenerateTableWorkbookFromGrid(grid); +await File.WriteAllBytesAsync("MyWorkbook.xlsx", workbook); +``` + +
+ +#### πŸ”— `getExcelForWebWorkbookUrl()` +Get the Excel for Web URL without opening (useful for custom integrations). + +
+TypeScript + +```typescript +async function getExcelForWebWorkbookUrl( + file: Blob, + filename?: string, + allowTyping?: boolean +): Promise +``` + +
+ +
+.NET + +> The .NET SDK does not open Excel for the Web. Generate the workbook bytes and hand them to your client application (or browser) to handle navigation. + +```csharp +var workbook = manager.GenerateSingleQueryWorkbook(query); +// send workbook to the caller through your preferred channel +``` + +
+ +--- + +## πŸ”§ Type Definitions + +### QueryInfo +Power Query configuration for connected workbooks. + +```typescript +interface QueryInfo { + queryMashup: string; // Power Query M language code + refreshOnOpen: boolean; // Auto-refresh when opened + queryName?: string; // Query identifier (default: "Query1") +} +``` + +### Grid +Data structure for tabular information. + +```typescript +interface Grid { + data: (string | number | boolean)[][]; // Raw data rows + config?: GridConfig; // Processing options +} + +interface GridConfig { + promoteHeaders?: boolean; // Use first row as headers + adjustColumnNames?: boolean; // Fix duplicate/invalid names +} +``` + +### FileConfigs +Advanced customization options. + +```typescript +interface FileConfigs { + templateFile?: File | Buffer; // Custom Excel template + docProps?: DocProps; // Document metadata + hostName?: string; // Creator application name + TempleteSettings?: TempleteSettings; // Template-specific settings +} + +interface TempleteSettings { + tableName?: string; // Target table name in template + sheetName?: string; // Target worksheet name +} +``` + +### DocProps +Document metadata and properties. + +```typescript +interface DocProps { + title?: string; // Document title + subject?: string; // Document subject + keywords?: string; // Search keywords + createdBy?: string; // Author name + description?: string; // Document description + lastModifiedBy?: string; // Last editor + category?: string; // Document category + revision?: string; // Version number +} +``` + +--- + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +### Getting Started +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests for new functionality +5. Submit a pull request + +### Development Setup +```bash +git clone https://github.com/microsoft/connected-workbooks.git +cd connected-workbooks +npm install +npm run build +npm test +``` +--- + +## πŸ“„ License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. + +## πŸ”— Related Resources + +- [πŸ“– Power Query Documentation](https://powerquery.microsoft.com/en-us/) +- [🏒 Excel for Developers](https://docs.microsoft.com/en-us/office/dev/excel/) +- [πŸ”§ Microsoft Graph Excel APIs](https://docs.microsoft.com/en-us/graph/api/resources/excel) + +--- + +## Trademarks + +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft +trademarks or logos is subject to and must follow +[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). +Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. +Any use of third-party trademarks or logos are subject to those third-party's policies. + +--- + +## Keywords + +Power Query, Excel, Office, Workbook, Refresh, Table, xlsx, export, CSV, data export, HTML table, web to Excel, JavaScript Excel, TypeScript Excel, Excel template, PivotTable, connected data, live data, data refresh, Excel for Web, browser Excel, spreadsheet, data visualization, Microsoft Office, Office 365, Excel API, workbook generation, table export, grid export, Excel automation, data processing, business intelligence diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 5fedbe4..77257d4 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -12,35 +12,65 @@ pool: steps: - task: NodeTool@0 inputs: - versionSpec: '10.x' + versionSpec: '18.x' displayName: 'Install Node.js' - script: | + cd typescript npm install npm run build - displayName: 'npm install and build' + npm run test + displayName: 'npm install, build, and test (TypeScript)' - task: Npm@1 inputs: command: 'custom' customCommand: 'pack' + workingDir: '$(Build.SourcesDirectory)/typescript' displayName: 'npm pack library' - task: CopyFiles@2 inputs: - SourceFolder: '$(Build.SourcesDirectory)' + SourceFolder: '$(Build.SourcesDirectory)/typescript' Contents: '*.tgz' - TargetFolder: '$(Build.SourcesDirectory)/out' + TargetFolder: '$(Build.ArtifactStagingDirectory)/typescript' + displayName: 'collect npm package' + +- task: UseDotNet@2 + inputs: + packageType: 'sdk' + version: '8.0.x' + displayName: 'Install .NET SDK' + +- script: | + cd dotnet + dotnet restore ConnectedWorkbooks.sln + dotnet build ConnectedWorkbooks.sln --configuration Release --no-restore + dotnet test ConnectedWorkbooks.sln --configuration Release --no-build + displayName: 'dotnet build and test' + +- task: DotNetCoreCLI@2 + displayName: 'dotnet pack' + inputs: + command: 'pack' + packagesToPack: 'dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj' + configuration: 'Release' + nobuild: true + includesymbols: true + buildProperties: 'ContinuousIntegrationBuild=true' + packDirectory: '$(Build.ArtifactStagingDirectory)/nuget' - task: PublishPipelineArtifact@1 inputs: - targetPath: '$(System.DefaultWorkingDirectory)/out' - artifact: 'connected-workbooks-drop-2' + targetPath: '$(Build.ArtifactStagingDirectory)' + artifact: 'connected-workbooks-drop' publishLocation: 'pipeline' - displayName: 'publish packed library' + displayName: 'publish build artifacts' - task: Npm@1 + displayName: 'publish npm package to internal feed' inputs: command: 'publish' publishRegistry: 'useFeed' - publishFeed: '3046f601-a835-471b-8758-5953e60cb1a1/4aa42cde-db13-4c30-91ab-e4b8bd1d09f8' \ No newline at end of file + publishFeed: '3046f601-a835-471b-8758-5953e60cb1a1/4aa42cde-db13-4c30-91ab-e4b8bd1d09f8' + workingDir: '$(Build.SourcesDirectory)/typescript' \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5fc506b..76860f3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,11 +12,25 @@ pool: steps: - task: NodeTool@0 inputs: - versionSpec: '10.x' + versionSpec: '18.x' displayName: 'Install Node.js' - script: | + cd typescript npm install npm run build npm run test - displayName: 'Build + Test' + displayName: 'Build + Test (TypeScript)' + +- task: UseDotNet@2 + inputs: + packageType: 'sdk' + version: '8.0.x' + displayName: 'Install .NET SDK' + +- script: | + cd dotnet + dotnet restore ConnectedWorkbooks.sln + dotnet build ConnectedWorkbooks.sln --configuration Release --no-restore + dotnet test ConnectedWorkbooks.sln --configuration Release --no-build + displayName: 'Build + Test (.NET)' diff --git a/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj b/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj index 5f6964f..2fbe021 100644 --- a/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj +++ b/dotnet/src/ConnectedWorkbooks/ConnectedWorkbooks.csproj @@ -5,10 +5,29 @@ enable enable true + Microsoft + Microsoft + Microsoft.ConnectedWorkbooks + Connected Workbooks SDK (.NET) + Generate Excel workbooks with Power Query connections, branded templates, and tabular data using .NET. + Excel;Power Query;OpenXML;Workbook + https://github.com/microsoft/connected-workbooks + https://github.com/microsoft/connected-workbooks + git + MIT + false + true + true + snupkg + true + true + $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb + ../README.md + @@ -21,6 +40,7 @@ PackagePath="lib\$(TargetFramework)\" Visible="false" Condition="Exists('$(OutputPath)$(AssemblyName).xml')" /> + diff --git a/typescript/README.md b/typescript/README.md index 6d08b16..c5c97b7 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -1,462 +1,208 @@
-# Open In Excel +# Connected Workbooks (TypeScript) [![TypeScript](https://img.shields.io/badge/TypeScript-007ACC?style=flat&logo=typescript&logoColor=white)](https://www.typescriptlang.org/) -[![License](https://img.shields.io/github/license/microsoft/connected-workbooks)](https://github.com/microsoft/connected-workbooks/blob/master/LICENSE) [![npm version](https://img.shields.io/npm/v/@microsoft/connected-workbooks)](https://www.npmjs.com/package/@microsoft/connected-workbooks) -[![Build Status](https://img.shields.io/github/workflow/status/microsoft/connected-workbooks/CI)](https://github.com/microsoft/connected-workbooks/actions) +[![Build Status](https://img.shields.io/github/actions/workflow/status/microsoft/connected-workbooks/azure-pipelines.yml?branch=main)](https://github.com/microsoft/connected-workbooks/actions) -**Open your data directly in Excel for the Web with zero installation** - A JavaScript library that converts web tables and data into interactive Excel workbooks with Power Query integration and custom branded templates - -
-Connected Workbooks Demo
- -
+> Need the high-level overview, feature tour, or repo structure? See the root [`README.md`](../README.md). This file covers the TypeScript package specifically. --- -## ✨ Key Features & Benefits - -Transform your web applications with enterprise-grade Excel integration that goes far beyond simple CSV exports. - -### 🎯 **Interactive Excel Workbooks, Not Static Files** -Convert raw data or HTML tables arrays to Excel tables while preserving data types, ensuring your data maintains its structure and formatting. instead of basic CSV exports that lose all structure and functionality. - -### 🌐 **Zero-Installation Excel Experience** -Launch workbooks directly in Excel for the Web through any browser without requiring Excel desktop installation, making your data accessible to any user anywhere. No installation required, works on any device. - -### 🎨 **Corporate Branding & Custom Dashboards** -Inject your data into pre-built Excel templates containing your company branding, PivotTables, charts, and business logic while preserving all formatting and calculations. Use your own branded Excel templates with PivotTables and charts to maintain corporate identity and pre-built analytics. - -### πŸ”„ **Live Data Connections with Power Query** -Create workbooks that automatically refresh from your web APIs, databases, or data sources using Microsoft's Power Query technology, eliminating manual data updates. Create workbooks that refresh data on-demand using Power Query for real-time data updates and automated reporting. - -### βš™οΈ **Advanced Configuration** -Full control over document properties including title and description for professional document management, allowing you to customize metadata and maintain enterprise standards. - ---- - -## 🏒 Where is this library used? - -Open In Excel powers data export functionality across Microsoft's enterprise platforms: - -
- -|Azure Data Explorer|Log Analytics|Datamart|Viva Sales| -|:---:|:---:|:---:|:---:| -|**Azure Data Explorer**|**Log Analytics**|**Datamart**|**Viva Sales**| - -
- - - ---- - -## πŸš€ Quick Start - -### Installation +## Installation ```bash npm install @microsoft/connected-workbooks ``` ---- +The library targets evergreen browsers. No native modules, build steps, or Office add-ins are required. -## πŸ’‘ Usage Examples +--- -### πŸ“‹ **HTML Table Export** +## Usage Examples -Perfect for quick data exports from existing web tables. +### HTML Table Export -```typescript +```ts import { workbookManager } from '@microsoft/connected-workbooks'; -// One line of code to convert any table -const blob = await workbookManager.generateTableWorkbookFromHtml( - document.querySelector('table') as HTMLTableElement -); - -// Open in Excel for the Web with editing enabled -workbookManager.openInExcelWeb(blob, "QuickExport.xlsx", true); +const table = document.querySelector('table') as HTMLTableElement; +const blob = await workbookManager.generateTableWorkbookFromHtml(table); +await workbookManager.openInExcelWeb(blob, 'QuickExport.xlsx', true); ``` -### πŸ“Š **Smart Data Formatting** +### Grid Export With Smart Headers -Transform raw data arrays into professionally formatted Excel tables. - -```typescript -import { workbookManager } from '@microsoft/connected-workbooks'; - -const salesData = { +```ts +const grid = { config: { - promoteHeaders: true, // First row becomes headers - adjustColumnNames: true // Clean up column names + promoteHeaders: true, + adjustColumnNames: true }, data: [ - ["Product", "Revenue", "InStock", "Category", "LastUpdated"], - ["Surface Laptop", 1299.99, true, "Hardware", "2024-10-26"], - ["Office 365", 99.99, true, "Software", "2024-10-26"], - ["Azure Credits", 500.00, false, "Cloud", "2024-10-25"], - ["Teams Premium", 149.99, true, "Software", "2024-10-24"] + ['Product', 'Revenue', 'InStock', 'Category'], + ['Surface Laptop', 1299.99, true, 'Hardware'], + ['Office 365', 99.99, true, 'Software'] ] }; -const blob = await workbookManager.generateTableWorkbookFromGrid(salesData); -workbookManager.openInExcelWeb(blob, "SalesReport.xlsx", true); +const blob = await workbookManager.generateTableWorkbookFromGrid(grid); +await workbookManager.openInExcelWeb(blob, 'SalesReport.xlsx', true); ``` -
-Smart Formatted Excel Table -
- - - -### 🎨 **Custom Branded Templates** - -Transform your data using pre-built Excel templates with your corporate branding. - - - -**Steps:** - -1. **Prepare Your Template File** - - Open Excel and create (or open) your branded file. -2. **Pick one sheet that will hold your data.** +### Inject Data Into Templates - The default "Sheet1"(3) -3. **Inside that sheet, choose were you want your data to be populated(1) and create a table (Insert β†’ Table).** +1. Design an `.xlsx` template that includes a table (for example `QuarterlyData`). +2. Upload or fetch the template in the browser. +3. Supply the file plus metadata: - The default table name is Table1(2) - The table need to have the same column structure as your incoming data. -4. **Add any charts, formulas, or formatting that reference this table.** - - Example: Pie chart using Gross column(4). -5. **Save the Excel file (e.g., my-template.xlsx).** -6. **Use the saved file as the template for your incoming data** - -The library will then populate the designated table with your data. Any functions, figures, or references linked to this table within the Excel template will automatically reflect the newly exported data. - -
-Custom Branded Excel Dashboard -
- - - -#### πŸ“ **Loading Template Files** - -```typescript -// Method 1: File upload from user -const templateInput = document.querySelector('#template-upload') as HTMLInputElement; -const templateFile = templateInput.files[0]; - -// Method 2: Fetch from your server +```ts const templateResponse = await fetch('/assets/templates/sales-dashboard.xlsx'); const templateFile = await templateResponse.blob(); -// Method 3: Drag and drop -function handleTemplateDrop(event: DragEvent) { - const templateFile = event.dataTransfer.files[0]; - // Use templateFile with the library -} -``` - -#### πŸ“Š **Generate Branded Workbook** - -```typescript -const quarterlyData = { - config: { promoteHeaders: true, adjustColumnNames: true }, - data: [ - ["Region", "Q3_Revenue", "Q4_Revenue", "Growth", "Target_Met"], - ["North America", 2500000, 2750000, "10%", true], - ["Europe", 1800000, 2100000, "17%", true], - ["Asia Pacific", 1200000, 1400000, "17%", true], - ["Latin America", 800000, 950000, "19%", true] - ] -}; - -// Inject data into your branded template const blob = await workbookManager.generateTableWorkbookFromGrid( quarterlyData, - undefined, // Use template's existing data structure + undefined, { - templateFile: templateFile, + templateFile, TempleteSettings: { - sheetName: "Dashboard", // Target worksheet - tableName: "QuarterlyData" // Target table name + sheetName: 'Dashboard', + tableName: 'QuarterlyData' } } ); -// Users get a fully branded report -workbookManager.openInExcelWeb(blob, "Q4_Executive_Dashboard.xlsx", true); +await workbookManager.openInExcelWeb(blob, 'ExecutiveDashboard.xlsx', true); ``` -
-Custom Branded Excel Dashboard -
- -> πŸ’‘ **Template Requirements**: Include a query named **"Query1"** connected to a **Table**. +### Power Query Workbooks -### πŸ”„ **Live Data Connections with Power Query** - -Create workbooks that automatically refresh from your data sources. - -```typescript -import { workbookManager } from '@microsoft/connected-workbooks'; - -// Create a workbook that connects to your API +```ts const blob = await workbookManager.generateSingleQueryWorkbook({ - queryMashup: `let - Source = {1..10} - in - Source`, + queryMashup: `let Source = Json.Document(Web.Contents('https://contoso/api/orders')) in Source`, refreshOnOpen: true }); -workbookManager.openInExcelWeb(blob, "MyData.xlsx", true); +await workbookManager.openInExcelWeb(blob, 'Orders.xlsx', true); ``` -> πŸ“š **Learn Power Query**: New to Power Query? Check out the [official documentation](https://docs.microsoft.com/en-us/power-query/) to unlock the full potential of live data connections. - -
-Live Data Workbook -
-### πŸ“„ **Professional Document Properties** +### Document Properties & Downloads -Add metadata and professional document properties for enterprise use. - -```typescript -const blob = await workbookManager.generateTableWorkbookFromHtml( - document.querySelector('table') as HTMLTableElement, - { - docProps: { - createdBy: 'John Doe', - lastModifiedBy: 'Jane Doe', - description: 'Sales Report Q4 2024', - title: 'Quarterly Sales Data' - } +```ts +const blob = await workbookManager.generateTableWorkbookFromHtml(table, { + docProps: { + createdBy: 'Contoso Portal', + description: 'Q4 pipeline export', + title: 'Executive Dashboard' } -); +}); -// Download for offline use -workbookManager.downloadWorkbook(blob, "MyTable.xlsx"); +workbookManager.downloadWorkbook(blob, 'Pipeline.xlsx'); ``` -
-Professional Document Properties -
- -## πŸ“š Complete API Reference +--- -### Core Functions +## API Surface -#### πŸ”— `generateSingleQueryWorkbook()` -Create Power Query connected workbooks with live data refresh capabilities. +### `generateSingleQueryWorkbook()` -```typescript +```ts async function generateSingleQueryWorkbook( - query: QueryInfo, - grid?: Grid, + query: QueryInfo, + grid?: Grid, fileConfigs?: FileConfigs ): Promise ``` -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `query` | [`QueryInfo`](#queryinfo) | βœ… **Required** | Power Query configuration | -| `grid` | [`Grid`](#grid) | Optional | Pre-populate with data | -| `fileConfigs` | [`FileConfigs`](#fileconfigs) | Optional | Customization options | +- `query`: Power Query definition (M script, refresh flag, query name). +- `grid`: Optional seed data. +- `fileConfigs`: Template, metadata, or host options. -#### πŸ“‹ `generateTableWorkbookFromHtml()` -Convert HTML tables to Excel workbooks instantly. +### `generateTableWorkbookFromHtml()` -```typescript +```ts async function generateTableWorkbookFromHtml( - htmlTable: HTMLTableElement, + htmlTable: HTMLTableElement, fileConfigs?: FileConfigs ): Promise ``` -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `htmlTable` | `HTMLTableElement` | βœ… **Required** | Source HTML table | -| `fileConfigs` | [`FileConfigs`](#fileconfigs) | Optional | Customization options | +### `generateTableWorkbookFromGrid()` -#### πŸ“Š `generateTableWorkbookFromGrid()` -Transform raw data arrays into formatted Excel tables. - -```typescript +```ts async function generateTableWorkbookFromGrid( - grid: Grid, + grid: Grid, fileConfigs?: FileConfigs ): Promise ``` -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `grid` | [`Grid`](#grid) | βœ… **Required** | Data and configuration | -| `fileConfigs` | [`FileConfigs`](#fileconfigs) | Optional | Customization options | +### `openInExcelWeb()` and `getExcelForWebWorkbookUrl()` -#### 🌐 `openInExcelWeb()` -Open workbooks directly in Excel for the Web. +Launch Excel for the Web immediately or just capture the URL for custom navigation flows. -```typescript -async function openInExcelWeb( - blob: Blob, - filename?: string, - allowTyping?: boolean -): Promise -``` +### `downloadWorkbook()` -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `blob` | `Blob` | βœ… **Required** | Generated workbook | -| `filename` | `string` | Optional | Custom filename | -| `allowTyping` | `boolean` | Optional | Enable editing (default: false) | - -#### πŸ’Ύ `downloadWorkbook()` -Trigger browser download of the workbook. - -```typescript -function downloadWorkbook(file: Blob, filename: string): void -``` - -#### πŸ”— `getExcelForWebWorkbookUrl()` -Get the Excel for Web URL without opening (useful for custom integrations). - -```typescript -async function getExcelForWebWorkbookUrl( - file: Blob, - filename?: string, - allowTyping?: boolean -): Promise -``` +Trigger a regular browser download of the generated blob. --- -## πŸ”§ Type Definitions - -### QueryInfo -Power Query configuration for connected workbooks. +## Type Definitions -```typescript +```ts interface QueryInfo { - queryMashup: string; // Power Query M language code - refreshOnOpen: boolean; // Auto-refresh when opened - queryName?: string; // Query identifier (default: "Query1") + queryMashup: string; + refreshOnOpen: boolean; + queryName?: string; // default: "Query1" } -``` - -### Grid -Data structure for tabular information. -```typescript interface Grid { - data: (string | number | boolean)[][]; // Raw data rows - config?: GridConfig; // Processing options -} - -interface GridConfig { - promoteHeaders?: boolean; // Use first row as headers - adjustColumnNames?: boolean; // Fix duplicate/invalid names + data: (string | number | boolean | null)[][]; + config?: { + promoteHeaders?: boolean; + adjustColumnNames?: boolean; + }; } -``` -### FileConfigs -Advanced customization options. - -```typescript interface FileConfigs { - templateFile?: File | Buffer; // Custom Excel template - docProps?: DocProps; // Document metadata - hostName?: string; // Creator application name - TempleteSettings?: TempleteSettings; // Template-specific settings + templateFile?: File | Blob | Buffer; + docProps?: DocProps; + hostName?: string; + TempleteSettings?: { + tableName?: string; + sheetName?: string; + }; } -interface TempleteSettings { - tableName?: string; // Target table name in template - sheetName?: string; // Target worksheet name -} -``` - -### DocProps -Document metadata and properties. - -```typescript interface DocProps { - title?: string; // Document title - subject?: string; // Document subject - keywords?: string; // Search keywords - createdBy?: string; // Author name - description?: string; // Document description - lastModifiedBy?: string; // Last editor - category?: string; // Document category - revision?: string; // Version number + title?: string; + subject?: string; + keywords?: string; + createdBy?: string; + description?: string; + lastModifiedBy?: string; + category?: string; + revision?: string; } ``` --- -## Contributing - -This project welcomes contributions and suggestions. Most contributions require you to agree to a -Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. - -When you submit a pull request, a CLA bot will automatically determine whether you need to provide -a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions -provided by the bot. You will only need to do this once across all repos using our CLA. - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +## Development -### Getting Started -1. Fork the repository -2. Create a feature branch -3. Make your changes -4. Add tests for new functionality -5. Submit a pull request - -### Development Setup ```bash -git clone https://github.com/microsoft/connected-workbooks.git -cd connected-workbooks +cd typescript npm install npm run build npm test ``` ---- - -## πŸ“„ License -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. - -## πŸ”— Related Resources - -- [πŸ“– Power Query Documentation](https://powerquery.microsoft.com/en-us/) -- [🏒 Excel for Developers](https://docs.microsoft.com/en-us/office/dev/excel/) -- [πŸ”§ Microsoft Graph Excel APIs](https://docs.microsoft.com/en-us/graph/api/resources/excel) +Use `npm run validate:implementations` to compare the TypeScript and .NET output when making cross-language changes. --- -## Trademarks - -This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft -trademarks or logos is subject to and must follow -[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). -Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. -Any use of third-party trademarks or logos are subject to those third-party's policies. - ---- - -## Keywords +## Contributing -Power Query, Excel, Office, Workbook, Refresh, Table, xlsx, export, CSV, data export, HTML table, web to Excel, JavaScript Excel, TypeScript Excel, Excel template, PivotTable, connected data, live data, data refresh, Excel for Web, browser Excel, spreadsheet, data visualization, Microsoft Office, Office 365, Excel API, workbook generation, table export, grid export, Excel automation, data processing, business intelligence +Follow the guidance in the root [`README.md`](../README.md#contributing). Pull requests should include unit tests (`npm test`) and adhere to the repo ESLint/Prettier settings before submission. diff --git a/typescript/scripts/inspectMashup.js b/typescript/scripts/inspectMashup.js index b367dde..2bd9379 100644 --- a/typescript/scripts/inspectMashup.js +++ b/typescript/scripts/inspectMashup.js @@ -2,24 +2,100 @@ const path = require("path"); const { extractWorkbookDetails } = require("./workbookDetails"); +function usage() { + console.error("Usage:"); + console.error(" node inspectMashup.js --pretty "); + console.error(" node inspectMashup.js "); +} + +function printPretty(details) { + const sectionPreview = details.sectionContent.split("\n").slice(0, 5).join("\n"); + const metadataPreview = details.metadataXml.slice(0, 400); + console.log(`\n${details.workbookPath}`); + console.log(` Query name : ${details.queryName}`); + console.log(` Metadata : ${details.metadataBytesLength ?? details.metadataXml.length} bytes`); + console.log(" Section1.m preview:\n" + sectionPreview.split("\n").map((line) => ` ${line}`).join("\n")); + console.log(" Metadata preview:\n" + metadataPreview.split("\n").map((line) => ` ${line}`).join("\n")); +} + +function compare(detailsA, detailsB) { + const checks = [ + ["Query name", detailsA.queryName, detailsB.queryName], + ["Connection name", detailsA.connection.name, detailsB.connection.name], + ["Connection description", detailsA.connection.description, detailsB.connection.description], + ["Connection location", detailsA.connection.location, detailsB.connection.location], + ["Connection command", detailsA.connection.command, detailsB.connection.command], + ["Refresh flag", detailsA.connection.refreshOnLoad, detailsB.connection.refreshOnLoad], + ]; + + const mismatches = []; + for (const [label, left, right] of checks) { + if (left !== right) { + mismatches.push(`${label} mismatch:\n ${detailsA.workbookPath}: ${left}\n ${detailsB.workbookPath}: ${right}`); + } + } + + const section1Filter = (paths) => paths.filter((entry) => entry.includes("Section1/")); + const leftPaths = section1Filter(detailsA.metadataItemPaths); + const rightPaths = section1Filter(detailsB.metadataItemPaths); + if (JSON.stringify(leftPaths) !== JSON.stringify(rightPaths)) { + mismatches.push("Metadata ItemPath entries differ"); + } + + if (!detailsA.sharedStrings.includes(detailsA.queryName)) { + mismatches.push(`${path.basename(detailsA.workbookPath)} sharedStrings missing query name`); + } + + if (!detailsB.sharedStrings.includes(detailsB.queryName)) { + mismatches.push(`${path.basename(detailsB.workbookPath)} sharedStrings missing query name`); + } + + return mismatches; +} + +async function prettyMode(workbook) { + const details = await extractWorkbookDetails(path.resolve(workbook)); + printPretty(details); +} + +async function comparisonMode(left, right) { + const leftDetails = await extractWorkbookDetails(path.resolve(left)); + const rightDetails = await extractWorkbookDetails(path.resolve(right)); + const mismatches = compare(leftDetails, rightDetails); + if (mismatches.length === 0) { + console.log("\nβœ… Workbooks match for inspected fields."); + return; + } + + console.error("\n❌ Workbooks differ:"); + mismatches.forEach((item) => console.error(` - ${item}`)); + process.exit(1); +} + async function main() { - const targets = process.argv.slice(2); - if (targets.length === 0) { - console.error("Usage: node inspectMashup.js [workbook...]"); + const args = process.argv.slice(2); + if (args.length === 0) { + usage(); process.exit(1); } - for (const target of targets) { - const fullPath = path.resolve(target); - const details = await extractWorkbookDetails(fullPath); - const sectionPreview = details.sectionContent.split("\n").slice(0, 5).join("\n"); - const metadataPreview = details.metadataXml.slice(0, 400); - console.log(`\n${fullPath}`); - console.log(` Query name : ${details.queryName}`); - console.log(` Metadata : ${details.metadataBytesLength ?? details.metadataXml.length} bytes`); - console.log(" Section1.m preview:\n" + sectionPreview.split("\n").map((line) => ` ${line}`).join("\n")); - console.log(" Metadata preview:\n" + metadataPreview.split("\n").map((line) => ` ${line}`).join("\n")); + if (args[0] === "--pretty") { + if (args.length !== 2) { + usage(); + process.exit(1); + } + + await prettyMode(args[1]); + return; + } + + if (args.length === 2) { + await comparisonMode(args[0], args[1]); + return; } + + usage(); + process.exit(1); } main().catch((error) => { diff --git a/typescript/scripts/validateImplementations.js b/typescript/scripts/validateImplementations.js index ed299bd..acb0583 100644 --- a/typescript/scripts/validateImplementations.js +++ b/typescript/scripts/validateImplementations.js @@ -1,7 +1,6 @@ /* eslint-disable @typescript-eslint/no-var-requires */ const path = require("path"); const { execSync } = require("child_process"); -const { extractWorkbookDetails } = require("./workbookDetails"); const repoRoot = path.resolve(__dirname, "..", ".."); const tsRoot = path.resolve(__dirname, ".."); @@ -16,43 +15,6 @@ async function ensureSamples() { run("node scripts/generateWeatherSample.js", tsRoot); } -function compare(detailsA, detailsB) { - const mismatches = []; - const checks = [ - ["Query name", detailsA.queryName, detailsB.queryName], - ["Connection name", detailsA.connection.name, detailsB.connection.name], - ["Connection description", detailsA.connection.description, detailsB.connection.description], - ["Connection location", detailsA.connection.location, detailsB.connection.location], - ["Connection command", detailsA.connection.command, detailsB.connection.command], - ["Refresh flag", detailsA.connection.refreshOnLoad, detailsB.connection.refreshOnLoad], - ]; - - for (const [label, left, right] of checks) { - if (left !== right) { - mismatches.push(`${label} mismatch: '${left}' vs '${right}'`); - } - } - - const leftPaths = detailsA.metadataItemPaths.filter((entry) => entry.includes("Section1/")); - const rightPaths = detailsB.metadataItemPaths.filter((entry) => entry.includes("Section1/")); - if (JSON.stringify(leftPaths) !== JSON.stringify(rightPaths)) { - mismatches.push("Metadata ItemPath entries differ"); - } - - if (!detailsA.sharedStrings.includes(detailsA.queryName)) { - mismatches.push(".NET sharedStrings missing query name"); - } - - if (!detailsB.sharedStrings.includes(detailsB.queryName)) { - mismatches.push("TypeScript sharedStrings missing query name"); - } - - if (mismatches.length > 0) { - const error = mismatches.join("\n - "); - throw new Error(`Workbooks are not aligned:\n - ${error}`); - } -} - async function main() { const args = process.argv.slice(2); let dotnetWorkbook; @@ -69,10 +31,8 @@ async function main() { process.exit(1); } - const dotnetDetails = await extractWorkbookDetails(dotnetWorkbook); - const tsDetails = await extractWorkbookDetails(tsWorkbook); - compare(dotnetDetails, tsDetails); - console.log("\nβœ… Validation succeeded. The .NET and TypeScript outputs match for the inspected fields."); + const inspectScript = path.resolve(tsRoot, "scripts", "inspectMashup.js"); + run(`node "${inspectScript}" "${dotnetWorkbook}" "${tsWorkbook}"`, repoRoot); } main().catch((error) => { From 98b80cd49227a7bdb1555d63a2969058e9f4ad53 Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Wed, 26 Nov 2025 16:15:56 +0100 Subject: [PATCH 5/5] add GH workflow --- .github/workflows/tests.yml | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..e84ae55 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,51 @@ +name: TypeScript and .NET Tests + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + workflow_dispatch: + +jobs: + ci: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Use Node.js 18 + uses: actions/setup-node@v4 + with: + node-version: '18' + cache: 'npm' + cache-dependency-path: 'typescript/package-lock.json' + + - name: Install TypeScript dependencies + run: npm ci + working-directory: typescript + + - name: Build TypeScript package + run: npm run build + working-directory: typescript + + - name: Test TypeScript package + run: npm test + working-directory: typescript + + - name: Setup .NET 8 SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore .NET solution + run: dotnet restore ConnectedWorkbooks.sln + working-directory: dotnet + + - name: Build .NET solution + run: dotnet build ConnectedWorkbooks.sln --configuration Release --no-restore + working-directory: dotnet + + - name: Test .NET solution + run: dotnet test ConnectedWorkbooks.sln --configuration Release --no-build + working-directory: dotnet