From f430b515620e8ab5abbefb3ff8cb9cfe85a40f71 Mon Sep 17 00:00:00 2001 From: CyrilGendarme Date: Mon, 2 Dec 2024 12:13:17 +0100 Subject: [PATCH 1/9] some fixes for the migration --- src/.vscode/launch.json | 10 +++++----- .../Paillave.EntityFrameworkCoreExtension.csproj | 2 +- src/Paillave.Etl.Autofac/Paillave.Etl.Autofac.csproj | 2 +- src/Paillave.Etl.Dropbox/Paillave.Etl.Dropbox.csproj | 6 +++--- .../Paillave.Etl.EntityFrameworkCore.csproj | 9 +++++---- .../Paillave.Etl.ExcelFile.csproj | 4 ++-- .../Paillave.Etl.ExecutionToolkit.csproj | 4 ++-- .../Paillave.Etl.FileSystem.csproj | 4 ++-- .../Paillave.Etl.FromConfigurationConnectors.csproj | 2 +- src/Paillave.Etl.Ftp/Paillave.Etl.Ftp.csproj | 8 ++++---- src/Paillave.Etl.GraphApi/Paillave.Etl.GraphApi.csproj | 10 +++++----- src/Paillave.Etl.Http/Paillave.Etl.Http.csproj | 6 +++--- src/Paillave.Etl.Mail/Paillave.Etl.Mail.csproj | 8 ++++---- src/Paillave.Etl.Pdf/Paillave.Etl.Pdf.csproj | 2 +- src/Paillave.Etl.S3/Paillave.Etl.S3.csproj | 6 +++--- src/Paillave.Etl.Sftp/Paillave.Etl.Sftp.csproj | 6 +++--- src/Paillave.Etl.Sftp/SftpFileValueProvider.cs | 2 +- .../Paillave.Etl.SqlServer.csproj | 6 +++--- src/Paillave.Etl.Tests/Paillave.Etl.Tests.csproj | 10 +++++----- src/Paillave.Etl.Zip/Paillave.Etl.Zip.csproj | 2 +- src/Paillave.Pdf/Paillave.Pdf.csproj | 2 +- src/Paillave.Pdf/SimpleLinesMethod.cs | 2 ++ src/Paillave.Pdf/TextTemplate.cs | 8 ++++---- src/Tutorials/BlogTutorial/BlogTutorial.csproj | 6 +++--- .../Paillave.Etl.Samples/Paillave.Etl.Samples.csproj | 8 ++++---- src/Tutorials/SimpleTutorial/SimpleTutorial.csproj | 6 +++--- 26 files changed, 72 insertions(+), 69 deletions(-) diff --git a/src/.vscode/launch.json b/src/.vscode/launch.json index 63dbeb12..155f92dd 100644 --- a/src/.vscode/launch.json +++ b/src/.vscode/launch.json @@ -9,7 +9,7 @@ "type": "coreclr", "request": "launch", "preLaunchTask": "build", - "program": "${workspaceFolder}/Paillave.Etl.Scheduler/bin/Debug/net7.0/Paillave.Etl.Scheduler.dll", + "program": "${workspaceFolder}/Paillave.Etl.Scheduler/bin/Debug/net9.0/Paillave.Etl.Scheduler.dll", "args": [ ], "cwd": "${workspaceFolder}/Paillave.Etl.Scheduler", "stopAtEntry": false, @@ -20,7 +20,7 @@ "type": "coreclr", "request": "launch", "preLaunchTask": "build", - "program": "${workspaceFolder}/Tutorials/Paillave.Etl.Samples/bin/Debug/net7.0/Paillave.Etl.Samples.dll", + "program": "${workspaceFolder}/Tutorials/Paillave.Etl.Samples/bin/Debug/net9.0/Paillave.Etl.Samples.dll", "args": [ "/home/stephane/Desktop/" ], @@ -33,7 +33,7 @@ "type": "coreclr", "request": "launch", "preLaunchTask": "build", - "program": "${workspaceFolder}/Tutorials/SimpleTutorial/bin/Debug/net7.0/SimpleTutorial.dll", + "program": "${workspaceFolder}/Tutorials/SimpleTutorial/bin/Debug/net9.0/SimpleTutorial.dll", "args": [ "data", "Server=localhost,1433;Database=SimpleTutorial;user=SimpleTutorial;password=TestEtl.TestEtl;MultipleActiveResultSets=True" @@ -47,7 +47,7 @@ "type": "coreclr", "request": "launch", "preLaunchTask": "build", - "program": "${workspaceFolder}/Tutorials/BlogTutorial/bin/Debug/net7.0/BlogTutorial.dll", + "program": "${workspaceFolder}/Tutorials/BlogTutorial/bin/Debug/net9.0/BlogTutorial.dll", "args": [ "data", "Server=localhost,1433;Database=BlogTutorial;user=BlogTutorial;password=TestEtl.TestEtl;MultipleActiveResultSets=True" @@ -61,7 +61,7 @@ "type": "coreclr", "request": "launch", "preLaunchTask": "build", - "program": "${workspaceFolder}/Tutorials/Paillave.Etl.Samples/bin/Debug/net7.0/Paillave.Etl.Samples.dll", + "program": "${workspaceFolder}/Tutorials/Paillave.Etl.Samples/bin/Debug/net9.0/Paillave.Etl.Samples.dll", "args": [ "data", "Server=localhost,1433;Database=BlogTutorial;user=BlogTutorial;password=TestEtl.TestEtl;MultipleActiveResultSets=True" diff --git a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj index 1a020988..05e90623 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj +++ b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj @@ -1,7 +1,7 @@ - net7.0 + net9.0;net8.0 Paillave.EntityFrameworkCoreExtension ETL .net core EF reactive Entity Framework extensions diff --git a/src/Paillave.Etl.Autofac/Paillave.Etl.Autofac.csproj b/src/Paillave.Etl.Autofac/Paillave.Etl.Autofac.csproj index fb695a41..41c57695 100644 --- a/src/Paillave.Etl.Autofac/Paillave.Etl.Autofac.csproj +++ b/src/Paillave.Etl.Autofac/Paillave.Etl.Autofac.csproj @@ -8,7 +8,7 @@ Autofac extensions for Etl.Net - + diff --git a/src/Paillave.Etl.Dropbox/Paillave.Etl.Dropbox.csproj b/src/Paillave.Etl.Dropbox/Paillave.Etl.Dropbox.csproj index 0d9a1624..2b85e5f3 100644 --- a/src/Paillave.Etl.Dropbox/Paillave.Etl.Dropbox.csproj +++ b/src/Paillave.Etl.Dropbox/Paillave.Etl.Dropbox.csproj @@ -1,15 +1,15 @@ - net7.0 + net9.0;net8.0 Paillave.EtlNet.Dropbox ETL .net core SSIS reactive Dropbox ETL.net Dropbox extensions Extensions for Etl.Net to query against Dropbox - - + + diff --git a/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj b/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj index 3edc4258..9f216a7f 100644 --- a/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj +++ b/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj @@ -1,7 +1,7 @@ - net7.0 + net9.0;net8.0 Paillave.EtlNet.EntityFrameworkCore ETL .net core SSIS reactive Entity Framework core ETL.net EntityFrameworkCore extensions @@ -9,9 +9,10 @@ - - - + + + + diff --git a/src/Paillave.Etl.ExcelFile/Paillave.Etl.ExcelFile.csproj b/src/Paillave.Etl.ExcelFile/Paillave.Etl.ExcelFile.csproj index 0a3efb8a..60c65435 100644 --- a/src/Paillave.Etl.ExcelFile/Paillave.Etl.ExcelFile.csproj +++ b/src/Paillave.Etl.ExcelFile/Paillave.Etl.ExcelFile.csproj @@ -9,8 +9,8 @@ - - + + diff --git a/src/Paillave.Etl.ExecutionToolkit/Paillave.Etl.ExecutionToolkit.csproj b/src/Paillave.Etl.ExecutionToolkit/Paillave.Etl.ExecutionToolkit.csproj index 2fcad437..24e94415 100644 --- a/src/Paillave.Etl.ExecutionToolkit/Paillave.Etl.ExecutionToolkit.csproj +++ b/src/Paillave.Etl.ExecutionToolkit/Paillave.Etl.ExecutionToolkit.csproj @@ -17,7 +17,7 @@ - - + + diff --git a/src/Paillave.Etl.FileSystem/Paillave.Etl.FileSystem.csproj b/src/Paillave.Etl.FileSystem/Paillave.Etl.FileSystem.csproj index e6266248..e314188d 100644 --- a/src/Paillave.Etl.FileSystem/Paillave.Etl.FileSystem.csproj +++ b/src/Paillave.Etl.FileSystem/Paillave.Etl.FileSystem.csproj @@ -1,7 +1,7 @@ - net7.0 + net9.0;net8.0 Paillave.EtlNet.FileSystem ETL .net core SSIS reactive text file csv ETL.net file system extensions @@ -11,6 +11,6 @@ - + diff --git a/src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj b/src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj index 2cb25c92..6dc11567 100644 --- a/src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj +++ b/src/Paillave.Etl.FromConfigurationConnectors/Paillave.Etl.FromConfigurationConnectors.csproj @@ -12,6 +12,6 @@ - + diff --git a/src/Paillave.Etl.Ftp/Paillave.Etl.Ftp.csproj b/src/Paillave.Etl.Ftp/Paillave.Etl.Ftp.csproj index 97ee98cb..01b953b4 100644 --- a/src/Paillave.Etl.Ftp/Paillave.Etl.Ftp.csproj +++ b/src/Paillave.Etl.Ftp/Paillave.Etl.Ftp.csproj @@ -1,7 +1,7 @@ - net7.0 + net9.0;net8.0 Paillave.EtlNet.Ftp ETL .net core SSIS reactive FTP ETL.net FTP extensions @@ -11,8 +11,8 @@ - - - + + + diff --git a/src/Paillave.Etl.GraphApi/Paillave.Etl.GraphApi.csproj b/src/Paillave.Etl.GraphApi/Paillave.Etl.GraphApi.csproj index 5e3fcb5c..809dbc81 100644 --- a/src/Paillave.Etl.GraphApi/Paillave.Etl.GraphApi.csproj +++ b/src/Paillave.Etl.GraphApi/Paillave.Etl.GraphApi.csproj @@ -1,21 +1,21 @@ - net7.0 + net9.0;net8.0 Paillave.EtlNet.GraphApi ETL .net core SSIS reactive GraphApi Mail ETL.net Mail extensions Extensions for Etl.Net to send of read EMails using GraphApi - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + diff --git a/src/Paillave.Etl.Http/Paillave.Etl.Http.csproj b/src/Paillave.Etl.Http/Paillave.Etl.Http.csproj index 159dfef4..0d501c77 100644 --- a/src/Paillave.Etl.Http/Paillave.Etl.Http.csproj +++ b/src/Paillave.Etl.Http/Paillave.Etl.Http.csproj @@ -1,15 +1,15 @@  - net7.0 + net9.0;net8.0 Paillave.EtlNet.Http ETL .net core SSIS Http Responses ETL.net Http extensions Extensions for Etl.Net to read Http Responses - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/src/Paillave.Etl.Mail/Paillave.Etl.Mail.csproj b/src/Paillave.Etl.Mail/Paillave.Etl.Mail.csproj index 7c377992..dbd5c93c 100644 --- a/src/Paillave.Etl.Mail/Paillave.Etl.Mail.csproj +++ b/src/Paillave.Etl.Mail/Paillave.Etl.Mail.csproj @@ -1,16 +1,16 @@ - net7.0 + net9.0;net8.0 Paillave.EtlNet.Mail ETL .net core SSIS reactive Mail ETL.net Mail extensions Extensions for Etl.Net to send of read EMails - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/src/Paillave.Etl.Pdf/Paillave.Etl.Pdf.csproj b/src/Paillave.Etl.Pdf/Paillave.Etl.Pdf.csproj index 249a47ba..87dd4cfe 100644 --- a/src/Paillave.Etl.Pdf/Paillave.Etl.Pdf.csproj +++ b/src/Paillave.Etl.Pdf/Paillave.Etl.Pdf.csproj @@ -8,7 +8,7 @@ Pdf files extensions for Etl.Net - + diff --git a/src/Paillave.Etl.S3/Paillave.Etl.S3.csproj b/src/Paillave.Etl.S3/Paillave.Etl.S3.csproj index cedca087..b4416724 100644 --- a/src/Paillave.Etl.S3/Paillave.Etl.S3.csproj +++ b/src/Paillave.Etl.S3/Paillave.Etl.S3.csproj @@ -1,7 +1,7 @@ - net7.0 + net9.0;net8.0 Paillave.EtlNet.S3 ETL .net core SSIS reactive S3 ETL.net S3 extensions @@ -10,8 +10,8 @@ - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/src/Paillave.Etl.Sftp/Paillave.Etl.Sftp.csproj b/src/Paillave.Etl.Sftp/Paillave.Etl.Sftp.csproj index 1a2092de..82b148da 100644 --- a/src/Paillave.Etl.Sftp/Paillave.Etl.Sftp.csproj +++ b/src/Paillave.Etl.Sftp/Paillave.Etl.Sftp.csproj @@ -1,15 +1,15 @@ - net7.0 + net9.0;net8.0 Paillave.EtlNet.Sftp ETL .net core SSIS reactive SFTP ETL.net SFTP extensions Extensions for Etl.Net to query against SFTP - - + + diff --git a/src/Paillave.Etl.Sftp/SftpFileValueProvider.cs b/src/Paillave.Etl.Sftp/SftpFileValueProvider.cs index e27267cd..02a7ffef 100644 --- a/src/Paillave.Etl.Sftp/SftpFileValueProvider.cs +++ b/src/Paillave.Etl.Sftp/SftpFileValueProvider.cs @@ -28,7 +28,7 @@ protected override void Provide(Action pushFileValue, SftpAdapterCon pushFileValue(new SftpFileValue(connectionParameters, folder, file.Name, this.Code, this.Name, this.ConnectionName)); } } - private Renci.SshNet.Sftp.SftpFile[] GetFileList(SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters) + private Renci.SshNet.Sftp.ISftpFile[] GetFileList(SftpAdapterConnectionParameters connectionParameters, SftpAdapterProviderParameters providerParameters) { var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? ""); var connectionInfo = connectionParameters.CreateConnectionInfo(); diff --git a/src/Paillave.Etl.SqlServer/Paillave.Etl.SqlServer.csproj b/src/Paillave.Etl.SqlServer/Paillave.Etl.SqlServer.csproj index 2519fc7c..414d42f1 100644 --- a/src/Paillave.Etl.SqlServer/Paillave.Etl.SqlServer.csproj +++ b/src/Paillave.Etl.SqlServer/Paillave.Etl.SqlServer.csproj @@ -11,8 +11,8 @@ - - - + + + diff --git a/src/Paillave.Etl.Tests/Paillave.Etl.Tests.csproj b/src/Paillave.Etl.Tests/Paillave.Etl.Tests.csproj index ea2614ad..3cbbfb3c 100644 --- a/src/Paillave.Etl.Tests/Paillave.Etl.Tests.csproj +++ b/src/Paillave.Etl.Tests/Paillave.Etl.Tests.csproj @@ -1,7 +1,7 @@ - net7.0 + net9.0;net8.0 latest enable @@ -9,10 +9,10 @@ - - - - + + + + diff --git a/src/Paillave.Etl.Zip/Paillave.Etl.Zip.csproj b/src/Paillave.Etl.Zip/Paillave.Etl.Zip.csproj index de2321e9..826d7e4a 100644 --- a/src/Paillave.Etl.Zip/Paillave.Etl.Zip.csproj +++ b/src/Paillave.Etl.Zip/Paillave.Etl.Zip.csproj @@ -12,6 +12,6 @@ - + diff --git a/src/Paillave.Pdf/Paillave.Pdf.csproj b/src/Paillave.Pdf/Paillave.Pdf.csproj index c3357400..821dbe07 100644 --- a/src/Paillave.Pdf/Paillave.Pdf.csproj +++ b/src/Paillave.Pdf/Paillave.Pdf.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/Paillave.Pdf/SimpleLinesMethod.cs b/src/Paillave.Pdf/SimpleLinesMethod.cs index 1716ba12..60326e45 100644 --- a/src/Paillave.Pdf/SimpleLinesMethod.cs +++ b/src/Paillave.Pdf/SimpleLinesMethod.cs @@ -95,6 +95,8 @@ private Letter CloneLetter(Letter letter) letter.Width, letter.FontSize, letter.Font, + UglyToad.PdfPig.Core.TextRenderingMode.Fill, + letter.Color, letter.Color, letter.PointSize, letter.TextSequence); diff --git a/src/Paillave.Pdf/TextTemplate.cs b/src/Paillave.Pdf/TextTemplate.cs index 4dd28cda..69d83124 100644 --- a/src/Paillave.Pdf/TextTemplate.cs +++ b/src/Paillave.Pdf/TextTemplate.cs @@ -147,15 +147,15 @@ private class BookmarkTextTemplateCheck : ITextTemplateCheck public BookmarkTextTemplateCheck(List> bookmarks) => _bookmarks = bookmarks; public bool Check(TextLine textLine, Page page, List lines) => GetMatchingBookmark((decimal)textLine.BoundingBox.Top, (decimal)textLine.BoundingBox.Bottom, page.Number) != null; - private List GetMatchingBookmark(decimal top, decimal bottom, int pageNumber) + private List? GetMatchingBookmark(decimal top, decimal bottom, int pageNumber) { if (_bookmarks == null) return null; var heigh = top - bottom; return _bookmarks.FirstOrDefault(i => { - var bBottom = i[0].Destination.Coordinates.Bottom ?? (i[0].Destination.Coordinates.Top.Value - heigh); - var bTop = i[0].Destination.Coordinates.Top ?? (i[0].Destination.Coordinates.Bottom.Value + heigh); - return i[0].PageNumber == pageNumber && bBottom <= top && bTop >= bottom; + var bBottom = i[0].Destination.Coordinates.Bottom ?? i[0].Destination.Coordinates.Top??0 - (double)heigh; + var bTop = i[0].Destination.Coordinates.Top ?? i[0].Destination.Coordinates.Bottom??0 + (double)heigh; + return i[0].PageNumber == pageNumber && bBottom <= (double)top && bTop >= (double)bottom; }); } } diff --git a/src/Tutorials/BlogTutorial/BlogTutorial.csproj b/src/Tutorials/BlogTutorial/BlogTutorial.csproj index 0f85fc1c..a1627252 100644 --- a/src/Tutorials/BlogTutorial/BlogTutorial.csproj +++ b/src/Tutorials/BlogTutorial/BlogTutorial.csproj @@ -16,16 +16,16 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + Exe - net7.0 + net9.0;net8.0 enable diff --git a/src/Tutorials/Paillave.Etl.Samples/Paillave.Etl.Samples.csproj b/src/Tutorials/Paillave.Etl.Samples/Paillave.Etl.Samples.csproj index 9af3db79..2ebcaa98 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Paillave.Etl.Samples.csproj +++ b/src/Tutorials/Paillave.Etl.Samples/Paillave.Etl.Samples.csproj @@ -19,17 +19,17 @@ - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + Exe - net7.0 + net9.0;net8.0 enable enable diff --git a/src/Tutorials/SimpleTutorial/SimpleTutorial.csproj b/src/Tutorials/SimpleTutorial/SimpleTutorial.csproj index 6e71a1ca..5b7b438e 100644 --- a/src/Tutorials/SimpleTutorial/SimpleTutorial.csproj +++ b/src/Tutorials/SimpleTutorial/SimpleTutorial.csproj @@ -20,16 +20,16 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + Exe - net7.0 + net9.0;net8.0 enable From a7e55313087c57f4447ac115a083ae56a4ded371 Mon Sep 17 00:00:00 2001 From: CyrilGendarme Date: Mon, 2 Dec 2024 12:16:57 +0100 Subject: [PATCH 2/9] fix: update VSCode settings to include 'src' directory in file explorer --- .vscode/settings.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 5b820d2d..7406c914 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ -{ - "files.exclude": { - "src": true, // this is a default value - "documentation": true, // this is a default value - "docs": true, // this is a default value - }, +{ + "files.exclude": { + "src": false, + "documentation": true, // this is a default value + "docs": true, // this is a default value + }, } \ No newline at end of file From 898c1361f1952b0c13778294f391e7d438fefbe2 Mon Sep 17 00:00:00 2001 From: CyrilGendarme Date: Mon, 2 Dec 2024 12:37:42 +0100 Subject: [PATCH 3/9] upgrade XmlFileTests .csproj --- .../Paillave.Etl.XmlFileTests.csproj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Paillave.Etl.XmlFileTests/Paillave.Etl.XmlFileTests.csproj b/src/Paillave.Etl.XmlFileTests/Paillave.Etl.XmlFileTests.csproj index 7e4fc41d..f950db5d 100644 --- a/src/Paillave.Etl.XmlFileTests/Paillave.Etl.XmlFileTests.csproj +++ b/src/Paillave.Etl.XmlFileTests/Paillave.Etl.XmlFileTests.csproj @@ -1,7 +1,7 @@ - net7.0 + net9.0;net8.0 latest enable @@ -9,10 +9,10 @@ - - - - + + + + From adf31537e28eab7c0e28d9d7458b763d9264ddb9 Mon Sep 17 00:00:00 2001 From: CyrilGendarme Date: Mon, 2 Dec 2024 12:46:46 +0100 Subject: [PATCH 4/9] refactor: update import logic and streamline position processing in Program2 --- .../Paillave.Etl.Samples/Program2.cs | 5 +- .../Paillave.Etl.Samples/TestImport2.cs | 188 +++++++++--------- 2 files changed, 97 insertions(+), 96 deletions(-) diff --git a/src/Tutorials/Paillave.Etl.Samples/Program2.cs b/src/Tutorials/Paillave.Etl.Samples/Program2.cs index 5ebecd0f..4fef8888 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Program2.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Program2.cs @@ -55,11 +55,12 @@ static async Task SimplyImportAsync(string[] args) /// static async Task ImportAndCreateFileAsync(string[] args) { - var processRunner = StreamProcessRunner.Create(TestImport3.Import); + var processRunner = StreamProcessRunner.Create(TestImport2.Import); var structure = processRunner.GetDefinitionStructure(); // structure.OpenEstimatedExecutionPlan(); - ITraceReporter traceReporter = new AdvancedConsoleExecutionDisplay(); + // ITraceReporter traceReporter = new AdvancedConsoleExecutionDisplay(); + ITraceReporter traceReporter = new SimpleConsoleExecutionDisplay(); var dataAccess = new DataAccess.TestDbContext(); await dataAccess.Database.EnsureCreatedAsync(); // dataAccess.Database.Migrate(); diff --git a/src/Tutorials/Paillave.Etl.Samples/TestImport2.cs b/src/Tutorials/Paillave.Etl.Samples/TestImport2.cs index 069f085a..b3fe56db 100644 --- a/src/Tutorials/Paillave.Etl.Samples/TestImport2.cs +++ b/src/Tutorials/Paillave.Etl.Samples/TestImport2.cs @@ -22,20 +22,20 @@ public static void Import(ISingleStream contextStream) }).IsColumnSeparated(',')) .SetForCorrelation("Correlate portfolio row"); - // var positionFileStream = contextStream - // .FromConnector("Get position files", "POS") - // .CrossApplyTextFile("Parse position file", FlatFileDefinition.Create(i => new - // { - // PortfolioCode = i.ToColumn("PortfolioCode"), - // SecurityCode = i.ToColumn("SecurityCode"), - // Isin = i.ToColumn("Isin"), - // SecurityName = i.ToColumn("SecurityName"), - // SecurityClass = i.ToColumn("SecurityClass"), - // Issuer = i.ToColumn("Issuer"), - // Date = i.ToDateColumn("Date", "yyyyMMdd"), - // Value = i.ToNumberColumn("Value", "."), - // }).IsColumnSeparated(',')) - // .SetForCorrelation("Correlate position row"); + var positionFileStream = contextStream + .FromConnector("Get position files", "POS") + .CrossApplyTextFile("Parse position file", FlatFileDefinition.Create(i => new + { + PortfolioCode = i.ToColumn("PortfolioCode"), + SecurityCode = i.ToColumn("SecurityCode"), + Isin = i.ToColumn("Isin"), + SecurityName = i.ToColumn("SecurityName"), + SecurityClass = i.ToColumn("SecurityClass"), + Issuer = i.ToColumn("Issuer"), + Date = i.ToDateColumn("Date", "yyyyMMdd"), + Value = i.ToNumberColumn("Value", "."), + }).IsColumnSeparated(',')) + .SetForCorrelation("Correlate position row"); var sicavStream = portfolioFileStream .Distinct("Distinct sicav", i => i.SicavCode, true) @@ -50,89 +50,89 @@ public static void Import(ISingleStream contextStream) // .DoNotUpdateIfExists() .WithMode(SaveMode.EntityFrameworkCore)); - // var portfolioStream = portfolioFileStream - // .Distinct("Distinct portfolio", i => i.PortfolioCode, true) - // .CorrelateToSingle("Get related sicav and create portfolio", sicavStream, (row, sicav) => new DataAccess.Portfolio - // { - // InternalCode = row.PortfolioCode, - // Name = row.PortfolioName, - // SicavId = sicav.Id - // }) - // .EfCoreSave("Save portfolio", o => o - // .SeekOn(i => i.InternalCode) - // .DoNotUpdateIfExists()); + var portfolioStream = portfolioFileStream + .Distinct("Distinct portfolio", i => i.PortfolioCode, true) + .CorrelateToSingle("Get related sicav and create portfolio", sicavStream, (row, sicav) => new DataAccess.Portfolio + { + InternalCode = row.PortfolioCode, + Name = row.PortfolioName, + SicavId = sicav.Id + }) + .EfCoreSave("Save portfolio", o => o + .SeekOn(i => i.InternalCode) + .DoNotUpdateIfExists()); - // var compositionStream = positionFileStream - // .Distinct("Distinct compositions", i => new { i.PortfolioCode, i.Date }, true) - // .Lookup("Get related portfolio", - // portfolioStream, - // i => i.PortfolioCode, - // i => i.InternalCode, - // (row, portfolio) => new - // { - // Portfolio = portfolio, - // Composition = new DataAccess.Composition - // { - // Date = row.Date, - // PortfolioId = portfolio.Id - // } - // }) - // .EfCoreSave("Save composition", o => o - // .Entity(i => i.Composition) - // .SeekOn(i => new { i.Date, i.PortfolioId }) - // .DoNotUpdateIfExists() - // .Output((i, e) => new - // { - // i.Portfolio, - // Composition = e - // })); + var compositionStream = positionFileStream + .Distinct("Distinct compositions", i => new { i.PortfolioCode, i.Date }, true) + .Lookup("Get related portfolio", + portfolioStream, + i => i.PortfolioCode, + i => i.InternalCode, + (row, portfolio) => new + { + Portfolio = portfolio, + Composition = new DataAccess.Composition + { + Date = row.Date, + PortfolioId = portfolio.Id + } + }) + .EfCoreSave("Save composition", o => o + .Entity(i => i.Composition) + .SeekOn(i => new { i.Date, i.PortfolioId }) + .DoNotUpdateIfExists() + .Output((i, e) => new + { + i.Portfolio, + Composition = e + })); - // var securityStream = positionFileStream - // .Distinct("Distinct securities", i => i.SecurityCode) - // .Select("Create security", i => - // { - // if (string.IsNullOrWhiteSpace(i.SecurityClass)) - // { - // return new DataAccess.Equity - // { - // InternalCode = i.SecurityCode, - // Name = i.SecurityName, - // Isin = i.Isin, - // Issuer = i.Issuer - // } as DataAccess.Security; - // } - // return new DataAccess.ShareClass - // { - // InternalCode = i.SecurityCode, - // Name = i.SecurityName, - // Isin = i.Isin, - // Class = i.SecurityClass - // } as DataAccess.Security; - // }) - // .EfCoreSave("Save security", o => o - // .SeekOn(i => i.Isin) - // .AlternativelySeekOn(i => i.InternalCode) - // .DoNotUpdateIfExists()); + var securityStream = positionFileStream + .Distinct("Distinct securities", i => i.SecurityCode) + .Select("Create security", i => + { + if (string.IsNullOrWhiteSpace(i.SecurityClass)) + { + return new DataAccess.Equity + { + InternalCode = i.SecurityCode, + Name = i.SecurityName, + Isin = i.Isin, + Issuer = i.Issuer + } as DataAccess.Security; + } + return new DataAccess.ShareClass + { + InternalCode = i.SecurityCode, + Name = i.SecurityName, + Isin = i.Isin, + Class = i.SecurityClass + } as DataAccess.Security; + }) + .EfCoreSave("Save security", o => o + .SeekOn(i => i.Isin) + .AlternativelySeekOn(i => i.InternalCode) + .DoNotUpdateIfExists()); - // positionFileStream - // .CorrelateToSingle("Get related security", securityStream, (row, security) => new { Row = row, SecurityId = security.Id }) - // .CorrelateToSingle("Get related composition and create position", compositionStream, (row, composition) => new DataAccess.Position - // { - // Value = row.Row.Value, - // SecurityId = row.SecurityId, - // CompositionId = composition.Composition.Id - // }) - // .Distinct("Distinct positions", i => new { i.CompositionId, i.SecurityId }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum)) - // .EfCoreSave("Save position") - // .CorrelateToSingle("Get position portfolio", compositionStream, (p, c) => new { c.Portfolio.InternalCode, c.Composition.Date, p.Value }) - // .Distinct("Aggregate position per portfolio", i => new { i.InternalCode, i.Date }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum)) - // .ToTextFileValue("Export portfolios weight", "PortfoliosWeight.csv", FlatFileDefinition.Create(i => new - // { - // InternalCode = i.ToColumn("Code"), - // Date = i.ToDateColumn("Date", "ddd dd MMM, yyyy"), - // Value = i.ToNumberColumn("Weight", "."), - // }).IsColumnSeparated(',')) - // .ToConnector("Save portfolios position file", "OUT"); + positionFileStream + .CorrelateToSingle("Get related security", securityStream, (row, security) => new { Row = row, SecurityId = security.Id }) + .CorrelateToSingle("Get related composition and create position", compositionStream, (row, composition) => new DataAccess.Position + { + Value = row.Row.Value, + SecurityId = row.SecurityId, + CompositionId = composition.Composition.Id + }) + .Distinct("Distinct positions", i => new { i.CompositionId, i.SecurityId }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum)) + .EfCoreSave("Save position") + .CorrelateToSingle("Get position portfolio", compositionStream, (p, c) => new { c.Portfolio.InternalCode, c.Composition.Date, p.Value }) + .Distinct("Aggregate position per portfolio", i => new { i.InternalCode, i.Date }, o => o.ForProperty(i => i.Value, DistinctAggregator.Sum)) + .ToTextFileValue("Export portfolios weight", "PortfoliosWeight.csv", FlatFileDefinition.Create(i => new + { + InternalCode = i.ToColumn("Code"), + Date = i.ToDateColumn("Date", "ddd dd MMM, yyyy"), + Value = i.ToNumberColumn("Weight", "."), + }).IsColumnSeparated(',')) + .ToConnector("Save portfolios position file", "OUT"); } } } \ No newline at end of file From e5c90bfb1d5c0c553e9b75b75f9e2a7ba141f838 Mon Sep 17 00:00:00 2001 From: CyrilGendarme Date: Mon, 2 Dec 2024 13:01:15 +0100 Subject: [PATCH 5/9] chore: update EntityFrameworkCore and related packages to version 9.0.0 --- .../Paillave.EntityFrameworkCoreExtension.csproj | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj index 05e90623..42b88582 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj +++ b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj @@ -9,9 +9,10 @@ - - - - + + + + + From 8fa15b5f18cce811e8519bb293aba8078fd02f5c Mon Sep 17 00:00:00 2001 From: CyrilGendarme Date: Mon, 2 Dec 2024 13:11:29 +0100 Subject: [PATCH 6/9] refactor: replace System.Data.SqlClient with Microsoft.Data.SqlClient in documentation and code --- README.md | 4 ++-- documentation/docs/tutorials/2-defineProcess.md | 4 ++-- documentation/docs/tutorials/3-trackAndCheck.md | 2 +- documentation/src/components/HomepageExamples.tsx | 2 +- documentation/src/components/QuickStart.tsx | 2 +- .../BulkSave/SqlServer/SqlServerSaveContextQuery.cs | 2 +- .../BulkSave/SqlServer/SqlServerUpdateContextQuery.cs | 2 +- .../Paillave.EntityFrameworkCoreExtension.csproj | 3 +-- .../Paillave.Etl.EntityFrameworkCore.csproj | 3 +-- src/Tutorials/SimpleTutorial/Program copy.cs | 2 +- src/Tutorials/SimpleTutorial/Program2.cs | 2 +- 11 files changed, 13 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e3412e3a..43194d27 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ using Paillave.Etl.FileSystem; using Paillave.Etl.Zip; using Paillave.Etl.TextFile; using Paillave.Etl.SqlServer; -using System.Data.SqlClient; +using Microsoft.Data.SqlClient; using System.Linq; namespace SimpleTutorial @@ -141,7 +141,7 @@ using Paillave.Etl.FileSystem; using Paillave.Etl.Zip; using Paillave.Etl.TextFile; using Paillave.Etl.SqlServer; -using System.Data.SqlClient; +using Microsoft.Data.SqlClient; namespace SimpleTutorial { diff --git a/documentation/docs/tutorials/2-defineProcess.md b/documentation/docs/tutorials/2-defineProcess.md index 6964d592..16a1372d 100644 --- a/documentation/docs/tutorials/2-defineProcess.md +++ b/documentation/docs/tutorials/2-defineProcess.md @@ -68,7 +68,7 @@ contextStream ## Setup the connection -By using `System.Data.SqlClient`, we create a connection to the database and we will inject it into the ETL process when triggering it. +By using `Microsoft.Data.SqlClient`, we create a connection to the database and we will inject it into the ETL process when triggering it. The extension that needs to operate with the database will get its connection through the DI setup here. @@ -200,7 +200,7 @@ using Paillave.Etl.FileSystem; using Paillave.Etl.Zip; using Paillave.Etl.TextFile; using Paillave.Etl.SqlServer; -using System.Data.SqlClient; +using Microsoft.Data.SqlClient; using Paillave.Etl.Core; namespace SimpleTutorial diff --git a/documentation/docs/tutorials/3-trackAndCheck.md b/documentation/docs/tutorials/3-trackAndCheck.md index 731d243f..914beb29 100644 --- a/documentation/docs/tutorials/3-trackAndCheck.md +++ b/documentation/docs/tutorials/3-trackAndCheck.md @@ -329,7 +329,7 @@ using Paillave.Etl.FileSystem; using Paillave.Etl.Zip; using Paillave.Etl.TextFile; using Paillave.Etl.SqlServer; -using System.Data.SqlClient; +using Microsoft.Data.SqlClient; namespace SimpleTutorial { diff --git a/documentation/src/components/HomepageExamples.tsx b/documentation/src/components/HomepageExamples.tsx index db334d73..67082f77 100644 --- a/documentation/src/components/HomepageExamples.tsx +++ b/documentation/src/components/HomepageExamples.tsx @@ -55,7 +55,7 @@ using Paillave.Etl.FileSystem; using Paillave.Etl.Zip; using Paillave.Etl.TextFile; using Paillave.Etl.SqlServer; -using System.Data.SqlClient; +using Microsoft.Data.SqlClient; namespace SimpleTutorial { diff --git a/documentation/src/components/QuickStart.tsx b/documentation/src/components/QuickStart.tsx index 863147a3..5a04a6f3 100644 --- a/documentation/src/components/QuickStart.tsx +++ b/documentation/src/components/QuickStart.tsx @@ -31,7 +31,7 @@ using Paillave.Etl.FileSystem; using Paillave.Etl.Zip; using Paillave.Etl.TextFile; using Paillave.Etl.SqlServer; -using System.Data.SqlClient; +using Microsoft.Data.SqlClient; using System.Linq; namespace SimpleTutorial diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerSaveContextQuery.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerSaveContextQuery.cs index e65396d2..796265b3 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerSaveContextQuery.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerSaveContextQuery.cs @@ -5,7 +5,7 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; -// using System.Data.SqlClient; +// using Microsoft.Data.SqlClient; using System.Linq; using System.Text; using System.Threading; diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerUpdateContextQuery.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerUpdateContextQuery.cs index c8900571..92fe88de 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerUpdateContextQuery.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerUpdateContextQuery.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Generic; using System.Data; -// using System.Data.SqlClient; +// using Microsoft.Data.SqlClient; using System.Linq; using System.Reflection; using System.Text; diff --git a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj index 42b88582..898577d4 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj +++ b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj @@ -12,7 +12,6 @@ - - + diff --git a/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj b/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj index 9f216a7f..309f6d97 100644 --- a/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj +++ b/src/Paillave.Etl.EntityFrameworkCore/Paillave.Etl.EntityFrameworkCore.csproj @@ -11,8 +11,7 @@ - - + diff --git a/src/Tutorials/SimpleTutorial/Program copy.cs b/src/Tutorials/SimpleTutorial/Program copy.cs index e72755db..e8030877 100644 --- a/src/Tutorials/SimpleTutorial/Program copy.cs +++ b/src/Tutorials/SimpleTutorial/Program copy.cs @@ -5,7 +5,7 @@ // using Paillave.Etl.Zip; // using Paillave.Etl.TextFile; // using Paillave.Etl.SqlServer; -// using System.Data.SqlClient; +// using Microsoft.Data.SqlClient; // using Paillave.Etl.ExecutionToolkit; // using System.Linq; // using System.IO; diff --git a/src/Tutorials/SimpleTutorial/Program2.cs b/src/Tutorials/SimpleTutorial/Program2.cs index b8e34352..3a9eeb88 100644 --- a/src/Tutorials/SimpleTutorial/Program2.cs +++ b/src/Tutorials/SimpleTutorial/Program2.cs @@ -5,7 +5,7 @@ // using Paillave.Etl.Zip; // using Paillave.Etl.TextFile; // using Paillave.Etl.SqlServer; -// using System.Data.SqlClient; +// using Microsoft.Data.SqlClient; // using System.Linq; // using Paillave.Etl.Bloomberg; From a5e80629ca608417a63100f82da00d6144effed1 Mon Sep 17 00:00:00 2001 From: Stephane Royer Date: Mon, 2 Dec 2024 13:46:01 +0100 Subject: [PATCH 7/9] feat: add Constants class and update Security properties to required; include new sample CSV and migration for schema changes --- .../BulkSave/BulkSaveEngineBase.cs | 4 +- .../BulkSave/Constants.cs | 6 + .../BulkSave/ObjectDataReader.cs | 6 +- .../.config/dotnet-tools.json | 5 +- .../DataAccess/Security.cs | 7 +- .../InputFiles/PortfoliosWeight.csv | 7 + .../20241202123241_Initial.Designer.cs | 276 ++++++++++++++++++ ...7_Initial.cs => 20241202123241_Initial.cs} | 14 +- ...cs => 20241202123950_Initial2.Designer.cs} | 12 +- .../Migrations/20241202123950_Initial2.cs | 108 +++++++ .../Migrations/TestDbContextModelSnapshot.cs | 8 +- 11 files changed, 425 insertions(+), 28 deletions(-) create mode 100644 src/Paillave.EntityFrameworkCoreExtension/BulkSave/Constants.cs create mode 100644 src/Tutorials/Paillave.Etl.Samples/InputFiles/PortfoliosWeight.csv create mode 100644 src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123241_Initial.Designer.cs rename src/Tutorials/Paillave.Etl.Samples/Migrations/{20241128161257_Initial.cs => 20241202123241_Initial.cs} (97%) rename src/Tutorials/Paillave.Etl.Samples/Migrations/{20241128161257_Initial.Designer.cs => 20241202123950_Initial2.Designer.cs} (96%) create mode 100644 src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123950_Initial2.cs diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngineBase.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngineBase.cs index 11f58b35..e7e48320 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngineBase.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngineBase.cs @@ -216,14 +216,14 @@ private void UpdateInputEntities(List propertiesToGetAfterSetInTarget .Where(j => !j.IsPrimaryKey() || string.Equals(GetDataReaderAction(resultDataRow, resultColumns), "INSERT", StringComparison.InvariantCultureIgnoreCase)) .Where(j => j.DeclaringType.ClrType.IsAssignableFrom(entry.Metadata.ClrType)) .Select(p => new { p.Name, Value = this.GetDataReaderValue(resultDataRow, resultColumns, p) }) - .Where(p => p.Value != DBNull.Value) + .Where(p => p.Value != Constants.DBNull) .ToDictionary(p => p.Name, p => p.Value); entry.CurrentValues.SetValues(dicoToSet); // TODO: do not update key if already exists entry.OriginalValues.SetValues(dicoToSet); foreach (var item in propertiesToGetAfterSetInTarget.Where(j => !j.IsShadowProperty() && j.DeclaringType.ClrType.IsAssignableFrom(entry.Metadata.ClrType)).ToList()) { var value = this.GetDataReaderValue(resultDataRow, resultColumns, item); - if (value != DBNull.Value) item.PropertyInfo.SetValue(inputEntity, value); + if (value != Constants.DBNull) item.PropertyInfo.SetValue(inputEntity, value); } } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/Constants.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/Constants.cs new file mode 100644 index 00000000..93b08e57 --- /dev/null +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/Constants.cs @@ -0,0 +1,6 @@ +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; + +public class Constants +{ + public static object? DBNull = System.DBNull.Value; +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/ObjectDataReader.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/ObjectDataReader.cs index 74c474ee..8e3ffbb9 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/ObjectDataReader.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/ObjectDataReader.cs @@ -217,15 +217,15 @@ public object this[string name] } else { - if (!_accessorsByType.TryGetValue(_currentType, out var acc)) return DBNull.Value; + if (!_accessorsByType.TryGetValue(_currentType, out var acc)) return Constants.DBNull; var val = acc[Current, name]; - if (val == null) return DBNull.Value; + if (val == null) return Constants.DBNull; if (_convertibleProperties.TryGetValue(name, out var converter)) return converter.ConvertToProvider(val); return val; } } } - public object this[int i] => this[_memberNames[i]] ?? DBNull.Value; + public object this[int i] => this[_memberNames[i]] ?? Constants.DBNull; } public class ObjectDataReaderConfig { diff --git a/src/Tutorials/Paillave.Etl.Samples/.config/dotnet-tools.json b/src/Tutorials/Paillave.Etl.Samples/.config/dotnet-tools.json index 0d1da733..4f487990 100644 --- a/src/Tutorials/Paillave.Etl.Samples/.config/dotnet-tools.json +++ b/src/Tutorials/Paillave.Etl.Samples/.config/dotnet-tools.json @@ -3,10 +3,11 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "6.0.1", + "version": "9.0.0", "commands": [ "dotnet-ef" - ] + ], + "rollForward": false } } } \ No newline at end of file diff --git a/src/Tutorials/Paillave.Etl.Samples/DataAccess/Security.cs b/src/Tutorials/Paillave.Etl.Samples/DataAccess/Security.cs index dd319c1a..5f997963 100644 --- a/src/Tutorials/Paillave.Etl.Samples/DataAccess/Security.cs +++ b/src/Tutorials/Paillave.Etl.Samples/DataAccess/Security.cs @@ -7,9 +7,9 @@ namespace Paillave.Etl.Samples.DataAccess public abstract class Security { public int Id { get; set; } - public string InternalCode { get; set; } - public string Isin { get; set; } - public string Name { get; set; } + public required string InternalCode { get; set; } + public string? Isin { get; set; } + public required string Name { get; set; } } public class SecurityConfiguration : IEntityTypeConfiguration { @@ -17,7 +17,6 @@ public void Configure(EntityTypeBuilder builder) { builder.ToTable(nameof(Security)); builder.HasKey(i => i.Id); - builder.Property(i => i.InternalCode).IsRequired(); builder.HasAlternateKey(i => i.InternalCode); builder.Property(i => i.Id).UseIdentityColumn(); } diff --git a/src/Tutorials/Paillave.Etl.Samples/InputFiles/PortfoliosWeight.csv b/src/Tutorials/Paillave.Etl.Samples/InputFiles/PortfoliosWeight.csv new file mode 100644 index 00000000..cf327f90 --- /dev/null +++ b/src/Tutorials/Paillave.Etl.Samples/InputFiles/PortfoliosWeight.csv @@ -0,0 +1,7 @@ +Code,Date,Weight +P1,"Tue 20 Oct, 2020",33 +P1,"Wed 21 Oct, 2020",36 +P2,"Tue 20 Oct, 2020",126 +P2,"Wed 21 Oct, 2020",145 +P3,"Tue 20 Oct, 2020",105 +P3,"Wed 21 Oct, 2020",36 diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123241_Initial.Designer.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123241_Initial.Designer.cs new file mode 100644 index 00000000..402516b2 --- /dev/null +++ b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123241_Initial.Designer.cs @@ -0,0 +1,276 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Paillave.Etl.Samples.DataAccess; + +#nullable disable + +namespace Paillave.Etl.Samples.Migrations +{ + [DbContext(typeof(TestDbContext))] + [Migration("20241202123241_Initial")] + partial class Initial + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Composition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Date") + .HasColumnType("DATE"); + + b.Property("PortfolioId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasAlternateKey("Date", "PortfolioId"); + + b.HasIndex("PortfolioId"); + + b.ToTable("Composition", (string)null); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Portfolio", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("InternalCode") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("SicavId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasAlternateKey("InternalCode"); + + b.HasIndex("SicavId"); + + b.ToTable("Portfolio", (string)null); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Position", b => + { + b.Property("CompositionId") + .HasColumnType("int"); + + b.Property("SecurityId") + .HasColumnType("int"); + + b.Property("Value") + .HasColumnType("decimal(18,2)"); + + b.HasKey("CompositionId", "SecurityId"); + + b.HasIndex("SecurityId"); + + b.ToTable("Position", (string)null); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Security", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Discriminator") + .IsRequired() + .HasMaxLength(13) + .HasColumnType("nvarchar(13)"); + + b.Property("InternalCode") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Isin") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasAlternateKey("InternalCode"); + + b.ToTable("Security", (string)null); + + b.HasDiscriminator().HasValue("Security"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Sicav", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("InternalCode") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasAlternateKey("InternalCode"); + + b.ToTable("Sicav", (string)null); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTable", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("SimpleTable", (string)null); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTableRelated", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("SimpleTableId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SimpleTableId"); + + b.ToTable("SimpleTableRelated", (string)null); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Equity", b => + { + b.HasBaseType("Paillave.Etl.Samples.DataAccess.Security"); + + b.Property("Issuer") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasDiscriminator().HasValue("Equity"); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.ShareClass", b => + { + b.HasBaseType("Paillave.Etl.Samples.DataAccess.Security"); + + b.Property("Class") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasDiscriminator().HasValue("ShareClass"); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Composition", b => + { + b.HasOne("Paillave.Etl.Samples.DataAccess.Portfolio", "Portfolio") + .WithMany() + .HasForeignKey("PortfolioId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Portfolio"); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Portfolio", b => + { + b.HasOne("Paillave.Etl.Samples.DataAccess.Sicav", "Sicav") + .WithMany() + .HasForeignKey("SicavId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Sicav"); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Position", b => + { + b.HasOne("Paillave.Etl.Samples.DataAccess.Composition", "Composition") + .WithMany("Positions") + .HasForeignKey("CompositionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Paillave.Etl.Samples.DataAccess.Security", "Security") + .WithMany() + .HasForeignKey("SecurityId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Composition"); + + b.Navigation("Security"); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTableRelated", b => + { + b.HasOne("Paillave.Etl.Samples.DataAccess.SimpleTable", null) + .WithMany("Relateds") + .HasForeignKey("SimpleTableId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Composition", b => + { + b.Navigation("Positions"); + }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTable", b => + { + b.Navigation("Relateds"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123241_Initial.cs similarity index 97% rename from src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.cs rename to src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123241_Initial.cs index be6da18f..07eca31b 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123241_Initial.cs @@ -18,9 +18,9 @@ protected override void Up(MigrationBuilder migrationBuilder) Id = table.Column(type: "int", nullable: false) .Annotation("SqlServer:Identity", "1, 1"), InternalCode = table.Column(type: "nvarchar(450)", nullable: false), - Isin = table.Column(type: "nvarchar(max)", nullable: false), - Name = table.Column(type: "nvarchar(max)", nullable: false), - Discriminator = table.Column(type: "nvarchar(max)", nullable: false), + Isin = table.Column(type: "nvarchar(max)", nullable: true), + Name = table.Column(type: "nvarchar(max)", nullable: true), + Discriminator = table.Column(type: "nvarchar(13)", maxLength: 13, nullable: false), Issuer = table.Column(type: "nvarchar(max)", nullable: true), Class = table.Column(type: "nvarchar(max)", nullable: true) }, @@ -37,7 +37,7 @@ protected override void Up(MigrationBuilder migrationBuilder) Id = table.Column(type: "int", nullable: false) .Annotation("SqlServer:Identity", "1, 1"), InternalCode = table.Column(type: "nvarchar(450)", nullable: false), - Name = table.Column(type: "nvarchar(max)", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: true), Type = table.Column(type: "nvarchar(max)", nullable: true) }, constraints: table => @@ -52,7 +52,7 @@ protected override void Up(MigrationBuilder migrationBuilder) { Id = table.Column(type: "int", nullable: false) .Annotation("SqlServer:Identity", "1, 1"), - Name = table.Column(type: "nvarchar(max)", nullable: false) + Name = table.Column(type: "nvarchar(max)", nullable: true) }, constraints: table => { @@ -66,7 +66,7 @@ protected override void Up(MigrationBuilder migrationBuilder) Id = table.Column(type: "int", nullable: false) .Annotation("SqlServer:Identity", "1, 1"), InternalCode = table.Column(type: "nvarchar(450)", nullable: false), - Name = table.Column(type: "nvarchar(max)", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: true), SicavId = table.Column(type: "int", nullable: false) }, constraints: table => @@ -87,7 +87,7 @@ protected override void Up(MigrationBuilder migrationBuilder) { Id = table.Column(type: "int", nullable: false) .Annotation("SqlServer:Identity", "1, 1"), - Name = table.Column(type: "nvarchar(max)", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: true), SimpleTableId = table.Column(type: "int", nullable: false) }, constraints: table => diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.Designer.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123950_Initial2.Designer.cs similarity index 96% rename from src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.Designer.cs rename to src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123950_Initial2.Designer.cs index 305a0c7f..f3987962 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.Designer.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123950_Initial2.Designer.cs @@ -12,15 +12,15 @@ namespace Paillave.Etl.Samples.Migrations { [DbContext(typeof(TestDbContext))] - [Migration("20241128161257_Initial")] - partial class Initial + [Migration("20241202123950_Initial2")] + partial class Initial2 { /// protected override void BuildTargetModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "7.0.8") + .HasAnnotation("ProductVersion", "9.0.0") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); @@ -104,14 +104,14 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Discriminator") .IsRequired() - .HasColumnType("nvarchar(max)"); + .HasMaxLength(13) + .HasColumnType("nvarchar(13)"); b.Property("InternalCode") .IsRequired() .HasColumnType("nvarchar(450)"); b.Property("Isin") - .IsRequired() .HasColumnType("nvarchar(max)"); b.Property("Name") @@ -124,7 +124,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("Security", (string)null); - b.HasDiscriminator("Discriminator").HasValue("Security"); + b.HasDiscriminator().HasValue("Security"); b.UseTphMappingStrategy(); }); diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123950_Initial2.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123950_Initial2.cs new file mode 100644 index 00000000..baed3a8f --- /dev/null +++ b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241202123950_Initial2.cs @@ -0,0 +1,108 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Paillave.Etl.Samples.Migrations +{ + /// + public partial class Initial2 : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "Name", + table: "SimpleTableRelated", + type: "nvarchar(max)", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "nvarchar(max)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "Name", + table: "SimpleTable", + type: "nvarchar(max)", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "nvarchar(max)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "Name", + table: "Sicav", + type: "nvarchar(max)", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "nvarchar(max)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "Name", + table: "Security", + type: "nvarchar(max)", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "nvarchar(max)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "Name", + table: "Portfolio", + type: "nvarchar(max)", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "nvarchar(max)", + oldNullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "Name", + table: "SimpleTableRelated", + type: "nvarchar(max)", + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + + migrationBuilder.AlterColumn( + name: "Name", + table: "SimpleTable", + type: "nvarchar(max)", + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + + migrationBuilder.AlterColumn( + name: "Name", + table: "Sicav", + type: "nvarchar(max)", + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + + migrationBuilder.AlterColumn( + name: "Name", + table: "Security", + type: "nvarchar(max)", + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + + migrationBuilder.AlterColumn( + name: "Name", + table: "Portfolio", + type: "nvarchar(max)", + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + } + } +} diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs index afd880b5..b018bbaf 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs @@ -17,7 +17,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "7.0.8") + .HasAnnotation("ProductVersion", "9.0.0") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); @@ -101,14 +101,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Discriminator") .IsRequired() - .HasColumnType("nvarchar(max)"); + .HasMaxLength(13) + .HasColumnType("nvarchar(13)"); b.Property("InternalCode") .IsRequired() .HasColumnType("nvarchar(450)"); b.Property("Isin") - .IsRequired() .HasColumnType("nvarchar(max)"); b.Property("Name") @@ -121,7 +121,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("Security", (string)null); - b.HasDiscriminator("Discriminator").HasValue("Security"); + b.HasDiscriminator().HasValue("Security"); b.UseTphMappingStrategy(); }); From 013a68631234002f64398e99b4b4b8458e0d259c Mon Sep 17 00:00:00 2001 From: Stephane Royer Date: Mon, 2 Dec 2024 16:07:59 +0100 Subject: [PATCH 8/9] refactor: enable nullable reference types and clean up interface definitions in Searcher namespace --- .../BulkSave/BulkSaveEngine.cs | 27 +- .../BulkSave/BulkSaveEngineBase.cs | 427 ++++++++---------- .../BulkSave/BulkUpdateEngine.cs | 27 +- .../BulkSave/BulkUpdateEngineBase.cs | 153 +++---- .../BulkSave/DbContextBulkExtensions.cs | 36 +- .../BulkSave/LambdaEqualityComparer.cs | 83 ++-- .../BulkSave/ObjectDataReader.cs | 384 ++++++++-------- .../BulkSave/SaveContextQueryBase.cs | 158 ++++--- .../BulkSave/SettersExtractor.cs | 41 +- .../SqlServer/SqlServerSaveContextQuery.cs | 259 ++++++----- .../SqlServer/SqlServerUpdateContextQuery.cs | 120 +++-- .../BulkSave/UpdateContextQueryBase.cs | 102 ++--- .../EntityTypeBuilderEx.cs | 6 +- ...illave.EntityFrameworkCoreExtension.csproj | 1 + .../Searcher/FieldSelector.cs | 26 +- .../Searcher/GroupingValue.cs | 25 +- .../Searcher/IFieldSelector.cs | 15 +- .../Searcher/INavigationSelector.cs | 15 +- .../Searcher/ISearchDescriptor.cs | 27 +- .../Searcher/NavigationSelector.cs | 25 +- .../Searcher/SearchDescriptorBase.cs | 379 ++++++++-------- .../Searcher/SearchMetadata.cs | 31 +- 22 files changed, 1136 insertions(+), 1231 deletions(-) diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngine.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngine.cs index e67dc836..7bdafc31 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngine.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngine.cs @@ -1,28 +1,25 @@ using System; using System.Collections.Generic; using System.Linq.Expressions; -using System.Text; using System.Threading; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata; -// using Paillave.EntityFrameworkCoreExtension.BulkSave.Postgres; using Paillave.EntityFrameworkCoreExtension.BulkSave.SqlServer; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; + +public class BulkSaveEngine : BulkSaveEngineBase where T : class { - public class BulkSaveEngine : BulkSaveEngineBase where T : class + public BulkSaveEngine(DbContext context, params Expression>[] pivotKeys) : base(context, pivotKeys) { - public BulkSaveEngine(DbContext context, params Expression>[] pivotKeys) : base(context, pivotKeys) - { - } + } - protected override SaveContextQueryBase CreateSaveContextQueryInstance(DbContext context, string schema, string table, List propertiesToInsert, List propertiesToUpdate, List> propertiesForPivotSet, List propertiesToBulkLoad, List entityTypes, CancellationToken cancellationToken) - { - if (context.Database.IsSqlServer()) - return new SqlServerSaveContextQuery(context, schema, table, propertiesToInsert, propertiesToUpdate, propertiesForPivotSet, propertiesToBulkLoad, entityTypes, cancellationToken, base.StoreObject); - // if (context.Database.IsNpgsql()) - // return new PostgresSaveContextQuery(context, schema, table, propertiesToInsert, propertiesToUpdate, propertiesForPivotSet, propertiesToBulkLoad, entityTypes); - throw new Exception("unsupported provider"); - } + protected override SaveContextQueryBase CreateSaveContextQueryInstance(DbContext context, string schema, string table, List propertiesToInsert, List propertiesToUpdate, List> propertiesForPivotSet, List propertiesToBulkLoad, List entityTypes, CancellationToken cancellationToken) + { + if (context.Database.IsSqlServer()) + return new SqlServerSaveContextQuery(context, schema, table, propertiesToInsert, propertiesToUpdate, propertiesForPivotSet, propertiesToBulkLoad, entityTypes, cancellationToken, base.StoreObject); + // if (context.Database.IsNpgsql()) + // return new PostgresSaveContextQuery(context, schema, table, propertiesToInsert, propertiesToUpdate, propertiesForPivotSet, propertiesToBulkLoad, entityTypes); + throw new Exception("unsupported provider"); } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngineBase.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngineBase.cs index e7e48320..51c62a36 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngineBase.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkSaveEngineBase.cs @@ -1,278 +1,245 @@ using System; using System.Collections.Generic; using System.Data; -using System.Data.Common; using System.Linq; using System.Linq.Expressions; using System.Threading; -using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata; using Paillave.EntityFrameworkCoreExtension.ContextMetadata; using Paillave.EntityFrameworkCoreExtension.Core; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; + +public abstract class BulkSaveEngineBase : IDisposable where T : class { - public abstract class BulkSaveEngineBase : IDisposable where T : class + private List _propertiesToInsert; // any column except computed + private List _propertiesToUpdate; // any column except pivot, computed + // private HashSet _propertiesNotToBeUpdatedToNull = new HashSet(); + private List> _propertiesForPivotSet; // pivot columns + // private List _propertiesForPivot; // pivot columns + private List _propertiesToGetAfterSetInTarget; // computed, and with default value column + private List _propertiesToBulkLoad; // any column except computed that is not pivot + private List _entityTypes; + + protected StoreObjectIdentifier StoreObject { get; } + private string _schema; + private string _table; + private bool disposedValue; + private readonly DbContext _context; + protected abstract SaveContextQueryBase CreateSaveContextQueryInstance(DbContext context, string schema, string table, List propertiesToInsert, List propertiesToUpdate, List> propertiesForPivotSet, List propertiesToBulkLoad, List entityTypes, CancellationToken cancellationToken); + private IEnumerable GetAllRelatedEntityTypes(IEntityType et) { - private List _propertiesToInsert; // any column except computed - private List _propertiesToUpdate; // any column except pivot, computed - // private HashSet _propertiesNotToBeUpdatedToNull = new HashSet(); - private List> _propertiesForPivotSet; // pivot columns - // private List _propertiesForPivot; // pivot columns - private List _propertiesToGetAfterSetInTarget; // computed, and with default value column - private List _propertiesToBulkLoad; // any column except computed that is not pivot - private List _entityTypes; - - protected StoreObjectIdentifier StoreObject { get; } - private string _schema; - private string _table; - private bool disposedValue; - private readonly DbContext _context; - protected abstract SaveContextQueryBase CreateSaveContextQueryInstance(DbContext context, string schema, string table, List propertiesToInsert, List propertiesToUpdate, List> propertiesForPivotSet, List propertiesToBulkLoad, List entityTypes, CancellationToken cancellationToken); - private IEnumerable GetAllRelatedEntityTypes(IEntityType et) + yield return et; + foreach (var item in et.GetDerivedTypes()) + foreach (var i in GetAllRelatedEntityTypes(item)) + yield return i; + } + private List GetPropertiesForPivot(List allProperties, IProperty? tenantIdProp, Expression> pivotKey) + { + var pivotKeyNames = KeyDefinitionExtractor.GetKeys(pivotKey).Select(i => i.Name).ToList(); + if (tenantIdProp != null) { - yield return et; - foreach (var item in et.GetDerivedTypes()) - foreach (var i in GetAllRelatedEntityTypes(item)) - yield return i; - // var allNavigationProperties = subClasses.SelectMany(et => et.GetNavigations()).Where(a => a.GetTargetType().IsOwned()).Distinct(); - // foreach (var navigationProperty in allNavigationProperties) - // { - // var property = navigationProperty.PropertyInfo; - // var ownedEntityType = context.Model.FindEntityType(property.PropertyType); - - // if (ownedEntityType == null) // when entity has more then one ownedType (e.g. Address HomeAddress, Address WorkAddress) or one ownedType is in multiple Entities like Audit is usually. - // ownedEntityType = context.Model.GetEntityTypes().SingleOrDefault(a => a.DefiningNavigationName == property.Name && a.DefiningEntityType.Name == entityType.Name); - - - // var ownedEntityPropertyNameColumnNameDict = ownedEntityType.GetProperties() - // .Where(ownedEntityProperty => !ownedEntityProperty.IsPrimaryKey()) - // .ToDictionary(ownedEntityProperty => ownedEntityProperty.Name, ownedEntityProperty => ownedEntityProperty.Relational().ColumnName); - - // foreach (var ownedProperty in property.PropertyType.GetProperties()) - // { - // if (ownedEntityPropertyNameColumnNameDict.ContainsKey(ownedProperty.Name)) - // { - // string columnName = ownedEntityPropertyNameColumnNameDict[ownedProperty.Name]; - // var ownedPropertyType = Nullable.GetUnderlyingType(ownedProperty.PropertyType) ?? ownedProperty.PropertyType; - // PropertyColumnNamesDict.Add(property.Name + "." + ownedProperty.Name, columnName); - // OutputPropertyColumnNamesDict.Add(property.Name + "." + ownedProperty.Name, columnName); - // } - // } - // } + var columnName = tenantIdProp.GetColumnName(StoreObject); + if (string.IsNullOrWhiteSpace(columnName)) + throw new InvalidOperationException($"The property {tenantIdProp.Name} does not have a column name defined"); + pivotKeyNames.Add(columnName); } - private List GetPropertiesForPivot(List allProperties, IProperty tenantIdProp, Expression> pivotKey) + return allProperties.Where(i => pivotKeyNames.Contains(i.Name)).ToList(); + } + public BulkSaveEngineBase(DbContext context, params Expression>[]? pivotKeys) + { + List dbComputedProperties; + List defaultValuesProperties; + List notPivotComputedProperties; + this._context = context; + + var entityType = context.Model.FindEntityType(typeof(T)); + if (entityType == null) + throw new InvalidOperationException("DbContext does not contain EntitySet for Type: " + typeof(T).Name); + StoreObject = StoreObjectIdentifier.Create(entityType, StoreObjectType.Table).GetValueOrDefault(); + var summary = entityType.GetEntityEssentials(); + _schema = summary.Schema; + var table = entityType.GetTableName(); + if (string.IsNullOrWhiteSpace(table)) + throw new InvalidOperationException("Table name is not defined"); + _table = table; + + _entityTypes = GetAllRelatedEntityTypes(entityType).Distinct().ToList(); + var allProperties = _entityTypes + .SelectMany(dt => dt.GetProperties()) + .DistinctBy(i => i.GetColumnName(StoreObject)) + .ToList(); + + if (pivotKeys == null || pivotKeys.Length == 0) { - var pivotKeyNames = KeyDefinitionExtractor.GetKeys(pivotKey).Select(i => i.Name).ToList(); - if (tenantIdProp != null) - pivotKeyNames.Add(tenantIdProp.GetColumnName(StoreObject)); - return allProperties.Where(i => pivotKeyNames.Contains(i.Name)).ToList(); + var primaryKey = entityType.FindPrimaryKey(); + if (primaryKey != null) + _propertiesForPivotSet = new[] { primaryKey.Properties.ToList() }.ToList(); + else + _propertiesForPivotSet = new List>(); } - public BulkSaveEngineBase(DbContext context, params Expression>[] pivotKeys) + else { - List dbComputedProperties; - List defaultValuesProperties; - List notPivotComputedProperties; - this._context = context; - - var entityType = context.Model.FindEntityType(typeof(T)); - StoreObject = StoreObjectIdentifier.Create(entityType, StoreObjectType.Table).GetValueOrDefault(); - if (entityType == null) - throw new InvalidOperationException("DbContext does not contain EntitySet for Type: " + typeof(T).Name); - var summary = entityType.GetEntityEssentials(); - _schema = summary.Schema; - _table = entityType.GetTableName(); + IProperty? tenantIdProp = null; - _entityTypes = GetAllRelatedEntityTypes(entityType).Distinct().ToList(); - var allProperties = _entityTypes.SelectMany(dt => dt.GetProperties()).Distinct(new LambdaEqualityComparer(i => i.GetColumnName(StoreObject))).ToList(); - - if (pivotKeys == null || pivotKeys.Length == 0) + if (typeof(IMultiTenantEntity).IsAssignableFrom(typeof(T))) { - _propertiesForPivotSet = new[] { entityType.FindPrimaryKey().Properties.ToList() }.ToList(); + tenantIdProp = allProperties.First(i => i.Name == nameof(IMultiTenantEntity.TenantId)); } - else - { - IProperty tenantIdProp = null; - if (typeof(IMultiTenantEntity).IsAssignableFrom(typeof(T))) - { - tenantIdProp = allProperties.First(i => i.Name == nameof(IMultiTenantEntity.TenantId)); - } - - _propertiesForPivotSet = pivotKeys.Select(pivotKey => this.GetPropertiesForPivot(allProperties, tenantIdProp, pivotKey)).ToList(); - } + _propertiesForPivotSet = pivotKeys.Select(pivotKey => this.GetPropertiesForPivot(allProperties, tenantIdProp, pivotKey)).ToList(); + } - //identityProperty = entityType.GetProperties().FirstOrDefault(i => i.SqlServer().ValueGenerationStrategy == SqlServerValueGenerationStrategy.IdentityColumn); - // string timestampDbTypeName = nameof(TimestampAttribute).Replace("Attribute", "").ToLower(); // = "timestamp"; - // dbComputedProperties = allProperties.Where(i => i.GetValueGenerationStrategy() != null).ToList(); - // computedProperties = allProperties.Where(i => (i.ValueGenerated & ValueGenerated.OnAddOrUpdate) != ValueGenerated.Never).ToList(); - // allProperties[0].GetDefaultValueSql() - dbComputedProperties = allProperties.Where(i => (i.ValueGenerated & ValueGenerated.OnAddOrUpdate) != ValueGenerated.Never).ToList(); - notPivotComputedProperties = dbComputedProperties.Except(_propertiesForPivotSet.SelectMany(i => i)).ToList(); + dbComputedProperties = allProperties.Where(i => (i.ValueGenerated & ValueGenerated.OnAddOrUpdate) != ValueGenerated.Never).ToList(); + notPivotComputedProperties = dbComputedProperties.Except(_propertiesForPivotSet.SelectMany(i => i)).ToList(); - defaultValuesProperties = allProperties.Where(i => i.GetDefaultValueSql() != null).ToList(); + defaultValuesProperties = allProperties.Where(i => i.GetDefaultValueSql() != null).ToList(); - _propertiesToBulkLoad = allProperties.Except(notPivotComputedProperties).ToList(); + _propertiesToBulkLoad = allProperties.Except(notPivotComputedProperties).ToList(); - _propertiesToInsert = allProperties - .Except(dbComputedProperties) - //.Except(new[] { identityProperty }) - .ToList(); - _propertiesToUpdate = allProperties - // .Except(_propertiesForPivot) - .Except(dbComputedProperties) - //.Except(new[] { identityProperty }) - .ToList(); - _propertiesToGetAfterSetInTarget = dbComputedProperties - //.Union(new[] { identityProperty }) - .Union(defaultValuesProperties) - .Distinct(new LambdaEqualityComparer(i => i.GetColumnName(StoreObject))) - .ToList(); - } - public void Save(IList entities, CancellationToken cancellationToken, bool doNotUpdateIfExists = false, bool insertOnly = false) + _propertiesToInsert = allProperties + .Except(dbComputedProperties) + .ToList(); + _propertiesToUpdate = allProperties + .Except(dbComputedProperties) + .ToList(); + _propertiesToGetAfterSetInTarget = dbComputedProperties + .Union(defaultValuesProperties) + .DistinctBy(i => i.GetColumnName(StoreObject)) + .ToList(); + } + public void Save(IList entities, CancellationToken cancellationToken, bool doNotUpdateIfExists = false, bool insertOnly = false) + { + if (entities.Count == 0) return; + var previousAutoDetect = _context.ChangeTracker.AutoDetectChangesEnabled; + _context.ChangeTracker.AutoDetectChangesEnabled = false; + this.TrySetTenant(entities); + var contextQuery = this.CreateSaveContextQueryInstance(_context, _schema, _table, _propertiesToInsert, _propertiesToUpdate, _propertiesForPivotSet, _propertiesToBulkLoad, _entityTypes, cancellationToken); + SetDiscriminatorValue(entities); + bool outputStagingTableCreated = false; + contextQuery.CreateStagingTable(); + try { - if (entities.Count == 0) return; - var previousAutoDetect = _context.ChangeTracker.AutoDetectChangesEnabled; - _context.ChangeTracker.AutoDetectChangesEnabled = false; - this.TrySetTenant(entities); - var contextQuery = this.CreateSaveContextQueryInstance(_context, _schema, _table, _propertiesToInsert, _propertiesToUpdate, _propertiesForPivotSet, _propertiesToBulkLoad, _entityTypes, cancellationToken); - SetDiscriminatorValue(entities); - bool outputStagingTableCreated = false; - contextQuery.CreateStagingTable(); - try - { - contextQuery.BulkSaveInStaging(entities); - if (!insertOnly && this.ShouldIndexStagingTable(entities.Count())) - foreach (var propertiesForPivot in this._propertiesForPivotSet) - contextQuery.IndexStagingTable(propertiesForPivot); - - contextQuery.CreateOutputStagingTable(); - outputStagingTableCreated = true; - if (insertOnly) - contextQuery.InsertFromStaging(); - else - contextQuery.MergeFromStaging(doNotUpdateIfExists); - if (this.ShouldIndexStagingTable(entities.Count())) - contextQuery.IndexOutputStagingTable(); + contextQuery.BulkSaveInStaging(entities); + if (!insertOnly && this.ShouldIndexStagingTable(entities.Count())) + foreach (var propertiesForPivot in this._propertiesForPivotSet) + contextQuery.IndexStagingTable(propertiesForPivot); + + contextQuery.CreateOutputStagingTable(); + outputStagingTableCreated = true; + if (insertOnly) + contextQuery.InsertFromStaging(); + else + contextQuery.MergeFromStaging(doNotUpdateIfExists); + if (this.ShouldIndexStagingTable(entities.Count())) + contextQuery.IndexOutputStagingTable(); - DataTable resultEntities; - if (_propertiesToUpdate.Count > 0) - resultEntities = contextQuery.GetOutputStaging(); - else - resultEntities = contextQuery.GetOutputStagingForComputedColumns(); - UpdateInputEntities(_propertiesToGetAfterSetInTarget, entities, resultEntities); + DataTable resultEntities; + if (_propertiesToUpdate.Count > 0) + resultEntities = contextQuery.GetOutputStaging(); + else + resultEntities = contextQuery.GetOutputStagingForComputedColumns(); + UpdateInputEntities(_propertiesToGetAfterSetInTarget, entities, resultEntities); - _context.ChangeTracker.AutoDetectChangesEnabled = previousAutoDetect; - } - finally - { - contextQuery.DeleteStagingTable(); - if (outputStagingTableCreated) - contextQuery.DeleteOutputStagingTable(); - } + _context.ChangeTracker.AutoDetectChangesEnabled = previousAutoDetect; } - protected virtual bool ShouldIndexStagingTable(int entitiesCount) => entitiesCount >= 10000; - private void TrySetTenant(IList entities) + finally { - if (this._context is MultiTenantDbContext mtCtx) - { - mtCtx.UpdateEntitiesForMultiTenancy(entities); - } + contextQuery.DeleteStagingTable(); + if (outputStagingTableCreated) + contextQuery.DeleteOutputStagingTable(); } - - private void SetDiscriminatorValue(IList entities) + } + protected virtual bool ShouldIndexStagingTable(int entitiesCount) => entitiesCount >= 10000; + private void TrySetTenant(IList entities) + { + if (this._context is MultiTenantDbContext mtCtx) { - if (_entityTypes.All(i => i.FindDiscriminatorProperty() == null)) return; - var discriminatorsDictionary = _entityTypes - .Select(i => new - { - i.ClrType, - EntityType = i, - DiscriminatorProperty = i.FindDiscriminatorProperty(), - DiscriminatorValue = i.FindDiscriminatorProperty() != null ? i.GetDiscriminatorValue() : null - // RelationalProperty = i.Relational() - }) - .Where(i => i.DiscriminatorProperty != null) - .ToDictionary( - i => i.ClrType, - i => new Dictionary(new[] { new KeyValuePair(i.DiscriminatorProperty.Name, i.DiscriminatorValue) })); - foreach (var entity in entities) - { - var type = entity.GetType(); - var relationalProperty = discriminatorsDictionary[type]; - this._context.Entry(entity).CurrentValues.SetValues(relationalProperty); - } + mtCtx.UpdateEntitiesForMultiTenancy(entities); } - private void UpdateInputEntities(List propertiesToGetAfterSetInTarget, IList inputEntities, DataTable resultEntities) - { - var resultColumns = resultEntities.Columns.OfType().ToDictionary(i => i.ColumnName.ToLower()); - for (int i = 0; i < inputEntities.Count; i++) + } + + private void SetDiscriminatorValue(IList entities) + { + if (_entityTypes.All(i => i.FindDiscriminatorProperty() == null)) return; + var discriminatorsDictionary = _entityTypes + .Select(i => new { - var inputEntity = inputEntities[i]; - var resultDataRow = resultEntities.Rows[i]; - var entry = _context.Entry(inputEntity); - var dicoToSet = propertiesToGetAfterSetInTarget - .Where(j => !j.IsPrimaryKey() || string.Equals(GetDataReaderAction(resultDataRow, resultColumns), "INSERT", StringComparison.InvariantCultureIgnoreCase)) - .Where(j => j.DeclaringType.ClrType.IsAssignableFrom(entry.Metadata.ClrType)) - .Select(p => new { p.Name, Value = this.GetDataReaderValue(resultDataRow, resultColumns, p) }) - .Where(p => p.Value != Constants.DBNull) - .ToDictionary(p => p.Name, p => p.Value); - entry.CurrentValues.SetValues(dicoToSet); // TODO: do not update key if already exists - entry.OriginalValues.SetValues(dicoToSet); - foreach (var item in propertiesToGetAfterSetInTarget.Where(j => !j.IsShadowProperty() && j.DeclaringType.ClrType.IsAssignableFrom(entry.Metadata.ClrType)).ToList()) - { - var value = this.GetDataReaderValue(resultDataRow, resultColumns, item); - if (value != Constants.DBNull) item.PropertyInfo.SetValue(inputEntity, value); - } - } - } - private string GetDataReaderAction(DataRow dataRow, Dictionary dataColumns) + i.ClrType, + EntityType = i, + DiscriminatorProperty = i.FindDiscriminatorProperty(), + DiscriminatorValue = i.FindDiscriminatorProperty() != null ? i.GetDiscriminatorValue() : null + // RelationalProperty = i.Relational() + }) + .Where(i => i.DiscriminatorProperty != null) + .ToDictionary( + i => i.ClrType, + i => new Dictionary(new[] { new KeyValuePair(i.DiscriminatorProperty!.Name, i.DiscriminatorValue ?? throw new Exception("no DiscriminatorValue")) })); + foreach (var entity in entities) { - var columnName = "_Action"; - var dataColumn = dataColumns[columnName.ToLower()]; - return dataRow.ItemArray[dataColumn.Ordinal] as string; + var type = entity.GetType(); + var relationalProperty = discriminatorsDictionary[type]; + this._context.Entry(entity).CurrentValues.SetValues(relationalProperty); } - private object GetDataReaderValue(DataRow dataRow, Dictionary dataColumns, IProperty property) + } + private void UpdateInputEntities(List propertiesToGetAfterSetInTarget, IList inputEntities, DataTable resultEntities) + { + var resultColumns = resultEntities.Columns.OfType().ToDictionary(i => i.ColumnName.ToLower()); + for (int i = 0; i < inputEntities.Count; i++) { - var columnName = property.GetColumnName(StoreObject); - var dataColumn = dataColumns[columnName.ToLower()]; - var dataRowValue = dataRow.ItemArray[dataColumn.Ordinal]; - var converter = property.GetTypeMapping().Converter; - if (converter == null) + var inputEntity = inputEntities[i]; + var resultDataRow = resultEntities.Rows[i]; + var entry = _context.Entry(inputEntity); + var dicoToSet = propertiesToGetAfterSetInTarget + .Where(j => !j.IsPrimaryKey() || string.Equals(GetDataReaderAction(resultDataRow, resultColumns), "INSERT", StringComparison.InvariantCultureIgnoreCase)) + .Where(j => j.DeclaringType.ClrType.IsAssignableFrom(entry.Metadata.ClrType)) + .Select(p => new { p.Name, Value = this.GetDataReaderValue(resultDataRow, resultColumns, p) }) + .Where(p => p.Value != Constants.DBNull) + .ToDictionary(p => p.Name, p => p.Value); + entry.CurrentValues.SetValues(dicoToSet); // TODO: do not update key if already exists + entry.OriginalValues.SetValues(dicoToSet); + foreach (var item in propertiesToGetAfterSetInTarget.Where(j => !j.IsShadowProperty() && j.DeclaringType.ClrType.IsAssignableFrom(entry.Metadata.ClrType)).ToList()) { - return dataRowValue; + var value = this.GetDataReaderValue(resultDataRow, resultColumns, item); + if (item.PropertyInfo == null) + throw new InvalidOperationException($"The property {item.Name} does not have a property info defined"); + if (value != Constants.DBNull) item.PropertyInfo.SetValue(inputEntity, value); } - return converter.ConvertFromProvider(dataRowValue); } - - protected virtual void Dispose(bool disposing) + } + private string? GetDataReaderAction(DataRow dataRow, Dictionary dataColumns) + { + var columnName = "_Action"; + var dataColumn = dataColumns[columnName.ToLower()]; + return dataRow.ItemArray[dataColumn.Ordinal] as string; + } + private object? GetDataReaderValue(DataRow dataRow, Dictionary dataColumns, IProperty property) + { + var columnName = property.GetColumnName(StoreObject); + if (string.IsNullOrWhiteSpace(columnName)) + throw new InvalidOperationException($"The property {property.Name} does not have a column name defined"); + var dataColumn = dataColumns[columnName.ToLower()]; + var dataRowValue = dataRow.ItemArray[dataColumn.Ordinal]; + var converter = property.GetTypeMapping().Converter; + if (converter == null) { - if (!disposedValue) - { - if (disposing) - { - // TODO: dispose managed state (managed objects) - } - - // TODO: free unmanaged resources (unmanaged objects) and override finalizer - // TODO: set large fields to null - disposedValue = true; - } + return dataRowValue; } + return converter.ConvertFromProvider(dataRowValue); + } - // // TODO: override finalizer only if 'Dispose(bool disposing)' has code to free unmanaged resources - // ~BulkSaveEngineBase() - // { - // // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - // Dispose(disposing: false); - // } - - public void Dispose() + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); + disposedValue = true; } } + + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkUpdateEngine.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkUpdateEngine.cs index 9d4185b4..e89b60f9 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkUpdateEngine.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkUpdateEngine.cs @@ -2,26 +2,23 @@ using System.Collections.Generic; using System.Linq.Expressions; using System.Reflection; -using System.Text; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata; -// using Paillave.EntityFrameworkCoreExtension.BulkSave.Postgres; using Paillave.EntityFrameworkCoreExtension.BulkSave.SqlServer; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; + +public class BulkUpdateEngine : BulkUpdateEngineBase where TEntity : class { - public class BulkUpdateEngine : BulkUpdateEngineBase where TEntity : class - { - public BulkUpdateEngine(DbContext context, Expression> updateKey, Expression> updateValues) - : base(context, updateKey, updateValues) { } + public BulkUpdateEngine(DbContext context, Expression> updateKey, Expression> updateValues) + : base(context, updateKey, updateValues) { } - protected override UpdateContextQueryBase CreateUpdateContextQueryInstance(DbContext context, string schema, string table, List propertiesToUpdate, List propertiesForPivot, List propertiesToBulkLoad, IEntityType baseType, IDictionary propertiesGetter) - { - if (context.Database.IsSqlServer()) - return new SqlServerUpdateContextQuery(context, schema, table, propertiesToUpdate, propertiesForPivot, propertiesToBulkLoad, baseType, propertiesGetter, base.StoreObject); - // if (context.Database.IsNpgsql()) - // return new PostgresUpdateContextQuery(context, schema, table, propertiesToUpdate, propertiesForPivot, propertiesToBulkLoad, baseType, propertiesGetter); - throw new Exception("unsupported provider"); - } + protected override UpdateContextQueryBase CreateUpdateContextQueryInstance(DbContext context, string schema, string table, List propertiesToUpdate, List propertiesForPivot, List propertiesToBulkLoad, IEntityType baseType, IDictionary propertiesGetter) + { + if (context.Database.IsSqlServer()) + return new SqlServerUpdateContextQuery(context, schema, table, propertiesToUpdate, propertiesForPivot, propertiesToBulkLoad, baseType, propertiesGetter, base.StoreObject); + // if (context.Database.IsNpgsql()) + // return new PostgresUpdateContextQuery(context, schema, table, propertiesToUpdate, propertiesForPivot, propertiesToBulkLoad, baseType, propertiesGetter); + throw new Exception("unsupported provider"); } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkUpdateEngineBase.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkUpdateEngineBase.cs index c7d70ac1..1e34eb78 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkUpdateEngineBase.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/BulkUpdateEngineBase.cs @@ -8,98 +8,79 @@ using Paillave.EntityFrameworkCoreExtension.ContextMetadata; using Paillave.EntityFrameworkCoreExtension.Core; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave -{ - public abstract class BulkUpdateEngineBase where T : class - { - private List _propertiesToUpdate; // any column except pivot, computed - private List _propertiesForPivot; // pivot columns - private List _propertiesToBulkLoad; // any column except computed that is not pivot - private IDictionary _propertyGetters; - private IEntityType _baseType; - protected StoreObjectIdentifier StoreObject { get; } - - private string _schema; - private string _table; - private readonly DbContext _context; - protected abstract UpdateContextQueryBase CreateUpdateContextQueryInstance(DbContext context, string schema, string table, List propertiesToUpdate, List propertiesForPivot, List propertiesToBulkLoad, IEntityType baseType, IDictionary propertiesGetter); - private IEnumerable GetAllRelatedEntityTypes(IEntityType et) - { - yield return et; - foreach (var item in et.GetDerivedTypes()) - foreach (var i in GetAllRelatedEntityTypes(item)) - yield return i; - // var allNavigationProperties = subClasses.SelectMany(et => et.GetNavigations()).Where(a => a.GetTargetType().IsOwned()).Distinct(); - // foreach (var navigationProperty in allNavigationProperties) - // { - // var property = navigationProperty.PropertyInfo; - // var ownedEntityType = context.Model.FindEntityType(property.PropertyType); - - // if (ownedEntityType == null) // when entity has more then one ownedType (e.g. Address HomeAddress, Address WorkAddress) or one ownedType is in multiple Entities like Audit is usually. - // ownedEntityType = context.Model.GetEntityTypes().SingleOrDefault(a => a.DefiningNavigationName == property.Name && a.DefiningEntityType.Name == entityType.Name); +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; +public abstract class BulkUpdateEngineBase where T : class +{ + private List _propertiesToUpdate; // any column except pivot, computed + private List _propertiesForPivot; // pivot columns + private List _propertiesToBulkLoad; // any column except computed that is not pivot + private IDictionary _propertyGetters; + private IEntityType _baseType; + protected StoreObjectIdentifier StoreObject { get; } - // var ownedEntityPropertyNameColumnNameDict = ownedEntityType.GetProperties() - // .Where(ownedEntityProperty => !ownedEntityProperty.IsPrimaryKey()) - // .ToDictionary(ownedEntityProperty => ownedEntityProperty.Name, ownedEntityProperty => ownedEntityProperty.Relational().ColumnName); + private string _schema; + private string _table; + private readonly DbContext _context; + protected abstract UpdateContextQueryBase CreateUpdateContextQueryInstance(DbContext context, string schema, string table, List propertiesToUpdate, List propertiesForPivot, List propertiesToBulkLoad, IEntityType baseType, IDictionary propertiesGetter); + private IEnumerable GetAllRelatedEntityTypes(IEntityType et) + { + yield return et; + foreach (var item in et.GetDerivedTypes()) + foreach (var i in GetAllRelatedEntityTypes(item)) + yield return i; + } + public BulkUpdateEngineBase(DbContext context, Expression> updateKey, Expression> updateValues) + { + List computedProperties; + this._context = context; + var entityType = context.Model.FindEntityType(typeof(T)); + if (entityType == null) + throw new InvalidOperationException("DbContext does not contain EntitySet for Type: " + typeof(T).Name); + this.StoreObject = StoreObjectIdentifier.Create(entityType, StoreObjectType.Table).GetValueOrDefault(); + var summary = entityType.GetEntityEssentials(); + _schema = summary.Schema; + var table = entityType.GetTableName(); + if (string.IsNullOrWhiteSpace(table)) + throw new InvalidOperationException("Table name is not defined for Type: " + typeof(T).Name); + _table = table; + List entityTypes = GetAllRelatedEntityTypes(entityType).Distinct().ToList(); + _baseType = entityTypes.First(i => i.BaseType == null); + var allProperties = entityTypes.SelectMany(dt => dt.GetProperties()).DistinctBy(i => i.GetColumnName(this.StoreObject)).ToList(); + computedProperties = allProperties.Where(i => (i.ValueGenerated & ValueGenerated.OnAddOrUpdate) != ValueGenerated.Never).ToList(); - // foreach (var ownedProperty in property.PropertyType.GetProperties()) - // { - // if (ownedEntityPropertyNameColumnNameDict.ContainsKey(ownedProperty.Name)) - // { - // string columnName = ownedEntityPropertyNameColumnNameDict[ownedProperty.Name]; - // var ownedPropertyType = Nullable.GetUnderlyingType(ownedProperty.PropertyType) ?? ownedProperty.PropertyType; - // PropertyColumnNamesDict.Add(property.Name + "." + ownedProperty.Name, columnName); - // OutputPropertyColumnNamesDict.Add(property.Name + "." + ownedProperty.Name, columnName); - // } - // } - // } - } - public BulkUpdateEngineBase(DbContext context, Expression> updateKey, Expression> updateValues) + var valuesSetters = SettersExtractor.GetGetters(updateValues); + var keySetters = SettersExtractor.GetGetters(updateKey); + if (typeof(IMultiTenantEntity).IsAssignableFrom(typeof(T))) { - List computedProperties; - this._context = context; - var entityType = context.Model.FindEntityType(typeof(T)); - this.StoreObject = StoreObjectIdentifier.Create(entityType, StoreObjectType.Table).GetValueOrDefault(); - if (entityType == null) - throw new InvalidOperationException("DbContext does not contain EntitySet for Type: " + typeof(T).Name); - var summary = entityType.GetEntityEssentials(); - _schema = summary.Schema; - _table = entityType.GetTableName(); - - List entityTypes = GetAllRelatedEntityTypes(entityType).Distinct().ToList(); - _baseType = entityTypes.FirstOrDefault(i => i.BaseType == null); - var allProperties = entityTypes.SelectMany(dt => dt.GetProperties()).Distinct(new LambdaEqualityComparer(i => i.GetColumnName(this.StoreObject))).ToList(); - computedProperties = allProperties.Where(i => (i.ValueGenerated & ValueGenerated.OnAddOrUpdate) != ValueGenerated.Never).ToList(); - - var valuesSetters = SettersExtractor.GetGetters(updateValues); - var keySetters = SettersExtractor.GetGetters(updateKey); - if (typeof(IMultiTenantEntity).IsAssignableFrom(typeof(T))) + var tenantIdProp = allProperties.First(i => i.Name == nameof(IMultiTenantEntity.TenantId)); + var columnName = tenantIdProp.GetColumnName(this.StoreObject); + if (columnName == null) + throw new InvalidOperationException("TenantId column is not defined for Type: " + typeof(T).Name); + if (!keySetters.ContainsKey(columnName)) { - var tenantIdProp = allProperties.First(i => i.Name == nameof(IMultiTenantEntity.TenantId)); - if (!keySetters.ContainsKey(tenantIdProp.GetColumnName(this.StoreObject))) - { - keySetters[tenantIdProp.GetColumnName(this.StoreObject)] = tenantIdProp.PropertyInfo; - } + if (tenantIdProp.PropertyInfo == null) + throw new InvalidOperationException("TenantId property is not defined for Type: " + typeof(T).Name); + keySetters[columnName] = tenantIdProp.PropertyInfo; } - _propertyGetters = valuesSetters.Union(keySetters).ToDictionary(i => i.Key, i => i.Value); - - _propertiesForPivot = allProperties.Where(i => keySetters.ContainsKey(i.Name)).ToList(); - _propertiesToUpdate = allProperties.Where(i => valuesSetters.ContainsKey(i.Name)).Except(computedProperties).Except(_propertiesForPivot).ToList(); - _propertiesToBulkLoad = _propertiesForPivot.Union(_propertiesToUpdate).ToList(); - } - public void Update(IList sources) - { - var previousAutoDetect = _context.ChangeTracker.AutoDetectChangesEnabled; - _context.ChangeTracker.AutoDetectChangesEnabled = false; - var contextQuery = this.CreateUpdateContextQueryInstance(_context, _schema, _table, _propertiesToUpdate, _propertiesForPivot, _propertiesToBulkLoad, _baseType, _propertyGetters); - contextQuery.CreateStagingTable(); - contextQuery.BulkSaveInStaging(sources); - if (sources.Count > 10000) - contextQuery.IndexStagingTable(); - contextQuery.MergeFromStaging(); - contextQuery.DeleteStagingTable(); - _context.ChangeTracker.AutoDetectChangesEnabled = previousAutoDetect; } + _propertyGetters = valuesSetters.Union(keySetters).ToDictionary(i => i.Key, i => i.Value); + + _propertiesForPivot = allProperties.Where(i => keySetters.ContainsKey(i.Name)).ToList(); + _propertiesToUpdate = allProperties.Where(i => valuesSetters.ContainsKey(i.Name)).Except(computedProperties).Except(_propertiesForPivot).ToList(); + _propertiesToBulkLoad = _propertiesForPivot.Union(_propertiesToUpdate).ToList(); + } + public void Update(IList sources) + { + var previousAutoDetect = _context.ChangeTracker.AutoDetectChangesEnabled; + _context.ChangeTracker.AutoDetectChangesEnabled = false; + var contextQuery = this.CreateUpdateContextQueryInstance(_context, _schema, _table, _propertiesToUpdate, _propertiesForPivot, _propertiesToBulkLoad, _baseType, _propertyGetters); + contextQuery.CreateStagingTable(); + contextQuery.BulkSaveInStaging(sources); + if (sources.Count > 10000) + contextQuery.IndexStagingTable(); + contextQuery.MergeFromStaging(); + contextQuery.DeleteStagingTable(); + _context.ChangeTracker.AutoDetectChangesEnabled = previousAutoDetect; } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/DbContextBulkExtensions.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/DbContextBulkExtensions.cs index db5495b7..58cb6e54 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/DbContextBulkExtensions.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/DbContextBulkExtensions.cs @@ -2,28 +2,26 @@ using System.Collections.Generic; using System.Linq.Expressions; using System.Threading; -using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; -using Paillave.EntityFrameworkCoreExtension.BulkSave.SqlServer; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; + +public static class DbContextBulkExtensions { - public static class DbContextBulkExtensions + public static void BulkSave(this DbContext context, IList entities, CancellationToken cancellationToken, Expression>? pivotKey = null, bool doNotUpdateIfExists = false, bool insertOnly = false) where T : class + { + var pivotKeys = pivotKey == null ? new Expression>[0] : new[] { pivotKey }; + BulkSaveEngineBase bulkSaveEngine = new BulkSaveEngine(context, pivotKeys); + bulkSaveEngine.Save(entities, cancellationToken, doNotUpdateIfExists, insertOnly); + } + public static void BulkSave(this DbContext context, IList entities, Expression>[] pivotKeys, CancellationToken cancellationToken, bool doNotUpdateIfExists = false, bool insertOnly = false) where T : class + { + BulkSaveEngineBase bulkSaveEngine = new BulkSaveEngine(context, pivotKeys); + bulkSaveEngine.Save(entities, cancellationToken, doNotUpdateIfExists, insertOnly); + } + public static void BulkUpdate(this DbContext context, IList sources, Expression> updateKey, Expression> updateValues) where TEntity : class { - public static void BulkSave(this DbContext context, IList entities, CancellationToken cancellationToken, Expression> pivotKey = null, bool doNotUpdateIfExists = false, bool insertOnly = false) where T : class - { - BulkSaveEngineBase bulkSaveEngine = new BulkSaveEngine(context, pivotKey); - bulkSaveEngine.Save(entities, cancellationToken, doNotUpdateIfExists, insertOnly); - } - public static void BulkSave(this DbContext context, IList entities, Expression>[] pivotKeys, CancellationToken cancellationToken, bool doNotUpdateIfExists = false, bool insertOnly = false) where T : class - { - BulkSaveEngineBase bulkSaveEngine = new BulkSaveEngine(context, pivotKeys); - bulkSaveEngine.Save(entities, cancellationToken, doNotUpdateIfExists, insertOnly); - } - public static void BulkUpdate(this DbContext context, IList sources, Expression> updateKey, Expression> updateValues) where TEntity : class - { - BulkUpdateEngineBase bulkUpdateEngine = new BulkUpdateEngine(context, updateKey, updateValues); - bulkUpdateEngine.Update(sources); - } + BulkUpdateEngineBase bulkUpdateEngine = new BulkUpdateEngine(context, updateKey, updateValues); + bulkUpdateEngine.Update(sources); } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/LambdaEqualityComparer.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/LambdaEqualityComparer.cs index 234fbfdc..2dc5c81d 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/LambdaEqualityComparer.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/LambdaEqualityComparer.cs @@ -1,43 +1,42 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +// using System; +// using System.Collections.Generic; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave -{ - public class LambdaEqualityComparer : IEqualityComparer - { - public Func Comparer { get; } - public LambdaEqualityComparer(Func comparer) - { - Comparer = comparer; - } - public bool Equals(T x, T y) - { - return Comparer(x, y); - } - public int GetHashCode(T obj) - { - return 0; - } - } - public class LambdaEqualityComparer : IEqualityComparer where K : IEquatable - { - private readonly Func _propertyToCompare; - public Func Comparer { get; } - public LambdaEqualityComparer(Func prop) - { - _propertyToCompare = prop; - Comparer = (t1, t2) => _propertyToCompare(t1).Equals(_propertyToCompare(t2)); - } - public bool Equals(T x, T y) - { - return Comparer(x, y); - } - public int GetHashCode(T obj) - { - return _propertyToCompare(obj)?.GetHashCode() ?? 0; - } - } -} +// namespace Paillave.EntityFrameworkCoreExtension.BulkSave +// { +// [Obsolete] +// public class LambdaEqualityComparer : IEqualityComparer +// { +// public Func Comparer { get; } +// public LambdaEqualityComparer(Func comparer) +// { +// Comparer = comparer; +// } +// public bool Equals(T x, T y) +// { +// return Comparer(x, y); +// } +// public int GetHashCode(T obj) +// { +// return 0; +// } +// } +// [Obsolete] +// public class LambdaEqualityComparer : IEqualityComparer where K : IEquatable +// { +// private readonly Func _propertyToCompare; +// public Func Comparer { get; } +// public LambdaEqualityComparer(Func prop) +// { +// _propertyToCompare = prop; +// Comparer = (t1, t2) => _propertyToCompare(t1).Equals(_propertyToCompare(t2)); +// } +// public bool Equals(T x, T y) +// { +// return Comparer(x, y); +// } +// public int GetHashCode(T obj) +// { +// return _propertyToCompare(obj)?.GetHashCode() ?? 0; +// } +// } +// } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/ObjectDataReader.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/ObjectDataReader.cs index 8e3ffbb9..0fb22f44 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/ObjectDataReader.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/ObjectDataReader.cs @@ -6,232 +6,234 @@ using System.Collections; using System.Collections.Generic; using System.Data; -using System.Data.Common; using System.Linq; -using System.Text; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; + +public class ObjectDataReader : IDataReader { - public class ObjectDataReader : IDataReader + private class ValueAccessor { - private class ValueAccessor - { - private HashSet _relatedProperties; - private TypeAccessor _accessor; + private HashSet _relatedProperties; + private TypeAccessor _accessor; - public ValueAccessor(Type type) - { - _accessor = TypeAccessor.Create(type); - _relatedProperties = new HashSet(type.GetProperties().Select(i => i.Name)); - } - public object this[object target, string key] - { - get - { - if (_relatedProperties.Contains(key)) - return _accessor[target, key]; - else - return null; - } - } + public ValueAccessor(Type type) + { + _accessor = TypeAccessor.Create(type); + _relatedProperties = new HashSet(type.GetProperties().Select(i => i.Name)); } - private IEnumerator _source; - private readonly Dictionary _accessorsByType; - private readonly HashSet _shadowProperties; - private readonly Dictionary _convertibleProperties; - private readonly DbContext _context; - private readonly string[] _memberNames; - private readonly Type[] _effectiveTypes; - private readonly bool[] _allowNull; - private string _tempColumnNumOrderName; - private int _rowCounter = 0; - public ObjectDataReader(IEnumerable source, ObjectDataReaderConfig config) + public object? this[object target, string key] { - if (source == null) throw new ArgumentOutOfRangeException("source"); - - _accessorsByType = config.Types.ToDictionary(i => i.ClrType, i => new ValueAccessor(i.ClrType)); - _tempColumnNumOrderName = config.TempColumnNumOrderName; - var sp = config.EfProperties.Where(i => i.IsShadowProperty()); - // var tmp= - _shadowProperties = new HashSet(sp.Select(i => i.Name)); - _convertibleProperties = config.EfProperties.Select(i => new { ValueConverter = i.GetValueConverter(), i.Name }).Where(i => i.ValueConverter != null).ToDictionary(i => i.Name, i => i.ValueConverter); - _context = config.Context; - - _currentType = null; - Current = null; - _memberNames = config.EfProperties.Select(i => i.Name).ToArray(); - if (!string.IsNullOrWhiteSpace(_tempColumnNumOrderName)) - _memberNames = _memberNames.Union(new[] { _tempColumnNumOrderName }).ToArray(); - _effectiveTypes = config.EfProperties.Select(i => Nullable.GetUnderlyingType(i.ClrType) ?? i.ClrType).ToArray(); - _allowNull = config.EfProperties.Select(i => i.IsColumnNullable()).ToArray(); - _source = source.GetEnumerator(); + get + { + if (_relatedProperties.Contains(key)) + return _accessor[target, key]; + else + return null; + } } - - protected object Current; - private Type _currentType; + } + private IEnumerator? _source; + private readonly Dictionary _accessorsByType; + private readonly HashSet _shadowProperties; + private readonly Dictionary _convertibleProperties; + private readonly DbContext _context; + private readonly string[] _memberNames; + private readonly Type[] _effectiveTypes; + private readonly bool[] _allowNull; + private string? _tempColumnNumOrderName; + private int _rowCounter = 0; + public ObjectDataReader(IEnumerable source, ObjectDataReaderConfig config) + { + if (source == null) throw new ArgumentOutOfRangeException("source"); + + _accessorsByType = config.Types.ToDictionary(i => i.ClrType, i => new ValueAccessor(i.ClrType)); + _tempColumnNumOrderName = config.TempColumnNumOrderName; + var sp = config.EfProperties.Where(i => i.IsShadowProperty()); + // var tmp= + _shadowProperties = new HashSet(sp.Select(i => i.Name)); + _convertibleProperties = config.EfProperties + .Select(i => new { ValueConverter = i.GetValueConverter(), i.Name }) + .Where(i => i.ValueConverter != null) + .ToDictionary(i => i.Name, i => i.ValueConverter!); + _context = config.Context; + + _currentType = null; + _current = null; + _memberNames = config.EfProperties.Select(i => i.Name).ToArray(); + if (!string.IsNullOrWhiteSpace(_tempColumnNumOrderName)) + _memberNames = _memberNames.Union(new[] { _tempColumnNumOrderName }).ToArray(); + _effectiveTypes = config.EfProperties.Select(i => Nullable.GetUnderlyingType(i.ClrType) ?? i.ClrType).ToArray(); + _allowNull = config.EfProperties.Select(i => i.IsColumnNullable()).ToArray(); + _source = source.GetEnumerator(); + } + private object? _current; + protected object Current => _current ?? throw new InvalidOperationException("No current item"); + private Type? _currentType; - public int Depth => 0; + public int Depth => 0; - public DataTable GetSchemaTable() - { - throw new NotImplementedException(); - } - public void Close() - { - HasRows = false; - Current = null; - _currentType = null; - _source = null; - } + public DataTable GetSchemaTable() + { + throw new NotImplementedException(); + } + public void Close() + { + HasRows = false; + _current = null; + _currentType = null; + _source = null; + } - public bool HasRows { get; private set; } = true; + public bool HasRows { get; private set; } = true; - public bool NextResult() - { - HasRows = false; - return false; - } - public bool Read() + public bool NextResult() + { + HasRows = false; + return false; + } + public bool Read() + { + if (HasRows) { - if (HasRows) + var tmp = _source; + if (tmp != null && tmp.MoveNext()) { - var tmp = _source; - if (tmp != null && tmp.MoveNext()) - { - Current = tmp.Current; - _currentType = Current.GetType(); - _rowCounter++; - return true; - } - else - { - HasRows = false; - } + _current = tmp.Current; + _currentType = Current.GetType(); + _rowCounter++; + return true; + } + else + { + HasRows = false; } - _currentType = null; - Current = null; - return false; - } - - public int RecordsAffected => 0; - - public void Dispose() - { - HasRows = false; - Current = null; - _currentType = null; - _source = null; } + _currentType = null; + _current = null; + return false; + } - public int FieldCount => _memberNames.Length; + public int RecordsAffected => 0; - public bool IsClosed => _source == null; + public void Dispose() + { + HasRows = false; + _current = null; + _currentType = null; + _source = null; + } - public bool GetBoolean(int i) => (bool)this[i]; + public int FieldCount => _memberNames.Length; - public byte GetByte(int i) => (byte)this[i]; + public bool IsClosed => _source == null; - public long GetBytes(int i, long fieldOffset, byte[] buffer, int bufferoffset, int length) - { - byte[] s = (byte[])this[i]; - int available = s.Length - (int)fieldOffset; - if (available <= 0) return 0; + public bool GetBoolean(int i) => (bool)(this[i] ?? throw new InvalidOperationException("null value instead of boolean")); - int count = Math.Min(length, available); - Buffer.BlockCopy(s, (int)fieldOffset, buffer, bufferoffset, count); - return count; - } + public byte GetByte(int i) => (byte)(this[i] ?? throw new InvalidOperationException("null value instead of byte")); - public char GetChar(int i) => (char)this[i]; + public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferOffset, int length) + { + byte[] s = (byte[])(this[i] ?? throw new InvalidOperationException("null value instead of byte[]")); + int available = s.Length - (int)fieldOffset; + if (available <= 0) return 0; + + int count = Math.Min(length, available); + buffer ??= new byte[count]; + Buffer.BlockCopy(s, (int)fieldOffset, buffer, bufferOffset, count); + return count; + } - public long GetChars(int i, long fieldoffset, char[] buffer, int bufferoffset, int length) - { - string s = (string)this[i]; - int available = s.Length - (int)fieldoffset; - if (available <= 0) return 0; + public char GetChar(int i) => (char)(this[i] ?? throw new InvalidOperationException("null value instead of char")); - int count = Math.Min(length, available); - s.CopyTo((int)fieldoffset, buffer, bufferoffset, count); - return count; - } - public string GetDataTypeName(int i) => (_effectiveTypes == null ? typeof(object) : _effectiveTypes[i]).Name; - public DateTime GetDateTime(int i) => (DateTime)this[i]; - public decimal GetDecimal(int i) => (decimal)this[i]; - public double GetDouble(int i) => (double)this[i]; - public Type GetFieldType(int i) => _effectiveTypes == null ? typeof(object) : _effectiveTypes[i]; - public float GetFloat(int i) => (float)this[i]; - public Guid GetGuid(int i) => (Guid)this[i]; - public short GetInt16(int i) => (short)this[i]; - public int GetInt32(int i) => (int)this[i]; - public long GetInt64(int i) => (long)this[i]; - public string GetName(int i) => _memberNames[i]; - public int GetOrdinal(string name) => Array.IndexOf(_memberNames, name); - public string GetString(int i) => (string)this[i]; - public object GetValue(int i) => this[i]; - public int GetValues(object[] values) - { - int count = Math.Min(values.Length, this._memberNames.Length); - for (int i = 0; i < count; i++) values[i] = this.GetValue(this.GetOrdinal(this._memberNames[i])); - return count; - } - public bool IsDBNull(int i) => this[i] is DBNull; - public IDataReader GetData(int i) - { - throw new NotImplementedException(); - } - private Dictionary> _shadowOfEntityDico = new Dictionary>(); - public object this[string name] + public long GetChars(int i, long fieldOffset, char[]? buffer, int bufferOffset, int length) + { + string s = (string)(this[i] ?? throw new InvalidOperationException("null value instead of string")); + int available = s.Length - (int)fieldOffset; + if (available <= 0) return 0; + + int count = Math.Min(length, available); + buffer ??= new char[count]; + s.CopyTo((int)fieldOffset, buffer, bufferOffset, count); + return count; + } + public string GetDataTypeName(int i) => (_effectiveTypes == null ? typeof(object) : _effectiveTypes[i]).Name; + public DateTime GetDateTime(int i) => (DateTime)(this[i] ?? throw new InvalidOperationException("null value instead of DateTime")); + public decimal GetDecimal(int i) => (decimal)(this[i] ?? throw new InvalidOperationException("null value instead of decimal")); + public double GetDouble(int i) => (double)(this[i] ?? throw new InvalidOperationException("null value instead of double")); + public Type GetFieldType(int i) => _effectiveTypes == null ? typeof(object) : _effectiveTypes[i]; + public float GetFloat(int i) => (float)(this[i] ?? throw new InvalidOperationException("null value instead of float")); + public Guid GetGuid(int i) => (Guid)(this[i] ?? throw new InvalidOperationException("null value instead of Guid")); + public short GetInt16(int i) => (short)(this[i] ?? throw new InvalidOperationException("null value instead of short")); + public int GetInt32(int i) => (int)(this[i] ?? throw new InvalidOperationException("null value instead of int")); + public long GetInt64(int i) => (long)(this[i] ?? throw new InvalidOperationException("null value instead of long")); + public string GetName(int i) => _memberNames[i]; + public int GetOrdinal(string name) => Array.IndexOf(_memberNames, name); + public string GetString(int i) => (string)(this[i] ?? throw new InvalidOperationException("null value instead of string")); + public object GetValue(int i) => this[i] ?? throw new InvalidOperationException("null value instead of object"); + public int GetValues(object[] values) + { + int count = Math.Min(values.Length, this._memberNames.Length); + for (int i = 0; i < count; i++) values[i] = this.GetValue(this.GetOrdinal(this._memberNames[i])); + return count; + } + public bool IsDBNull(int i) => this[i] is DBNull; + public IDataReader GetData(int i) + { + throw new NotImplementedException(); + } + private Dictionary> _shadowOfEntityDico = new Dictionary>(); + public object? this[string name] + { + get { - get + if (name == _tempColumnNumOrderName) { - if (name == _tempColumnNumOrderName) - { - return _rowCounter - 1; - } - else if (_shadowProperties.Contains(name)) + return _rowCounter - 1; + } + else if (_shadowProperties.Contains(name)) + { + var etr = _context.Entry(Current); + if (_shadowOfEntityDico.TryGetValue(etr.Metadata.Name, out var subDic)) { - var etr = _context.Entry(Current); - if (_shadowOfEntityDico.TryGetValue(etr.Metadata.Name, out var subDic)) + if (subDic.TryGetValue(name, out var hasIt)) { - if (subDic.TryGetValue(name, out var hasIt)) - { - if (hasIt) - return etr.Property(name).CurrentValue; - else - return null; - } - var elt = etr.Properties.FirstOrDefault(i => i.Metadata.Name == name); - subDic[name] = elt != null; - if (elt != null) - return elt.CurrentValue; + if (hasIt) + return etr.Property(name).CurrentValue; else return null; } - var elt2 = etr.Properties.FirstOrDefault(i => i.Metadata.Name == name); - _shadowOfEntityDico[etr.Metadata.Name] = new Dictionary { [name] = elt2 != null }; - if (elt2 == null) return null; - return elt2.CurrentValue; - // if(etr.Properties.FirstOrDefault()) - // return etr.Property(name).CurrentValue; - } - else - { - if (!_accessorsByType.TryGetValue(_currentType, out var acc)) return Constants.DBNull; - var val = acc[Current, name]; - if (val == null) return Constants.DBNull; - if (_convertibleProperties.TryGetValue(name, out var converter)) return converter.ConvertToProvider(val); - return val; + var elt = etr.Properties.FirstOrDefault(i => i.Metadata.Name == name); + subDic[name] = elt != null; + if (elt != null) + return elt.CurrentValue; + else + return null; } + var elt2 = etr.Properties.FirstOrDefault(i => i.Metadata.Name == name); + _shadowOfEntityDico[etr.Metadata.Name] = new Dictionary { [name] = elt2 != null }; + if (elt2 == null) return null; + return elt2.CurrentValue; + // if(etr.Properties.FirstOrDefault()) + // return etr.Property(name).CurrentValue; + } + else + { + if (!_accessorsByType.TryGetValue(_currentType, out var acc)) return Constants.DBNull; + var val = acc[Current, name]; + if (val == null) return Constants.DBNull; + if (_convertibleProperties.TryGetValue(name, out var converter)) return converter.ConvertToProvider(val); + return val; } } - public object this[int i] => this[_memberNames[i]] ?? Constants.DBNull; - } - public class ObjectDataReaderConfig - { - public IEnumerable EfProperties { get; set; } - public IEnumerable Types { get; set; } - public DbContext Context { get; set; } - public string TempColumnNumOrderName { get; set; } } + public object? this[int i] => this[_memberNames[i]] ?? Constants.DBNull; +} +public class ObjectDataReaderConfig +{ + public required IEnumerable EfProperties { get; set; } + public required IEnumerable Types { get; set; } + public required DbContext Context { get; set; } + public string? TempColumnNumOrderName { get; set; } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SaveContextQueryBase.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SaveContextQueryBase.cs index 72bb7c23..c0f213eb 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SaveContextQueryBase.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SaveContextQueryBase.cs @@ -3,94 +3,90 @@ using System; using System.Collections.Generic; using System.Data; -using System.Data.Common; -using System.Linq; -using System.Linq.Expressions; -using System.Text; using System.Threading; -using System.Threading.Tasks; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; + +public abstract class SaveContextQueryBase where T : class { - public abstract class SaveContextQueryBase where T : class - { - protected string Table { get; } - protected StoreObjectIdentifier StoreObject { get; } - protected string StagingId { get; } = Guid.NewGuid().ToString().Substring(0, 8); - protected string Schema { get; } - protected List EntityTypes { get; } + protected string Table { get; } + protected StoreObjectIdentifier StoreObject { get; } + protected string StagingId { get; } = Guid.NewGuid().ToString().Substring(0, 8); + protected string Schema { get; } + protected List EntityTypes { get; } - /// - /// Any column except computed - /// - protected List PropertiesToInsert { get; } - /// - /// any column except pivot, computed - /// - protected List PropertiesToUpdate { get; } - /// - /// pivot columns - /// - protected List> PropertiesForPivotSet { get; } + /// + /// Any column except computed + /// + protected List PropertiesToInsert { get; } + /// + /// any column except pivot, computed + /// + protected List PropertiesToUpdate { get; } + /// + /// pivot columns + /// + protected List> PropertiesForPivotSet { get; } - /// - /// Any column except computed that is not pivot - /// - protected List PropertiesToBulkLoad { get; } + /// + /// Any column except computed that is not pivot + /// + protected List PropertiesToBulkLoad { get; } - protected CancellationToken CancellationToken { get; } + protected CancellationToken CancellationToken { get; } - protected DbContext Context { get; } - public SaveContextQueryBase(DbContext context, string schema, string table, List propertiesToInsert, List propertiesToUpdate, List> propertiesForPivotSet, List propertiesToBulkLoad, List entityTypes, CancellationToken cancellationToken, StoreObjectIdentifier storeObject) - { - this.StoreObject = storeObject; - this.CancellationToken = cancellationToken; - this.PropertiesToInsert = propertiesToInsert; - this.PropertiesToUpdate = propertiesToUpdate; - this.PropertiesForPivotSet = propertiesForPivotSet; - this.PropertiesToBulkLoad = propertiesToBulkLoad; - this.Schema = schema; - this.Table = table; - this.Context = context; - this.EntityTypes = entityTypes; - } + protected DbContext Context { get; } + public SaveContextQueryBase(DbContext context, string schema, string table, List propertiesToInsert, List propertiesToUpdate, List> propertiesForPivotSet, List propertiesToBulkLoad, List entityTypes, CancellationToken cancellationToken, StoreObjectIdentifier storeObject) + { + this.StoreObject = storeObject; + this.CancellationToken = cancellationToken; + this.PropertiesToInsert = propertiesToInsert; + this.PropertiesToUpdate = propertiesToUpdate; + this.PropertiesForPivotSet = propertiesForPivotSet; + this.PropertiesToBulkLoad = propertiesToBulkLoad; + this.Schema = schema; + this.Table = table; + this.Context = context; + this.EntityTypes = entityTypes; + } - /// - /// Create the staging that is meant to receive the raw bulk load - /// - public abstract int CreateStagingTable(); - /// - /// Create the output staging table that is mean to receive the result of the merge in the right order to update entities - /// - public abstract int CreateOutputStagingTable(); - /// - /// save entities in the staging table - /// - /// entities that will saved in staging - public abstract void BulkSaveInStaging(IEnumerable entities); - /// - /// merge the staging table in the target table - /// - public abstract void MergeFromStaging(bool doNotUpdateIfExists = false); - public abstract void InsertFromStaging(); - public abstract void IndexStagingTable(List propertiesForPivot); - public abstract void IndexOutputStagingTable(); - public abstract DataTable GetOutputStaging(); - public abstract DataTable GetOutputStagingForComputedColumns(); - public abstract void DeleteStagingTable(); - public abstract void DeleteOutputStagingTable(); - protected DataTable StrictlyExecuteSql(string sqlQuery) - { - var cmd = this.Context.Database.GetDbConnection().CreateCommand(); - if (cmd.Connection.State == ConnectionState.Closed) - cmd.Connection.Open(); - cmd.CommandType = System.Data.CommandType.Text; - cmd.CommandText = sqlQuery; - var dataReader = cmd.ExecuteReader(); - DataTable dataTable = new DataTable(); - dataTable.Load(dataReader); - // https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/queries-in-linq-to-dataset - return dataTable; - } + /// + /// Create the staging that is meant to receive the raw bulk load + /// + public abstract int CreateStagingTable(); + /// + /// Create the output staging table that is mean to receive the result of the merge in the right order to update entities + /// + public abstract int CreateOutputStagingTable(); + /// + /// save entities in the staging table + /// + /// entities that will saved in staging + public abstract void BulkSaveInStaging(IEnumerable entities); + /// + /// merge the staging table in the target table + /// + public abstract void MergeFromStaging(bool doNotUpdateIfExists = false); + public abstract void InsertFromStaging(); + public abstract void IndexStagingTable(List propertiesForPivot); + public abstract void IndexOutputStagingTable(); + public abstract DataTable GetOutputStaging(); + public abstract DataTable GetOutputStagingForComputedColumns(); + public abstract void DeleteStagingTable(); + public abstract void DeleteOutputStagingTable(); + protected DataTable StrictlyExecuteSql(string sqlQuery) + { + var cmd = this.Context.Database.GetDbConnection().CreateCommand(); + if (cmd.Connection == null) + throw new InvalidOperationException("Connection is null"); + if (cmd.Connection.State == ConnectionState.Closed) + cmd.Connection.Open(); + cmd.CommandType = System.Data.CommandType.Text; + cmd.CommandText = sqlQuery; + var dataReader = cmd.ExecuteReader(); + DataTable dataTable = new DataTable(); + dataTable.Load(dataReader); + // https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/queries-in-linq-to-dataset + return dataTable; } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SettersExtractor.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SettersExtractor.cs index 330a5b27..bdd1b794 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SettersExtractor.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SettersExtractor.cs @@ -3,34 +3,33 @@ using System.Linq.Expressions; using System.Reflection; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; + +public class SettersExtractor { - public class SettersExtractor + public static IDictionary GetGetters(Expression> setValues) { - public static IDictionary GetGetters(Expression> setValues) - { - var vis = new MyVisitor(); - vis.Visit(setValues); - return vis.MemberInfos; - } + var vis = new MyVisitor(); + vis.Visit(setValues); + return vis.MemberInfos; + } - private class MyVisitor : ExpressionVisitor + private class MyVisitor : ExpressionVisitor + { + public IDictionary MemberInfos { get; set; } = new Dictionary(); + protected override MemberBinding VisitMemberBinding(MemberBinding node) { - public IDictionary MemberInfos { get; set; } = new Dictionary(); - protected override MemberBinding VisitMemberBinding(MemberBinding node) + if (node.BindingType == MemberBindingType.Assignment) { - if (node.BindingType == MemberBindingType.Assignment) + var memberAssignment = (MemberAssignment)node; + switch (memberAssignment.Expression) { - var memberAssignment = (MemberAssignment)node; - switch (memberAssignment.Expression) - { - case MemberExpression me: - MemberInfos[memberAssignment.Member.Name] = me.Member; - break; - } + case MemberExpression me: + MemberInfos[memberAssignment.Member.Name] = me.Member; + break; } - return node; } + return node; } } -} \ No newline at end of file +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerSaveContextQuery.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerSaveContextQuery.cs index 796265b3..334ba40e 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerSaveContextQuery.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerSaveContextQuery.cs @@ -4,188 +4,185 @@ using System; using System.Collections.Generic; using System.Data; -using System.Data.Common; -// using Microsoft.Data.SqlClient; using System.Linq; -using System.Text; using System.Threading; -using System.Threading.Tasks; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave.SqlServer +namespace Paillave.EntityFrameworkCoreExtension.BulkSave.SqlServer; + +public class SqlServerSaveContextQuery : SaveContextQueryBase where T : class { - public class SqlServerSaveContextQuery : SaveContextQueryBase where T : class + private const string TempColumnNumOrderName = "_TempColumnNumOrder"; + private const string TempColumnAction = "_Action"; + private string SqlTargetTable => $"[{this.Schema}].[{this.Table}]"; + private string SqlStagingTableName => $"[{this.Schema}].[{this.Table}_temp_{this.StagingId}]"; + private string SqlOutputStagingTableName => $"[{this.Schema}].[{this.Table}_tempoutput_{this.StagingId}]"; + + public SqlServerSaveContextQuery(DbContext Context, string schema, string table, List propertiesToInsert, List propertiesToUpdate, List> propertiesForPivotSet, List propertiesToBulkLoad, List entityTypes, CancellationToken cancellationToken, StoreObjectIdentifier storeObject) + : base(Context, schema ?? "dbo", table, propertiesToInsert, propertiesToUpdate, propertiesForPivotSet, propertiesToBulkLoad, entityTypes, cancellationToken, storeObject) { - private const string TempColumnNumOrderName = "_TempColumnNumOrder"; - private const string TempColumnAction = "_Action"; - private string SqlTargetTable => $"[{this.Schema}].[{this.Table}]"; - private string SqlStagingTableName => $"[{this.Schema}].[{this.Table}_temp_{this.StagingId}]"; - private string SqlOutputStagingTableName => $"[{this.Schema}].[{this.Table}_tempoutput_{this.StagingId}]"; - - public SqlServerSaveContextQuery(DbContext Context, string schema, string table, List propertiesToInsert, List propertiesToUpdate, List> propertiesForPivotSet, List propertiesToBulkLoad, List entityTypes, CancellationToken cancellationToken, StoreObjectIdentifier storeObject) - : base(Context, schema ?? "dbo", table, propertiesToInsert, propertiesToUpdate, propertiesForPivotSet, propertiesToBulkLoad, entityTypes, cancellationToken, storeObject) - { - } + } - public override int CreateStagingTable() - => this.Context.Database.ExecuteSqlRaw(this.CreateStagingTableSql()); + public override int CreateStagingTable() + => this.Context.Database.ExecuteSqlRaw(this.CreateStagingTableSql()); - protected virtual string CreateStagingTableSql() - => $@"SELECT TOP 0 { string.Join(",", PropertiesToBulkLoad.Select(i => $"T.[{i.GetColumnName(base.StoreObject)}]")) }, 0 as [{TempColumnNumOrderName}] + protected virtual string CreateStagingTableSql() + => $@"SELECT TOP 0 {string.Join(",", PropertiesToBulkLoad.Select(i => $"T.[{i.GetColumnName(base.StoreObject)}]"))}, 0 as [{TempColumnNumOrderName}] INTO {SqlStagingTableName} FROM {SqlTargetTable} AS T LEFT JOIN {SqlTargetTable} AS Source ON 1 = 0 option(recompile);"; - public override void IndexStagingTable(List propertiesForPivot) - { - this.Context.Database.ExecuteSqlRaw(this.IndexStagingTableSql(propertiesForPivot)); - this.Context.Database.ExecuteSqlRaw(this.CounterIndexStagingTableSql(propertiesForPivot)); - } + public override void IndexStagingTable(List propertiesForPivot) + { + this.Context.Database.ExecuteSqlRaw(this.IndexStagingTableSql(propertiesForPivot)); + this.Context.Database.ExecuteSqlRaw(this.CounterIndexStagingTableSql(propertiesForPivot)); + } - protected virtual string IndexStagingTableSql(List propertiesForPivot) - => $@"CREATE UNIQUE CLUSTERED INDEX IX_{this.Table}_temp_{this.StagingId}_PivotKey ON {SqlStagingTableName} ({string.Join(", ", propertiesForPivot.Select(i => $"[{i.GetColumnName(base.StoreObject)}]"))})"; + protected virtual string IndexStagingTableSql(List propertiesForPivot) + => $@"CREATE UNIQUE CLUSTERED INDEX IX_{this.Table}_temp_{this.StagingId}_PivotKey ON {SqlStagingTableName} ({string.Join(", ", propertiesForPivot.Select(i => $"[{i.GetColumnName(base.StoreObject)}]"))})"; - protected virtual string CounterIndexStagingTableSql(List propertiesForPivot) - => $@"CREATE UNIQUE INDEX IX_{this.Table}_temp_{this.StagingId}_{TempColumnNumOrderName} ON {SqlStagingTableName} ({TempColumnNumOrderName})"; + protected virtual string CounterIndexStagingTableSql(List propertiesForPivot) + => $@"CREATE UNIQUE INDEX IX_{this.Table}_temp_{this.StagingId}_{TempColumnNumOrderName} ON {SqlStagingTableName} ({TempColumnNumOrderName})"; - public override void DeleteStagingTable() - => this.Context.Database.ExecuteSqlRaw(this.DeleteStagingTableSql()); + public override void DeleteStagingTable() + => this.Context.Database.ExecuteSqlRaw(this.DeleteStagingTableSql()); - protected virtual string DeleteStagingTableSql() - => $@"IF OBJECT_ID('{SqlStagingTableName}', 'U') IS NOT NULL DROP TABLE {SqlStagingTableName}"; + protected virtual string DeleteStagingTableSql() + => $@"IF OBJECT_ID('{SqlStagingTableName}', 'U') IS NOT NULL DROP TABLE {SqlStagingTableName}"; - public override void DeleteOutputStagingTable() - => this.Context.Database.ExecuteSqlRaw(this.DeleteOutputStagingTableSql()); + public override void DeleteOutputStagingTable() + => this.Context.Database.ExecuteSqlRaw(this.DeleteOutputStagingTableSql()); - protected virtual string DeleteOutputStagingTableSql() - => $@"IF OBJECT_ID('{SqlOutputStagingTableName}', 'U') IS NOT NULL DROP TABLE {SqlOutputStagingTableName}"; + protected virtual string DeleteOutputStagingTableSql() + => $@"IF OBJECT_ID('{SqlOutputStagingTableName}', 'U') IS NOT NULL DROP TABLE {SqlOutputStagingTableName}"; - public override int CreateOutputStagingTable() - => this.Context.Database.ExecuteSqlRaw(this.CreateOutputStagingTableSql()); + public override int CreateOutputStagingTable() + => this.Context.Database.ExecuteSqlRaw(this.CreateOutputStagingTableSql()); - protected virtual string CreateOutputStagingTableSql() - => $@"SELECT TOP 0 T.*, 0 as [{TempColumnNumOrderName}], 'mergeaction' as [{TempColumnAction}] + protected virtual string CreateOutputStagingTableSql() + => $@"SELECT TOP 0 T.*, 0 as [{TempColumnNumOrderName}], 'mergeaction' as [{TempColumnAction}] INTO {SqlOutputStagingTableName} FROM {SqlTargetTable} AS T LEFT JOIN {SqlTargetTable} AS Source ON 1 = 0 option(recompile);"; - public override void IndexOutputStagingTable() - => this.Context.Database.ExecuteSqlRaw(this.IndexOutputStagingTableSql()); + public override void IndexOutputStagingTable() + => this.Context.Database.ExecuteSqlRaw(this.IndexOutputStagingTableSql()); - protected virtual string IndexOutputStagingTableSql() - => $@"CREATE UNIQUE CLUSTERED INDEX IX_{this.Table}_tempoutput_{this.StagingId} ON {SqlOutputStagingTableName} ([{TempColumnNumOrderName}])"; + protected virtual string IndexOutputStagingTableSql() + => $@"CREATE UNIQUE CLUSTERED INDEX IX_{this.Table}_tempoutput_{this.StagingId} ON {SqlOutputStagingTableName} ([{TempColumnNumOrderName}])"; - public override void BulkSaveInStaging(IEnumerable entities) - { - var sqlBulkCopy = new SqlBulkCopy(OpenAndGetSqlConnection(), SqlBulkCopyOptions.Default, null); - sqlBulkCopy.DestinationTableName = SqlStagingTableName; - sqlBulkCopy.BulkCopyTimeout = 300; - sqlBulkCopy.BatchSize = entities.Count(); - foreach (var element in PropertiesToBulkLoad) - sqlBulkCopy.ColumnMappings.Add(element.Name, element.GetColumnName(base.StoreObject)); - sqlBulkCopy.ColumnMappings.Add(TempColumnNumOrderName, TempColumnNumOrderName); - - - var dataReader = new ObjectDataReader(entities, new ObjectDataReaderConfig - { - EfProperties = PropertiesToBulkLoad, - Types = EntityTypes, - Context = Context, - TempColumnNumOrderName = TempColumnNumOrderName - }); - // sqlBulkCopy.SqlRowsCopied += (sender, eventArgs) => - // { - // Console.WriteLine("Wrote " + eventArgs.RowsCopied + " records."); - // }; - sqlBulkCopy.WriteToServer(dataReader); - } - protected SqlConnection OpenAndGetSqlConnection() + public override void BulkSaveInStaging(IEnumerable entities) + { + var sqlBulkCopy = new SqlBulkCopy(OpenAndGetSqlConnection(), SqlBulkCopyOptions.Default, null); + sqlBulkCopy.DestinationTableName = SqlStagingTableName; + sqlBulkCopy.BulkCopyTimeout = 300; + sqlBulkCopy.BatchSize = entities.Count(); + foreach (var element in PropertiesToBulkLoad) + sqlBulkCopy.ColumnMappings.Add(element.Name, element.GetColumnName(base.StoreObject)); + sqlBulkCopy.ColumnMappings.Add(TempColumnNumOrderName, TempColumnNumOrderName); + + + var dataReader = new ObjectDataReader(entities, new ObjectDataReaderConfig { - var connection = Context.Database.GetDbConnection(); - if (connection.State != ConnectionState.Open) connection.Open(); - return (SqlConnection)connection; - } + EfProperties = PropertiesToBulkLoad, + Types = EntityTypes, + Context = Context, + TempColumnNumOrderName = TempColumnNumOrderName + }); + // sqlBulkCopy.SqlRowsCopied += (sender, eventArgs) => + // { + // Console.WriteLine("Wrote " + eventArgs.RowsCopied + " records."); + // }; + sqlBulkCopy.WriteToServer(dataReader); + } + protected SqlConnection OpenAndGetSqlConnection() + { + var connection = Context.Database.GetDbConnection(); + if (connection.State != ConnectionState.Open) connection.Open(); + return (SqlConnection)connection; + } - public override DataTable GetOutputStaging() - => base.StrictlyExecuteSql(GetOutputStagingSql()); + public override DataTable GetOutputStaging() + => base.StrictlyExecuteSql(GetOutputStagingSql()); - public override DataTable GetOutputStagingForComputedColumns() - => base.StrictlyExecuteSql(GetOutputStagingForComputedColumnsSql()); + public override DataTable GetOutputStagingForComputedColumns() + => base.StrictlyExecuteSql(GetOutputStagingForComputedColumnsSql()); - protected virtual string GetOutputStagingSql() - => $@"select * from {SqlOutputStagingTableName} order by [{TempColumnNumOrderName}] option(recompile)"; //the sp_execute is to prevent EF core to wrap the query into another subquery that will involve a different sorting depending in the situtation (not too proud of this solution, but I really didn't find better) - // => $@"sp_executesql N'set nocount on; select * from {SqlOutputStagingTableName} order by {TempColumnNumOrderName}';"; //the sp_execute is to prevent EF core to wrap the query into another subquery that will involve a different sorting depending in the situtation (not too proud of this solution, but I really didn't find better) + protected virtual string GetOutputStagingSql() + => $@"select * from {SqlOutputStagingTableName} order by [{TempColumnNumOrderName}] option(recompile)"; //the sp_execute is to prevent EF core to wrap the query into another subquery that will involve a different sorting depending in the situtation (not too proud of this solution, but I really didn't find better) + // => $@"sp_executesql N'set nocount on; select * from {SqlOutputStagingTableName} order by {TempColumnNumOrderName}';"; //the sp_execute is to prevent EF core to wrap the query into another subquery that will involve a different sorting depending in the situtation (not too proud of this solution, but I really didn't find better) - protected virtual string GetOutputStagingForComputedColumnsSql() - => $@"select T.* from {SqlStagingTableName} as S left join {SqlTargetTable} as T on {string.Join(" OR ", this.PropertiesForPivotSet.Select(propertiesForPivot => string.Join(" AND ", propertiesForPivot.Select(i => CreateEqualityConditionSql("T", "S", i)))))} order by S.{TempColumnNumOrderName} option(recompile)"; + protected virtual string GetOutputStagingForComputedColumnsSql() + => $@"select T.* from {SqlStagingTableName} as S left join {SqlTargetTable} as T on {string.Join(" OR ", this.PropertiesForPivotSet.Select(propertiesForPivot => string.Join(" AND ", propertiesForPivot.Select(i => CreateEqualityConditionSql("T", "S", i)))))} order by S.{TempColumnNumOrderName} option(recompile)"; - public override void MergeFromStaging(bool doNotUpdateIfExists = false) - => this.Context.Database.ExecuteSqlRaw(this.MergeFromStagingSql(doNotUpdateIfExists)); + public override void MergeFromStaging(bool doNotUpdateIfExists = false) + => this.Context.Database.ExecuteSqlRaw(this.MergeFromStagingSql(doNotUpdateIfExists)); - public override void InsertFromStaging() - => this.Context.Database.ExecuteSqlRaw(this.InsertFromStagingSql()); + public override void InsertFromStaging() + => this.Context.Database.ExecuteSqlRaw(this.InsertFromStagingSql()); - private string CreateEqualityConditionSql(string aliasLeft, string aliasRight, IProperty property) - { - string regularEquality = $"{aliasLeft}.[{property.GetColumnName(base.StoreObject)}] = {aliasRight}.[{property.GetColumnName(base.StoreObject)}]"; - if (property.IsColumnNullable()) - return $"{aliasRight}.[{property.GetColumnName(base.StoreObject)}] is not null and {regularEquality}"; - else - return regularEquality; - } - private string BuildPivotCriteria() - { - return string.Join(" OR ", PropertiesForPivotSet.Select(this.BuildUnitPivotCriteria)); - } - private string BuildUnitPivotCriteria(List propertiesForPivot) - { - return "(" + string.Join(" AND ", propertiesForPivot.Select(i => CreateEqualityConditionSql("T", "S", i))) + ")"; - } - private string GetWhenMatchedMergeStatement(HashSet pivotColumns, bool doNotUpdateIfExists) + private string CreateEqualityConditionSql(string aliasLeft, string aliasRight, IProperty property) + { + string regularEquality = $"{aliasLeft}.[{property.GetColumnName(base.StoreObject)}] = {aliasRight}.[{property.GetColumnName(base.StoreObject)}]"; + if (property.IsColumnNullable()) + return $"{aliasRight}.[{property.GetColumnName(base.StoreObject)}] is not null and {regularEquality}"; + else + return regularEquality; + } + private string BuildPivotCriteria() + { + return string.Join(" OR ", PropertiesForPivotSet.Select(this.BuildUnitPivotCriteria)); + } + private string BuildUnitPivotCriteria(List propertiesForPivot) + { + return "(" + string.Join(" AND ", propertiesForPivot.Select(i => CreateEqualityConditionSql("T", "S", i))) + ")"; + } + private string GetWhenMatchedMergeStatement(HashSet pivotColumns, bool doNotUpdateIfExists) + { + if (PropertiesToUpdate.Count == 0) + return ""; + if (doNotUpdateIfExists) { - if (PropertiesToUpdate.Count == 0) - return ""; - if (doNotUpdateIfExists) - { - return $@"WHEN MATCHED THEN - UPDATE SET {string.Join(", ", PropertiesToUpdate.Select(i => i.GetColumnName(base.StoreObject)).Select(columnName => $"T.[{columnName}] = T.[{columnName}]").ToArray())}"; - } return $@"WHEN MATCHED THEN - UPDATE SET {string.Join(", ", PropertiesToUpdate.Select(i => - { - var columnName = i.GetColumnName(base.StoreObject); - if (pivotColumns.Contains(columnName) && i.IsColumnNullable()) - return $"T.[{columnName}] = ISNULL(T.[{columnName}], S.[{columnName}])"; - else - return $"T.[{columnName}] = S.[{columnName}]"; - }))}"; + UPDATE SET {string.Join(", ", PropertiesToUpdate.Select(i => i.GetColumnName(base.StoreObject)).Select(columnName => $"T.[{columnName}] = T.[{columnName}]").ToArray())}"; } - protected virtual string MergeFromStagingSql(bool doNotUpdateIfExists = false) + return $@"WHEN MATCHED THEN + UPDATE SET {string.Join(", ", PropertiesToUpdate.Select(i => { - var pivotColumns = PropertiesForPivotSet.SelectMany(p => p.Select(i => i.GetColumnName(base.StoreObject))).ToHashSet(); - string whenNotMatchedStatement = $@"WHEN NOT MATCHED BY TARGET THEN + var columnName = i.GetColumnName(base.StoreObject); + if (string.IsNullOrWhiteSpace(columnName)) + throw new Exception("Column name is empty"); + if (pivotColumns.Contains(columnName) && i.IsColumnNullable()) + return $"T.[{columnName}] = ISNULL(T.[{columnName}], S.[{columnName}])"; + else + return $"T.[{columnName}] = S.[{columnName}]"; + }))}"; + } + protected virtual string MergeFromStagingSql(bool doNotUpdateIfExists = false) + { + var pivotColumns = PropertiesForPivotSet.SelectMany(p => p.Select(i => i.GetColumnName(base.StoreObject)).Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => i!)).ToHashSet(); + string whenNotMatchedStatement = $@"WHEN NOT MATCHED BY TARGET THEN INSERT ({string.Join(", ", PropertiesToInsert.Select(i => $"[{i.GetColumnName(base.StoreObject)}]"))}) VALUES ({string.Join(", ", PropertiesToInsert.Select(i => $"S.[{i.GetColumnName(base.StoreObject)}]"))})"; - string whenMatchedStatement = this.GetWhenMatchedMergeStatement(pivotColumns, doNotUpdateIfExists); - string pivotCriteria = this.BuildPivotCriteria(); - return $@"MERGE {SqlTargetTable} AS T + string whenMatchedStatement = this.GetWhenMatchedMergeStatement(pivotColumns, doNotUpdateIfExists); + string pivotCriteria = this.BuildPivotCriteria(); + return $@"MERGE {SqlTargetTable} AS T USING {SqlStagingTableName} AS S ON {pivotCriteria} {whenNotMatchedStatement} {whenMatchedStatement} OUTPUT INSERTED.*, S.[{TempColumnNumOrderName}], $action INTO {SqlOutputStagingTableName} option(recompile);"; - } - protected virtual string InsertFromStagingSql() - { - string whenNotMatchedStatement = $@"WHEN NOT MATCHED BY TARGET THEN + } + protected virtual string InsertFromStagingSql() + { + string whenNotMatchedStatement = $@"WHEN NOT MATCHED BY TARGET THEN INSERT ({string.Join(", ", PropertiesToInsert.Select(i => $"[{i.GetColumnName(base.StoreObject)}]"))}) VALUES ({string.Join(", ", PropertiesToInsert.Select(i => $"S.[{i.GetColumnName(base.StoreObject)}]"))})"; - return $@"MERGE {SqlTargetTable} AS T + return $@"MERGE {SqlTargetTable} AS T USING {SqlStagingTableName} AS S ON 1=0 {whenNotMatchedStatement} OUTPUT INSERTED.*, S.[{TempColumnNumOrderName}], $action INTO {SqlOutputStagingTableName} option(recompile);"; - } } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerUpdateContextQuery.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerUpdateContextQuery.cs index 92fe88de..d7bd0fc0 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerUpdateContextQuery.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/SqlServer/SqlServerUpdateContextQuery.cs @@ -4,90 +4,86 @@ using System; using System.Collections.Generic; using System.Data; -// using Microsoft.Data.SqlClient; using System.Linq; using System.Reflection; -using System.Text; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave.SqlServer +namespace Paillave.EntityFrameworkCoreExtension.BulkSave.SqlServer; +public class SqlServerUpdateContextQuery : UpdateContextQueryBase { - public class SqlServerUpdateContextQuery : UpdateContextQueryBase - { - private string SqlTargetTable => $"[{this.Schema}].[{this.Table}]"; - private string SqlStagingTableName => $"[{this.Schema}].[{this.Table}_temp_{this.StagingId}]"; + private string SqlTargetTable => $"[{this.Schema}].[{this.Table}]"; + private string SqlStagingTableName => $"[{this.Schema}].[{this.Table}_temp_{this.StagingId}]"; - public SqlServerUpdateContextQuery(DbContext context, string schema, string table, List propertiesToUpdate, List propertiesForPivot, List propertiesToBulkLoad, IEntityType baseType, IDictionary propertiesGetter, StoreObjectIdentifier storeObject) - : base(context, schema ?? "dbo", table, propertiesToUpdate, propertiesForPivot, propertiesToBulkLoad, baseType, propertiesGetter, storeObject) - { - } + public SqlServerUpdateContextQuery(DbContext context, string schema, string table, List propertiesToUpdate, List propertiesForPivot, List propertiesToBulkLoad, IEntityType baseType, IDictionary propertiesGetter, StoreObjectIdentifier storeObject) + : base(context, schema ?? "dbo", table, propertiesToUpdate, propertiesForPivot, propertiesToBulkLoad, baseType, propertiesGetter, storeObject) + { + } - protected virtual string CreateStagingTableSql() - => $@"SELECT TOP 0 { string.Join(",", PropertiesToBulkLoad.Select(i => $"T.{i.GetColumnName(base.StoreObject)}")) } + protected virtual string CreateStagingTableSql() + => $@"SELECT TOP 0 {string.Join(",", PropertiesToBulkLoad.Select(i => $"T.{i.GetColumnName(base.StoreObject)}"))} INTO {SqlStagingTableName} FROM {SqlTargetTable} AS T LEFT JOIN {SqlTargetTable} AS Source ON 1 = 0;"; - public override void CreateStagingTable() - => this.Context.Database.ExecuteSqlRaw(this.CreateStagingTableSql()); + public override void CreateStagingTable() + => this.Context.Database.ExecuteSqlRaw(this.CreateStagingTableSql()); - public override void BulkSaveInStaging(IEnumerable sources) - { - var sqlBulkCopy = new SqlBulkCopy(OpenAndGetSqlConnection(), SqlBulkCopyOptions.Default, null); - sqlBulkCopy.DestinationTableName = SqlStagingTableName; - sqlBulkCopy.BatchSize = sources.Count(); - foreach (var element in PropertiesToBulkLoad) - sqlBulkCopy.ColumnMappings.Add(base.PropertyGetters[element.Name].Name, element.GetColumnName(base.StoreObject)); + public override void BulkSaveInStaging(IEnumerable sources) + { + var sqlBulkCopy = new SqlBulkCopy(OpenAndGetSqlConnection(), SqlBulkCopyOptions.Default, null); + sqlBulkCopy.DestinationTableName = SqlStagingTableName; + sqlBulkCopy.BatchSize = sources.Count(); + foreach (var element in PropertiesToBulkLoad) + sqlBulkCopy.ColumnMappings.Add(base.PropertyGetters[element.Name].Name, element.GetColumnName(base.StoreObject)); - var dataReader = new ObjectDataReader(sources, new ObjectDataReaderConfig - { - EfProperties = PropertiesToBulkLoad, - Types = new[] { base.BaseType }, - Context = Context, - TempColumnNumOrderName = null - }); - sqlBulkCopy.WriteToServer(dataReader); - } + var dataReader = new ObjectDataReader(sources, new ObjectDataReaderConfig + { + EfProperties = PropertiesToBulkLoad, + Types = new[] { base.BaseType }, + Context = Context, + TempColumnNumOrderName = null + }); + sqlBulkCopy.WriteToServer(dataReader); + } - public override void IndexStagingTable() - => this.Context.Database.ExecuteSqlRaw(this.IndexStagingTableSql()); + public override void IndexStagingTable() + => this.Context.Database.ExecuteSqlRaw(this.IndexStagingTableSql()); - protected virtual string IndexStagingTableSql() - => $@"CREATE UNIQUE CLUSTERED INDEX IX_{this.Table}_temp_{this.StagingId} ON {SqlStagingTableName} ({string.Join(", ", this.PropertiesForPivot.Select(i => i.GetColumnName(base.StoreObject)))})"; + protected virtual string IndexStagingTableSql() + => $@"CREATE UNIQUE CLUSTERED INDEX IX_{this.Table}_temp_{this.StagingId} ON {SqlStagingTableName} ({string.Join(", ", this.PropertiesForPivot.Select(i => i.GetColumnName(base.StoreObject)))})"; - public override void DeleteStagingTable() - => this.Context.Database.ExecuteSqlRaw(this.DeleteStagingTableSql()); + public override void DeleteStagingTable() + => this.Context.Database.ExecuteSqlRaw(this.DeleteStagingTableSql()); - protected virtual string DeleteStagingTableSql() - => $@"IF OBJECT_ID('{SqlStagingTableName}', 'U') IS NOT NULL DROP TABLE {SqlStagingTableName}"; + protected virtual string DeleteStagingTableSql() + => $@"IF OBJECT_ID('{SqlStagingTableName}', 'U') IS NOT NULL DROP TABLE {SqlStagingTableName}"; - protected SqlConnection OpenAndGetSqlConnection() - { - var connection = Context.Database.GetDbConnection(); - if (connection.State != ConnectionState.Open) connection.Open(); - return (SqlConnection)connection; - } + protected SqlConnection OpenAndGetSqlConnection() + { + var connection = Context.Database.GetDbConnection(); + if (connection.State != ConnectionState.Open) connection.Open(); + return (SqlConnection)connection; + } - public override void MergeFromStaging() - => this.Context.Database.ExecuteSqlRaw(this.MergeFromStagingSql()); + public override void MergeFromStaging() + => this.Context.Database.ExecuteSqlRaw(this.MergeFromStagingSql()); - private string CreateEqualityConditionSql(string aliasLeft, string aliasRight, IProperty property) - { - string regularEquality = $"{aliasLeft}.{property.GetColumnName(base.StoreObject)} = {aliasRight}.{property.GetColumnName(base.StoreObject)}"; - if (property.IsColumnNullable()) - return $"({aliasLeft}.{property.GetColumnName(base.StoreObject)} is null and {regularEquality})"; - else - return regularEquality; - } - private string CreateSetValueSql(string aliasRight, IProperty property) - => $"{property.GetColumnName(base.StoreObject)} = {aliasRight}.{property.GetColumnName(base.StoreObject)}"; - - protected virtual string MergeFromStagingSql() - { - return $@"UDPATE t + private string CreateEqualityConditionSql(string aliasLeft, string aliasRight, IProperty property) + { + string regularEquality = $"{aliasLeft}.{property.GetColumnName(base.StoreObject)} = {aliasRight}.{property.GetColumnName(base.StoreObject)}"; + if (property.IsColumnNullable()) + return $"({aliasLeft}.{property.GetColumnName(base.StoreObject)} is null and {regularEquality})"; + else + return regularEquality; + } + private string CreateSetValueSql(string aliasRight, IProperty property) + => $"{property.GetColumnName(base.StoreObject)} = {aliasRight}.{property.GetColumnName(base.StoreObject)}"; + + protected virtual string MergeFromStagingSql() + { + return $@"UDPATE t SET {string.Join(", ", this.PropertiesToUpdate.Select(i => CreateSetValueSql("s", i)))} FROM {this.SqlStagingTableName} AS s INNER JOIN {this.SqlTargetTable} AS t ON {string.Join(" AND ", this.PropertiesForPivot.Select(i => CreateEqualityConditionSql("s", "t", i)))}"; - } } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/UpdateContextQueryBase.cs b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/UpdateContextQueryBase.cs index 1801f4c3..337df39d 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/BulkSave/UpdateContextQueryBase.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/BulkSave/UpdateContextQueryBase.cs @@ -2,64 +2,60 @@ using Microsoft.EntityFrameworkCore.Metadata; using System; using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; using System.Reflection; -using System.Text; -namespace Paillave.EntityFrameworkCoreExtension.BulkSave -{ - public abstract class UpdateContextQueryBase - { - protected string Table { get; } - protected string StagingId { get; } = Guid.NewGuid().ToString().Substring(0, 8); - protected string Schema { get; } - protected IEntityType BaseType { get; } - protected StoreObjectIdentifier StoreObject { get; } - protected IDictionary PropertyGetters { get; } - /// - /// any column except pivot, computed - /// - protected List PropertiesToUpdate { get; } - /// - /// pivot columns - /// - protected List PropertiesForPivot { get; } +namespace Paillave.EntityFrameworkCoreExtension.BulkSave; - /// - /// Any column except computed that is not pivot - /// - protected List PropertiesToBulkLoad { get; } +public abstract class UpdateContextQueryBase +{ + protected string Table { get; } + protected string StagingId { get; } = Guid.NewGuid().ToString().Substring(0, 8); + protected string Schema { get; } + protected IEntityType BaseType { get; } + protected StoreObjectIdentifier StoreObject { get; } + protected IDictionary PropertyGetters { get; } + /// + /// any column except pivot, computed + /// + protected List PropertiesToUpdate { get; } + /// + /// pivot columns + /// + protected List PropertiesForPivot { get; } - protected DbContext Context { get; } - public UpdateContextQueryBase(DbContext context, string schema, string table, List propertiesToUpdate, List propertiesForPivot, List propertiesToBulkLoad, IEntityType baseType, IDictionary propertyGetters, StoreObjectIdentifier storeObject) - { - this.StoreObject = storeObject; - this.PropertiesToUpdate = propertiesToUpdate; - this.PropertiesForPivot = propertiesForPivot; - this.PropertiesToBulkLoad = propertiesToBulkLoad; - this.PropertyGetters = propertyGetters; + /// + /// Any column except computed that is not pivot + /// + protected List PropertiesToBulkLoad { get; } - this.Schema = schema; - this.Table = table; - this.Context = context; - this.BaseType = baseType; - } + protected DbContext Context { get; } + public UpdateContextQueryBase(DbContext context, string schema, string table, List propertiesToUpdate, List propertiesForPivot, List propertiesToBulkLoad, IEntityType baseType, IDictionary propertyGetters, StoreObjectIdentifier storeObject) + { + this.StoreObject = storeObject; + this.PropertiesToUpdate = propertiesToUpdate; + this.PropertiesForPivot = propertiesForPivot; + this.PropertiesToBulkLoad = propertiesToBulkLoad; + this.PropertyGetters = propertyGetters; - /// - /// Create the staging that is meant to receive the raw bulk load - /// - public abstract void CreateStagingTable(); - /// - /// save entities in the staging table - /// - /// entities that will saved in staging - public abstract void BulkSaveInStaging(IEnumerable sources); - /// - /// merge the staging table in the target table - /// - public abstract void MergeFromStaging(); - public abstract void IndexStagingTable(); - public abstract void DeleteStagingTable(); + this.Schema = schema; + this.Table = table; + this.Context = context; + this.BaseType = baseType; } + + /// + /// Create the staging that is meant to receive the raw bulk load + /// + public abstract void CreateStagingTable(); + /// + /// save entities in the staging table + /// + /// entities that will saved in staging + public abstract void BulkSaveInStaging(IEnumerable sources); + /// + /// merge the staging table in the target table + /// + public abstract void MergeFromStaging(); + public abstract void IndexStagingTable(); + public abstract void DeleteStagingTable(); } diff --git a/src/Paillave.EntityFrameworkCoreExtension/EntityTypeBuilderEx.cs b/src/Paillave.EntityFrameworkCoreExtension/EntityTypeBuilderEx.cs index c6b86a43..895de433 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/EntityTypeBuilderEx.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/EntityTypeBuilderEx.cs @@ -10,14 +10,14 @@ public static EntityTypeBuilder ToTable(this EntityTypeBuilder { var type = typeof(TEntity); var tableName = type.Name; - var schemaName = type.Namespace.Split('.').Last(); + var schemaName = type.Namespace?.Split('.').Last(); return entityTypeBuilder.ToTable(tableName, schemaName); } - public static OwnedNavigationBuilder ToTable(this OwnedNavigationBuilder entityTypeBuilder) where TFrom : class where TTo:class + public static OwnedNavigationBuilder ToTable(this OwnedNavigationBuilder entityTypeBuilder) where TFrom : class where TTo : class { var type = typeof(TTo); var tableName = type.Name; - var schemaName = type.Namespace.Split('.').Last(); + var schemaName = type.Namespace?.Split('.').Last(); return entityTypeBuilder.ToTable(tableName, schemaName); } } diff --git a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj index 898577d4..d703cd71 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj +++ b/src/Paillave.EntityFrameworkCoreExtension/Paillave.EntityFrameworkCoreExtension.csproj @@ -6,6 +6,7 @@ ETL .net core EF reactive Entity Framework extensions Entity Framework Extensions + enable diff --git a/src/Paillave.EntityFrameworkCoreExtension/Searcher/FieldSelector.cs b/src/Paillave.EntityFrameworkCoreExtension/Searcher/FieldSelector.cs index f06ca248..6871c784 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Searcher/FieldSelector.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/Searcher/FieldSelector.cs @@ -1,21 +1,17 @@ using System; -using System.Collections.Generic; -using System.Linq; using System.Linq.Expressions; -namespace Paillave.EntityFrameworkCoreExtension.Searcher +namespace Paillave.EntityFrameworkCoreExtension.Searcher; +public class FieldSelector : IFieldSelector where TEntity : class { - public class FieldSelector : IFieldSelector where TEntity : class + public FieldSelector(string name, Expression> getValue, Func convertFromString) { - public FieldSelector(string name, Expression> getValue, Func convertFromString) - { - this.Name = name; - this.GetValueExpression = getValue; - this.ConvertFromString = i => convertFromString(i); - } - public string Name { get; } - // Expression> - public LambdaExpression GetValueExpression { get; } - public Func ConvertFromString { get; } + this.Name = name; + this.GetValueExpression = getValue; + this.ConvertFromString = i => convertFromString(i); } -} \ No newline at end of file + public string Name { get; } + // Expression> + public LambdaExpression GetValueExpression { get; } + public Func ConvertFromString { get; } +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/Searcher/GroupingValue.cs b/src/Paillave.EntityFrameworkCoreExtension/Searcher/GroupingValue.cs index 775b8e1a..49ec3844 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Searcher/GroupingValue.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/Searcher/GroupingValue.cs @@ -1,16 +1,15 @@ using System; -namespace Paillave.EntityFrameworkCoreExtension.Searcher +namespace Paillave.EntityFrameworkCoreExtension.Searcher; + +/// +/// Permits to make a dictionary on a nullable key +/// +public struct GroupingValue : IEquatable { - /// - /// Permits to make a dictionary on a nullable key - /// - public struct GroupingValue : IEquatable - { - public GroupingValue(object value = null) => Value = value; - public object Value { get; } - public override int GetHashCode() => this.Value?.GetHashCode() ?? 0; - public override string ToString() => this.Value?.ToString() ?? ""; - public bool Equals(GroupingValue v) => v.Value == Value; - } -} \ No newline at end of file + public GroupingValue(object? value = null) => Value = value; + public object? Value { get; } + public override int GetHashCode() => this.Value?.GetHashCode() ?? 0; + public override string ToString() => this.Value?.ToString() ?? ""; + public bool Equals(GroupingValue v) => v.Value == Value; +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/Searcher/IFieldSelector.cs b/src/Paillave.EntityFrameworkCoreExtension/Searcher/IFieldSelector.cs index 3ab92519..62c88c69 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Searcher/IFieldSelector.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/Searcher/IFieldSelector.cs @@ -1,13 +1,10 @@ using System; -using System.Collections.Generic; using System.Linq.Expressions; -namespace Paillave.EntityFrameworkCoreExtension.Searcher +namespace Paillave.EntityFrameworkCoreExtension.Searcher; +public interface IFieldSelector { - public interface IFieldSelector - { - string Name { get; } - LambdaExpression GetValueExpression { get; } - Func ConvertFromString { get; } - } -} \ No newline at end of file + string Name { get; } + LambdaExpression GetValueExpression { get; } + Func ConvertFromString { get; } +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/Searcher/INavigationSelector.cs b/src/Paillave.EntityFrameworkCoreExtension/Searcher/INavigationSelector.cs index 327ed4f4..d77201e0 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Searcher/INavigationSelector.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/Searcher/INavigationSelector.cs @@ -1,11 +1,10 @@ using System.Linq.Expressions; -namespace Paillave.EntityFrameworkCoreExtension.Searcher +namespace Paillave.EntityFrameworkCoreExtension.Searcher; + +public interface INavigationSelector { - public interface INavigationSelector - { - string Name { get; } - ISearchDescriptor ObjectDescriptor { get; } - LambdaExpression GetTargetExpression { get; } - } -} \ No newline at end of file + string Name { get; } + ISearchDescriptor ObjectDescriptor { get; } + LambdaExpression GetTargetExpression { get; } +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/Searcher/ISearchDescriptor.cs b/src/Paillave.EntityFrameworkCoreExtension/Searcher/ISearchDescriptor.cs index 115c3904..0a46e5dc 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Searcher/ISearchDescriptor.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/Searcher/ISearchDescriptor.cs @@ -1,21 +1,14 @@ -using System; using System.Collections.Generic; -using System.Linq; using System.Linq.Expressions; -using System.Threading.Tasks; -namespace Paillave.EntityFrameworkCoreExtension.Searcher +namespace Paillave.EntityFrameworkCoreExtension.Searcher; + +public interface ISearchDescriptor { - public interface ISearchDescriptor - { - string Name { get; } - SearchMetadata GetMetadata(); - IDictionary Navigations { get; } - IDictionary Fields { get; } - IFieldSelector DefaultValueProperty { get; } - Expression GetFilterExpression(Dictionary> filters); - // Expression GetValueExpression(string pathToProperty); - // List> SearchIds(IDictionary> filters, string pathToGroupProperty); - // List SearchIds(IDictionary> filters); - } -} \ No newline at end of file + string Name { get; } + SearchMetadata GetMetadata(); + IDictionary Navigations { get; } + IDictionary Fields { get; } + IFieldSelector DefaultValueProperty { get; } + Expression GetFilterExpression(Dictionary> filters); +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/Searcher/NavigationSelector.cs b/src/Paillave.EntityFrameworkCoreExtension/Searcher/NavigationSelector.cs index cf713096..7c87cb9c 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Searcher/NavigationSelector.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/Searcher/NavigationSelector.cs @@ -1,19 +1,18 @@ using System; using System.Linq.Expressions; -namespace Paillave.EntityFrameworkCoreExtension.Searcher +namespace Paillave.EntityFrameworkCoreExtension.Searcher; + +public class NavigationSelector : INavigationSelector where TEntity : class where TTarget : class { - public class NavigationSelector : INavigationSelector where TEntity : class where TTarget : class + public NavigationSelector(string name, Expression> getTargetExpression, SearchDescriptorBase levelDescriptor) { - public NavigationSelector(string name, Expression> getTargetExpression, SearchDescriptorBase levelDescriptor) - { - this.Name = name; - this.GetTargetExpression = getTargetExpression; - this.ObjectDescriptor = levelDescriptor; - } - // Expression> - public LambdaExpression GetTargetExpression { get; } - public string Name { get; set; } - public ISearchDescriptor ObjectDescriptor { get; } + this.Name = name; + this.GetTargetExpression = getTargetExpression; + this.ObjectDescriptor = levelDescriptor; } -} \ No newline at end of file + // Expression> + public LambdaExpression GetTargetExpression { get; } + public string Name { get; set; } + public ISearchDescriptor ObjectDescriptor { get; } +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchDescriptorBase.cs b/src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchDescriptorBase.cs index 431ebab2..94789bcb 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchDescriptorBase.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchDescriptorBase.cs @@ -4,209 +4,206 @@ using System.Linq.Expressions; using System.Reflection; using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Paillave.EntityFrameworkCoreExtension.Core; -namespace Paillave.EntityFrameworkCoreExtension.Searcher +namespace Paillave.EntityFrameworkCoreExtension.Searcher; + +public abstract class SearchDescriptorBase : ISearchDescriptor where TEntity : class { - public abstract class SearchDescriptorBase : ISearchDescriptor where TEntity : class - { - // the actual regex (without c# escapings): - // \.(?=(?:[^"]*"[^"]*")*(?![^"]*")) - //private static Regex _regexSplitter = new Regex("\\.(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))", RegexOptions.Singleline); - protected DbContext DbContext { get; } - private Func, IQueryable> _include = null; - public SearchDescriptorBase(DbContext dbContext) - { - this.DbContext = dbContext; - } - protected abstract Expression> GetIdExpression { get; } - public virtual string Name => typeof(TEntity).Name; - public IDictionary Navigations { get; } = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - public IDictionary Fields { get; } = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - public IFieldSelector DefaultValueProperty { get; private set; } = null; - public SearchMetadata GetMetadata() => this.GetStructure(this.Name, this); - private SearchMetadata GetStructure(string name, ISearchDescriptor descriptor) - => new SearchMetadata( - name, - descriptor.Name, - descriptor.Fields.Keys - .Select(k => new SearchMetadata(k, descriptor.Name)) - .Union(descriptor.Navigations.Select(v => GetStructure(v.Key, v.Value.ObjectDescriptor)) - ).ToList()); - public SearchDescriptorBase AddValue(string name, bool isOnKey, Expression> getValue, Func convert) - { - var valueProperty = new FieldSelector(name, getValue, convert); - this.Fields[name] = valueProperty; - if (isOnKey) this.DefaultValueProperty = valueProperty; - return this; - } - public SearchDescriptorBase AddValue(string name, bool isOnKey, Expression> getValue) - => this.AddValue(name, isOnKey, getValue, i => i); - public SearchDescriptorBase SetIncludes(Func, IQueryable> include) - { - _include = include; - return this; - } - public SearchDescriptorBase AddNavigation(string name, SearchDescriptorBase levelDescriptor, Expression> getTargetExpression) where TTarget : class - { - var navigationProperty = new NavigationSelector(name, getTargetExpression, levelDescriptor); - this.Navigations[name] = navigationProperty; - return this; - } - Expression ISearchDescriptor.GetFilterExpression(Dictionary> filters) - => this.GetFilterExpression(filters); - public Expression> GetFilterExpression(Dictionary> filters) - { - var entityParameterExpression = Expression.Parameter(typeof(TEntity), "entity"); + // the actual regex (without c# escapings): + // \.(?=(?:[^"]*"[^"]*")*(?![^"]*")) + //private static Regex _regexSplitter = new Regex("\\.(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))", RegexOptions.Singleline); + protected DbContext DbContext { get; } + private Func, IQueryable> _include = null; + public SearchDescriptorBase(DbContext dbContext) + { + this.DbContext = dbContext; + } + protected abstract Expression> GetIdExpression { get; } + public virtual string Name => typeof(TEntity).Name; + public IDictionary Navigations { get; } = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + public IDictionary Fields { get; } = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + public IFieldSelector DefaultValueProperty { get; private set; } = null; + public SearchMetadata GetMetadata() => this.GetStructure(this.Name, this); + private SearchMetadata GetStructure(string name, ISearchDescriptor descriptor) + => new SearchMetadata( + name, + descriptor.Name, + descriptor.Fields.Keys + .Select(k => new SearchMetadata(k, descriptor.Name)) + .Union(descriptor.Navigations.Select(v => GetStructure(v.Key, v.Value.ObjectDescriptor)) + ).ToList()); + public SearchDescriptorBase AddValue(string name, bool isOnKey, Expression> getValue, Func convert) + { + var valueProperty = new FieldSelector(name, getValue, convert); + this.Fields[name] = valueProperty; + if (isOnKey) this.DefaultValueProperty = valueProperty; + return this; + } + public SearchDescriptorBase AddValue(string name, bool isOnKey, Expression> getValue) + => this.AddValue(name, isOnKey, getValue, i => i); + public SearchDescriptorBase SetIncludes(Func, IQueryable> include) + { + _include = include; + return this; + } + public SearchDescriptorBase AddNavigation(string name, SearchDescriptorBase levelDescriptor, Expression> getTargetExpression) where TTarget : class + { + var navigationProperty = new NavigationSelector(name, getTargetExpression, levelDescriptor); + this.Navigations[name] = navigationProperty; + return this; + } + Expression ISearchDescriptor.GetFilterExpression(Dictionary> filters) + => this.GetFilterExpression(filters); + public Expression> GetFilterExpression(Dictionary> filters) + { + var entityParameterExpression = Expression.Parameter(typeof(TEntity), "entity"); - Expression filterExpression = null; - foreach (var filter in filters) - { - var subFilter = GetFilterExpressionBody(entityParameterExpression, filter.Key, filter.Value); - if (filterExpression == null) - filterExpression = subFilter; - else - filterExpression = Expression.AndAlso(filterExpression, subFilter); - } - if (filterExpression == null) - return i => true; - return (Expression>)Expression.Lambda(filterExpression, entityParameterExpression); - } - // Expression IObjectDescriptor.GetFilterExpression(string pathToProperty, IEnumerable acceptedValues, IEnumerable levelDescriptors) - // => this.GetFilterExpression(pathToProperty, acceptedValues, levelDescriptors); - // private Type GetExpressionType(EXpression expression) - // { - // switch (expression) - // { - // case ParameterExpression parameterExpression: return parameterExpression.Type; - // case InvocationExpression invocationExpression: return invocationExpression.Type; - // } - // } - private Expression GetFilterExpressionBody(ParameterExpression entityParameterExpression, string pathToProperty, IEnumerable acceptedValues) + Expression filterExpression = null; + foreach (var filter in filters) { - (var valueExpressionBody, var converter) = GetValueBodyExpression(entityParameterExpression, pathToProperty); - var vals = CallMethod(nameof(ConvertAcceptedValues), new[] { valueExpressionBody.Type }, new object[] { acceptedValues, converter }); - var valueListExpression = Expression.Constant(vals); - var queryableType = typeof(Enumerable); - var containsMethod = queryableType.GetMethods() - .First(i => i.Name == nameof(Queryable.Contains) && i.IsGenericMethod && i.GetGenericArguments().Length == 1 && i.GetParameters().Length == 2 && i.IsStatic); - var typedContainsMethod = containsMethod.MakeGenericMethod(valueExpressionBody.Type); - var containsCallExpression = Expression.Call(typedContainsMethod, valueListExpression, valueExpressionBody); - return containsCallExpression; + var subFilter = GetFilterExpressionBody(entityParameterExpression, filter.Key, filter.Value); + if (filterExpression == null) + filterExpression = subFilter; + else + filterExpression = Expression.AndAlso(filterExpression, subFilter); } - // Expression GetValueExpression(string pathToProperty, List levelDescriptors) - // => ; - // Expression IObjectDescriptor.GetValueExpression(string pathToProperty, IEnumerable levelDescriptors) - // => this.GetValueExpression(pathToProperty, levelDescriptors); - private TElement[] ConvertAcceptedValues(IEnumerable acceptedValues, Func converter) - => acceptedValues.Select(i => (TElement)converter(i)).ToArray(); - private List ParsePropertyPath(string line) => JsonSerializer.Deserialize>(line); - private (Expression BodyExpression, Func Converter) GetValueBodyExpression(ParameterExpression entityParameter, string pathToProperty) - { - if (string.IsNullOrWhiteSpace(pathToProperty)) - return (entityParameter, i => i); + if (filterExpression == null) + return i => true; + return (Expression>)Expression.Lambda(filterExpression, entityParameterExpression); + } + // Expression IObjectDescriptor.GetFilterExpression(string pathToProperty, IEnumerable acceptedValues, IEnumerable levelDescriptors) + // => this.GetFilterExpression(pathToProperty, acceptedValues, levelDescriptors); + // private Type GetExpressionType(EXpression expression) + // { + // switch (expression) + // { + // case ParameterExpression parameterExpression: return parameterExpression.Type; + // case InvocationExpression invocationExpression: return invocationExpression.Type; + // } + // } + private Expression GetFilterExpressionBody(ParameterExpression entityParameterExpression, string pathToProperty, IEnumerable acceptedValues) + { + (var valueExpressionBody, var converter) = GetValueBodyExpression(entityParameterExpression, pathToProperty); + var vals = CallMethod(nameof(ConvertAcceptedValues), new[] { valueExpressionBody.Type }, new object[] { acceptedValues, converter }); + var valueListExpression = Expression.Constant(vals); + var queryableType = typeof(Enumerable); + var containsMethod = queryableType.GetMethods() + .First(i => i.Name == nameof(Queryable.Contains) && i.IsGenericMethod && i.GetGenericArguments().Length == 1 && i.GetParameters().Length == 2 && i.IsStatic); + var typedContainsMethod = containsMethod.MakeGenericMethod(valueExpressionBody.Type); + var containsCallExpression = Expression.Call(typedContainsMethod, valueListExpression, valueExpressionBody); + return containsCallExpression; + } + // Expression GetValueExpression(string pathToProperty, List levelDescriptors) + // => ; + // Expression IObjectDescriptor.GetValueExpression(string pathToProperty, IEnumerable levelDescriptors) + // => this.GetValueExpression(pathToProperty, levelDescriptors); + private TElement[] ConvertAcceptedValues(IEnumerable acceptedValues, Func converter) + => acceptedValues.Select(i => (TElement)converter(i)).ToArray(); + private List ParsePropertyPath(string line) => JsonSerializer.Deserialize>(line); + private (Expression BodyExpression, Func Converter) GetValueBodyExpression(ParameterExpression entityParameter, string pathToProperty) + { + if (string.IsNullOrWhiteSpace(pathToProperty)) + return (entityParameter, i => i); - var pathSegments = this.ParsePropertyPath(pathToProperty); + var pathSegments = this.ParsePropertyPath(pathToProperty); - // INavigationSelector navigationSelector = null; - ISearchDescriptor objectDescriptor = this; - Expression valueExpression = entityParameter; - foreach (var pathSegment in pathSegments) + // INavigationSelector navigationSelector = null; + ISearchDescriptor objectDescriptor = this; + Expression valueExpression = entityParameter; + foreach (var pathSegment in pathSegments) + { + if (objectDescriptor.Navigations.TryGetValue(pathSegment, out var navigation)) { - if (objectDescriptor.Navigations.TryGetValue(pathSegment, out var navigation)) - { - valueExpression = GetExpressionBodyReferencingValue(navigation.GetTargetExpression, valueExpression); - // valueExpression = Expression.Invoke(navigation.GetTargetExpression, valueExpression); - objectDescriptor = navigation.ObjectDescriptor; - } - else if (objectDescriptor.Fields.TryGetValue(pathSegment, out var field)) - { - return (GetExpressionBodyReferencingValue(field.GetValueExpression, valueExpression), field.ConvertFromString); - // return (Expression.Invoke(field.GetValueExpression, valueExpression), field.ConvertFromString); - } + valueExpression = GetExpressionBodyReferencingValue(navigation.GetTargetExpression, valueExpression); + // valueExpression = Expression.Invoke(navigation.GetTargetExpression, valueExpression); + objectDescriptor = navigation.ObjectDescriptor; } - var defaultField = objectDescriptor.DefaultValueProperty; - return (GetExpressionBodyReferencingValue(defaultField.GetValueExpression, valueExpression), defaultField.ConvertFromString); - // return (Expression.Invoke(defaultField.GetValueExpression, valueExpression), defaultField.ConvertFromString); - } - private Expression GetExpressionBodyReferencingValue(LambdaExpression expression, Expression newParameterExpression) - { - var parameterToBeReplaced = expression.Parameters[0]; - var visitor = new ReplacementVisitor(parameterToBeReplaced, newParameterExpression); - return visitor.Visit(expression.Body); - } - private LambdaExpression GetValueLambdaExpression(string pathToProperty) - { - var entityParameterExpression = Expression.Parameter(typeof(TEntity), "entity"); - (var valueExpressionBody, var converter) = GetValueBodyExpression(entityParameterExpression, pathToProperty); - return Expression.Lambda(valueExpressionBody, entityParameterExpression); - } - private object CallMethod(string methodName, Type[] genericParameters, object[] args) - { - // var methods = this.GetType().GetMethods(BindingFlags.NonPublic | BindingFlags.Instance); - var method = typeof(SearchDescriptorBase<,>).MakeGenericType(typeof(TEntity), typeof(TId)).GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.FlattenHierarchy); - var groupQueryableMethodInfo = method.MakeGenericMethod(genericParameters); - return groupQueryableMethodInfo.Invoke(this, args); - } - public Dictionary> Search(Dictionary> filters, string pathToGroupProperty) - { - var queryable = Search(filters); - var groupingLambdaExpression = GetValueLambdaExpression(pathToGroupProperty); - var groupingType = groupingLambdaExpression.ReturnType; - return (Dictionary>)CallMethod(nameof(GroupQueryable), new[] { groupingLambdaExpression.ReturnType }, new object[] { queryable, groupingLambdaExpression }); - // var groupQueryableMethodInfo = this.GetType().GetMethod(nameof(GroupQueryable), BindingFlags.NonPublic | BindingFlags.Instance).MakeGenericMethod(groupingType); - // return (List>)groupQueryableMethodInfo.Invoke(this, new object[] { queryable, groupingLambdaExpression }); - } - private Dictionary> GroupQueryable(IQueryable queryable, Expression> groupingExpression) - { - var entityParameterExpression = Expression.Parameter(typeof(TEntity), "entity"); - var keyedRowType = typeof(KeyedRow<>).MakeGenericType(typeof(TEntity), typeof(TId), typeof(TKey)); - var constructorInfo = keyedRowType.GetConstructor(new Type[] { }); - var callConstructor = Expression.New(constructorInfo); - var initializedObject = Expression.MemberInit(callConstructor, - // Expression.Bind(keyedRowType.GetProperty(nameof(KeyedRow.Key)), Expression.Invoke(groupingExpression, entityParameterExpression)), - Expression.Bind(keyedRowType.GetProperty(nameof(KeyedRow.Key)), GetExpressionBodyReferencingValue(groupingExpression, entityParameterExpression)), - Expression.Bind(keyedRowType.GetProperty(nameof(KeyedRow.Row)), entityParameterExpression)); - var getGroupingKeyLambdaExpression = (Expression>>)Expression.Lambda(initializedObject, entityParameterExpression); - return queryable.Select(getGroupingKeyLambdaExpression).ToList().GroupBy(i => i.Key).ToDictionary(i => new GroupingValue(i.Key), i => i.Select(j => j.Row).ToList()); - } - // private Expression> - public virtual IQueryable Search(Dictionary> filters) - { - var queryable = this.GetQueryable(); - if (filters == null || filters.Count == 0) return queryable; - return queryable.Where(GetFilterExpression(filters)); - } - public virtual List SearchIds(Dictionary> filters) - => this.Search(filters).Select(this.GetIdExpression).ToList(); - public virtual Dictionary> SearchIds(Dictionary> filters, string pathToGroupProperty) - { - var getId = this.GetIdExpression.Compile(); - var defaultValue = this.DefaultValueProperty.GetValueExpression.Compile() as Func; - return this.Search(filters, pathToGroupProperty).ToDictionary(i => i.Key, i => i.Value.Select(j => + else if (objectDescriptor.Fields.TryGetValue(pathSegment, out var field)) { - var id = getId(j); - string label = getId(j).ToString(); - try { label = defaultValue(j)?.ToString(); } - catch { } - return (id, label); - }).ToList()); + return (GetExpressionBodyReferencingValue(field.GetValueExpression, valueExpression), field.ConvertFromString); + // return (Expression.Invoke(field.GetValueExpression, valueExpression), field.ConvertFromString); + } } - protected virtual IQueryable GetQueryable() => Include(this.DbContext.Set().AsQueryable()); + var defaultField = objectDescriptor.DefaultValueProperty; + return (GetExpressionBodyReferencingValue(defaultField.GetValueExpression, valueExpression), defaultField.ConvertFromString); + // return (Expression.Invoke(defaultField.GetValueExpression, valueExpression), defaultField.ConvertFromString); + } + private Expression GetExpressionBodyReferencingValue(LambdaExpression expression, Expression newParameterExpression) + { + var parameterToBeReplaced = expression.Parameters[0]; + var visitor = new ReplacementVisitor(parameterToBeReplaced, newParameterExpression); + return visitor.Visit(expression.Body); + } + private LambdaExpression GetValueLambdaExpression(string pathToProperty) + { + var entityParameterExpression = Expression.Parameter(typeof(TEntity), "entity"); + (var valueExpressionBody, var converter) = GetValueBodyExpression(entityParameterExpression, pathToProperty); + return Expression.Lambda(valueExpressionBody, entityParameterExpression); + } + private object CallMethod(string methodName, Type[] genericParameters, object[] args) + { + // var methods = this.GetType().GetMethods(BindingFlags.NonPublic | BindingFlags.Instance); + var method = typeof(SearchDescriptorBase<,>).MakeGenericType(typeof(TEntity), typeof(TId)).GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.FlattenHierarchy); + var groupQueryableMethodInfo = method.MakeGenericMethod(genericParameters); + return groupQueryableMethodInfo.Invoke(this, args); + } + public Dictionary> Search(Dictionary> filters, string pathToGroupProperty) + { + var queryable = Search(filters); + var groupingLambdaExpression = GetValueLambdaExpression(pathToGroupProperty); + var groupingType = groupingLambdaExpression.ReturnType; + return (Dictionary>)CallMethod(nameof(GroupQueryable), new[] { groupingLambdaExpression.ReturnType }, new object[] { queryable, groupingLambdaExpression }); + // var groupQueryableMethodInfo = this.GetType().GetMethod(nameof(GroupQueryable), BindingFlags.NonPublic | BindingFlags.Instance).MakeGenericMethod(groupingType); + // return (List>)groupQueryableMethodInfo.Invoke(this, new object[] { queryable, groupingLambdaExpression }); + } + private Dictionary> GroupQueryable(IQueryable queryable, Expression> groupingExpression) + { + var entityParameterExpression = Expression.Parameter(typeof(TEntity), "entity"); + var keyedRowType = typeof(KeyedRow<>).MakeGenericType(typeof(TEntity), typeof(TId), typeof(TKey)); + var constructorInfo = keyedRowType.GetConstructor(new Type[] { }); + var callConstructor = Expression.New(constructorInfo); + var initializedObject = Expression.MemberInit(callConstructor, + // Expression.Bind(keyedRowType.GetProperty(nameof(KeyedRow.Key)), Expression.Invoke(groupingExpression, entityParameterExpression)), + Expression.Bind(keyedRowType.GetProperty(nameof(KeyedRow.Key)), GetExpressionBodyReferencingValue(groupingExpression, entityParameterExpression)), + Expression.Bind(keyedRowType.GetProperty(nameof(KeyedRow.Row)), entityParameterExpression)); + var getGroupingKeyLambdaExpression = (Expression>>)Expression.Lambda(initializedObject, entityParameterExpression); + return queryable.Select(getGroupingKeyLambdaExpression).ToList().GroupBy(i => i.Key).ToDictionary(i => new GroupingValue(i.Key), i => i.Select(j => j.Row).ToList()); + } + // private Expression> + public virtual IQueryable Search(Dictionary> filters) + { + var queryable = this.GetQueryable(); + if (filters == null || filters.Count == 0) return queryable; + return queryable.Where(GetFilterExpression(filters)); + } + public virtual List SearchIds(Dictionary> filters) + => this.Search(filters).Select(this.GetIdExpression).ToList(); + public virtual Dictionary> SearchIds(Dictionary> filters, string pathToGroupProperty) + { + var getId = this.GetIdExpression.Compile(); + var defaultValue = this.DefaultValueProperty.GetValueExpression.Compile() as Func; + return this.Search(filters, pathToGroupProperty).ToDictionary(i => i.Key, i => i.Value.Select(j => + { + var id = getId(j); + string label = getId(j).ToString(); + try { label = defaultValue(j)?.ToString(); } + catch { } + return (id, label); + }).ToList()); + } + protected virtual IQueryable GetQueryable() => Include(this.DbContext.Set().AsQueryable()); - protected IQueryable Include(IQueryable query) - { - if (_include == null) - return query; - return _include(query); - } - private class KeyedRow - { - public TKey Key { get; set; } - public TEntity Row { get; set; } - } + protected IQueryable Include(IQueryable query) + { + if (_include == null) + return query; + return _include(query); + } + private class KeyedRow + { + public TKey Key { get; set; } + public TEntity Row { get; set; } } -} \ No newline at end of file +} diff --git a/src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchMetadata.cs b/src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchMetadata.cs index ef4471ed..588f7b89 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchMetadata.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/Searcher/SearchMetadata.cs @@ -1,22 +1,21 @@ using System.Collections.Generic; -namespace Paillave.EntityFrameworkCoreExtension.Searcher +namespace Paillave.EntityFrameworkCoreExtension.Searcher; + +public class SearchMetadata { - public class SearchMetadata + public SearchMetadata() { - public SearchMetadata() - { - - } - public SearchMetadata(string name, string type, List subLevels = null) - { - Type = type; - Name = name; - SubLevels = subLevels; - } - public string Name { get; set; } - public string Type { get; set; } - public List SubLevels { get; set; } } -} \ No newline at end of file + public SearchMetadata(string name, string type, List subLevels = null) + { + Type = type; + Name = name; + SubLevels = subLevels; + } + + public string Name { get; set; } + public string Type { get; set; } + public List SubLevels { get; set; } +} From 5ee451c7c54100a09f39663bf046fa36b69e8ce1 Mon Sep 17 00:00:00 2001 From: Stephane Royer Date: Mon, 2 Dec 2024 16:31:11 +0100 Subject: [PATCH 9/9] chore: update .NET Core version to 9.x in CI and Release workflows; update SharedSettings version to 2.2.0-beta --- .github/workflows/CI.yml | 2 +- .github/workflows/Release.yml | 2 +- src/SharedSettings.props | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5a0e13d6..c0e0851f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -14,7 +14,7 @@ jobs: - name: Setup .NET Core uses: actions/setup-dotnet@v1 with: - dotnet-version: '7.x' + dotnet-version: '9.x' - name: Install dependencies working-directory: src run: dotnet restore diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 6bfa5b3a..c77635b4 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -13,7 +13,7 @@ jobs: - name: Setup .NET Core uses: actions/setup-dotnet@v1 with: - dotnet-version: '7.x' + dotnet-version: '9.x' - name: Install dependencies working-directory: src run: dotnet restore diff --git a/src/SharedSettings.props b/src/SharedSettings.props index 66db9dac..41bc9c4a 100644 --- a/src/SharedSettings.props +++ b/src/SharedSettings.props @@ -2,7 +2,7 @@ latest enable - 2.1.38-beta + 2.2.0-beta NugetIcon.png README.md Stéphane Royer