diff --git a/src/Paillave.EntityFrameworkCoreExtension/EfSave/EfSaveEngine.cs b/src/Paillave.EntityFrameworkCoreExtension/EfSave/EfSaveEngine.cs index 1bf91e58..c368fd74 100644 --- a/src/Paillave.EntityFrameworkCoreExtension/EfSave/EfSaveEngine.cs +++ b/src/Paillave.EntityFrameworkCoreExtension/EfSave/EfSaveEngine.cs @@ -94,7 +94,8 @@ private void InsertOrUpdateEntity(bool doNotUpdateIfExists, DbSet contextSet, var existingEntity = contextSet.AsNoTracking().FirstOrDefault(entityCondition); if (existingEntity == null) { - _context.Entry(entity).State = EntityState.Added; + // _context.Entry(entity).State = EntityState.Added; + contextSet.Update(entity); } else { diff --git a/src/Paillave.Etl.Bloomberg/BloombergValuesProvider.cs b/src/Paillave.Etl.Bloomberg/BloombergValuesProvider.cs index 23673637..635b9d55 100644 --- a/src/Paillave.Etl.Bloomberg/BloombergValuesProvider.cs +++ b/src/Paillave.Etl.Bloomberg/BloombergValuesProvider.cs @@ -109,7 +109,7 @@ public override void PushValues(IFileValue input, Action private bool _respectHeaderCase = false; public int FirstLinesToIgnore { get; private set; } - public Func? LinePreProcessor { get; set; } + public Func? LinePreProcessor { get; set; } + public Dictionary>? ValuePreProcessor = null; private IEnumerable GetDefaultColumnNames() { @@ -53,11 +54,18 @@ public FlatFileDefinition IgnoreFirstLines(int firstLinesToIgnore) FirstLinesToIgnore = firstLinesToIgnore; return this; } - public FlatFileDefinition WithLinePreProcessor(Func linePreProcessor) + public FlatFileDefinition WithLinePreProcessor(Func linePreProcessor) { LinePreProcessor = linePreProcessor; return this; } + public FlatFileDefinition WithValuePreProcessor(string propertyName, Func valuePreProcessor) + { + if (ValuePreProcessor == null) + ValuePreProcessor = new Dictionary>(); + ValuePreProcessor[propertyName] = valuePreProcessor; + return this; + } public FlatFileDefinition WithMap(Expression> expression) { MapperVisitor vis = new MapperVisitor(); diff --git a/src/Paillave.Etl.TextFile/FlatFileValuesProvider.cs b/src/Paillave.Etl.TextFile/FlatFileValuesProvider.cs index b45077bb..33178866 100644 --- a/src/Paillave.Etl.TextFile/FlatFileValuesProvider.cs +++ b/src/Paillave.Etl.TextFile/FlatFileValuesProvider.cs @@ -46,7 +46,7 @@ public override void PushValues(IFileValue input, Action push, Cancellatio TParsed parsed; try { - parsed = lineSerializer.Deserialize(line, sourceName, index); + parsed = lineSerializer.Deserialize(line, sourceName, index, _args.Mapping.ValuePreProcessor); } catch (Exception ex) { @@ -71,7 +71,7 @@ public override void PushValues(IFileValue input, Action push, Cancellatio TParsed parsed; try { - parsed = lineSerializer.Deserialize(line, sourceName, index); + parsed = lineSerializer.Deserialize(line, sourceName, index, _args.Mapping.ValuePreProcessor); } catch (Exception ex) { diff --git a/src/Paillave.Etl.TextFile/LineSerializer.cs b/src/Paillave.Etl.TextFile/LineSerializer.cs index 168c6e3b..3cf3235e 100644 --- a/src/Paillave.Etl.TextFile/LineSerializer.cs +++ b/src/Paillave.Etl.TextFile/LineSerializer.cs @@ -23,7 +23,7 @@ public LineSerializer(ILineSplitter splitter, IDictionary>? valuePreProcessor) { var stringValues = this.Splitter.Split(line); var values = this._indexToPropertySerializerDictionary.ToDictionary(i => i.Value.PropertyName, i => @@ -40,8 +40,10 @@ public T Deserialize(string line, string sourceName, int rowNumber) // if(string.IsNullOrWhiteSpace(valueToParse) && Nullable.) try { + if (valuePreProcessor != null && valuePreProcessor.TryGetValue(i.Value.PropertyName, out var preProcessor)) + valueToParse = preProcessor(valueToParse); return i.Value.Deserialize(valueToParse); - } + } catch (Exception ex) { throw new FlatFileFieldDeserializeException(i.Key, i.Value.PropertyName, valueToParse, ex); diff --git a/src/SharedSettings.props b/src/SharedSettings.props index 807fe6a4..66db9dac 100644 --- a/src/SharedSettings.props +++ b/src/SharedSettings.props @@ -2,7 +2,7 @@ latest enable - 2.1.37-beta + 2.1.38-beta NugetIcon.png README.md Stéphane Royer diff --git a/src/Tutorials/Paillave.Etl.Samples/DataAccess/SimpleTable.cs b/src/Tutorials/Paillave.Etl.Samples/DataAccess/SimpleTable.cs index a4851974..caabd69c 100644 --- a/src/Tutorials/Paillave.Etl.Samples/DataAccess/SimpleTable.cs +++ b/src/Tutorials/Paillave.Etl.Samples/DataAccess/SimpleTable.cs @@ -2,20 +2,37 @@ using Microsoft.EntityFrameworkCore.Metadata.Builders; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -namespace Paillave.Etl.Samples.DataAccess +namespace Paillave.Etl.Samples.DataAccess; +public class SimpleTable { - public class SimpleTable + public int Id { get; set; } + public string Name { get; set; } + public List Relateds { get; set; } +} +public class SimpleTableConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) { - public int Id { get; set; } - public string Name { get; set; } + builder.ToTable(nameof(SimpleTable)); + builder.HasKey(i => i.Id); + builder.Property(i => i.Id).UseIdentityColumn(); + builder.HasMany(i => i.Relateds).WithOne().OnDelete(DeleteBehavior.Cascade).HasForeignKey(i => i.SimpleTableId); } - public class SimpleTableConfiguration : IEntityTypeConfiguration +} + + +public class SimpleTableRelated +{ + public int Id { get; set; } + public string Name { get; set; } + public int SimpleTableId { get; set; } +} +public class SimpleTableRelatedConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) { - public void Configure(EntityTypeBuilder builder) - { - builder.ToTable(nameof(SimpleTable)); - builder.HasKey(i => i.Id); - builder.Property(i => i.Id).UseIdentityColumn(); - } + builder.ToTable(nameof(SimpleTableRelated)); + builder.HasKey(i => i.Id); + builder.Property(i => i.Id).UseIdentityColumn(); } } \ No newline at end of file diff --git a/src/Tutorials/Paillave.Etl.Samples/InputFiles/Input01.Portfolios.csv b/src/Tutorials/Paillave.Etl.Samples/InputFiles/Input01.Portfolios.csv index 3b1b0e5b..dd7d72a7 100644 --- a/src/Tutorials/Paillave.Etl.Samples/InputFiles/Input01.Portfolios.csv +++ b/src/Tutorials/Paillave.Etl.Samples/InputFiles/Input01.Portfolios.csv @@ -1,4 +1,4 @@ SicavCode,SicavName,SicavType,PortfolioCode,PortfolioName -S1,Sicav1AA,UCITS,P1,Portfolio1 -S2,Sicav2AA,AIFM,P2,Portfolio2 -S1,Sicav1AA,UCITS,P3,Portfolio3 \ No newline at end of file +S1,Sicav1,UCITS,P1,Portfolio1 +S2,Sicav2,AIFM,P2,Portfolio2 +S1,Sicav1,UCITS,P3,Portfolio3 \ No newline at end of file diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241127170031_Initial.Designer.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.Designer.cs similarity index 86% rename from src/Tutorials/Paillave.Etl.Samples/Migrations/20241127170031_Initial.Designer.cs rename to src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.Designer.cs index f20e8354..305a0c7f 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241127170031_Initial.Designer.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.Designer.cs @@ -12,7 +12,7 @@ namespace Paillave.Etl.Samples.Migrations { [DbContext(typeof(TestDbContext))] - [Migration("20241127170031_Initial")] + [Migration("20241128161257_Initial")] partial class Initial { /// @@ -172,6 +172,28 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.ToTable("SimpleTable", (string)null); }); + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTableRelated", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SimpleTableId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SimpleTableId"); + + b.ToTable("SimpleTableRelated", (string)null); + }); + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Equity", b => { b.HasBaseType("Paillave.Etl.Samples.DataAccess.Security"); @@ -235,10 +257,24 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Navigation("Security"); }); + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTableRelated", b => + { + b.HasOne("Paillave.Etl.Samples.DataAccess.SimpleTable", null) + .WithMany("Relateds") + .HasForeignKey("SimpleTableId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Composition", b => { b.Navigation("Positions"); }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTable", b => + { + b.Navigation("Relateds"); + }); #pragma warning restore 612, 618 } } diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241127170031_Initial.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.cs similarity index 85% rename from src/Tutorials/Paillave.Etl.Samples/Migrations/20241127170031_Initial.cs rename to src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.cs index c2076c5e..be6da18f 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Migrations/20241127170031_Initial.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Migrations/20241128161257_Initial.cs @@ -81,6 +81,26 @@ protected override void Up(MigrationBuilder migrationBuilder) onDelete: ReferentialAction.Cascade); }); + migrationBuilder.CreateTable( + name: "SimpleTableRelated", + columns: table => new + { + Id = table.Column(type: "int", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + Name = table.Column(type: "nvarchar(max)", nullable: false), + SimpleTableId = table.Column(type: "int", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_SimpleTableRelated", x => x.Id); + table.ForeignKey( + name: "FK_SimpleTableRelated_SimpleTable_SimpleTableId", + column: x => x.SimpleTableId, + principalTable: "SimpleTable", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + migrationBuilder.CreateTable( name: "Composition", columns: table => new @@ -141,6 +161,11 @@ protected override void Up(MigrationBuilder migrationBuilder) name: "IX_Position_SecurityId", table: "Position", column: "SecurityId"); + + migrationBuilder.CreateIndex( + name: "IX_SimpleTableRelated_SimpleTableId", + table: "SimpleTableRelated", + column: "SimpleTableId"); } /// @@ -150,7 +175,7 @@ protected override void Down(MigrationBuilder migrationBuilder) name: "Position"); migrationBuilder.DropTable( - name: "SimpleTable"); + name: "SimpleTableRelated"); migrationBuilder.DropTable( name: "Composition"); @@ -158,6 +183,9 @@ protected override void Down(MigrationBuilder migrationBuilder) migrationBuilder.DropTable( name: "Security"); + migrationBuilder.DropTable( + name: "SimpleTable"); + migrationBuilder.DropTable( name: "Portfolio"); diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs index 623d85b7..afd880b5 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs @@ -169,6 +169,28 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("SimpleTable", (string)null); }); + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTableRelated", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SimpleTableId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SimpleTableId"); + + b.ToTable("SimpleTableRelated", (string)null); + }); + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Equity", b => { b.HasBaseType("Paillave.Etl.Samples.DataAccess.Security"); @@ -232,10 +254,24 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Navigation("Security"); }); + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTableRelated", b => + { + b.HasOne("Paillave.Etl.Samples.DataAccess.SimpleTable", null) + .WithMany("Relateds") + .HasForeignKey("SimpleTableId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Composition", b => { b.Navigation("Positions"); }); + + modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTable", b => + { + b.Navigation("Relateds"); + }); #pragma warning restore 612, 618 } } diff --git a/src/Tutorials/Paillave.Etl.Samples/Program2.cs b/src/Tutorials/Paillave.Etl.Samples/Program2.cs index 6385685b..5ebecd0f 100644 --- a/src/Tutorials/Paillave.Etl.Samples/Program2.cs +++ b/src/Tutorials/Paillave.Etl.Samples/Program2.cs @@ -55,7 +55,7 @@ static async Task SimplyImportAsync(string[] args) /// static async Task ImportAndCreateFileAsync(string[] args) { - var processRunner = StreamProcessRunner.Create(TestImport2.Import); + var processRunner = StreamProcessRunner.Create(TestImport3.Import); var structure = processRunner.GetDefinitionStructure(); // structure.OpenEstimatedExecutionPlan(); diff --git a/src/Tutorials/Paillave.Etl.Samples/TestImport3.cs b/src/Tutorials/Paillave.Etl.Samples/TestImport3.cs index 73f9ad80..01c8c477 100644 --- a/src/Tutorials/Paillave.Etl.Samples/TestImport3.cs +++ b/src/Tutorials/Paillave.Etl.Samples/TestImport3.cs @@ -11,9 +11,23 @@ public static void Import(ISingleStream contextStream) var portfolioFileStream = contextStream .FromConnector("Get portfolio files", "PTF") .CrossApplyTextFile("Parse portfolio file", o => o - .UseMap(i => new DataAccess.SimpleTable { Name = i.ToColumn("SicavName") }).IsColumnSeparated(',')); + .UseMap(i => new DataAccess.SimpleTable + { + Name = i.ToColumn("SicavName") + }).IsColumnSeparated(',')); var sicavStream = portfolioFileStream .Distinct("Distinct sicav", i => i.Name, true) + .Select("Create sicav", i => new DataAccess.SimpleTable + { + Name = i.Name, + Relateds = new List + { + new DataAccess.SimpleTableRelated + { + Name = "Related" + } + } + }) .EfCoreSave("Save sicav", o => o.WithMode(SaveMode.EntityFrameworkCore)); } }