Merge pull request #515 from paillave/v

chore: update version to 2.1.38-beta in SharedSettings.props; modify …
paillave · Nov 28, 2024 · ed9b8a7 · ed9b8a7
2 parents 725247b + d48c46e
commit ed9b8a7
Show file tree

Hide file tree

Showing 13 changed files with 169 additions and 27 deletions.
diff --git a/src/Paillave.EntityFrameworkCoreExtension/EfSave/EfSaveEngine.cs b/src/Paillave.EntityFrameworkCoreExtension/EfSave/EfSaveEngine.cs
@@ -94,7 +94,8 @@ private void InsertOrUpdateEntity(bool doNotUpdateIfExists, DbSet<T> contextSet,
         var existingEntity = contextSet.AsNoTracking().FirstOrDefault(entityCondition);
         if (existingEntity == null)
         {
-            _context.Entry(entity).State = EntityState.Added;
+            // _context.Entry(entity).State = EntityState.Added;
+            contextSet.Update(entity);
         }
         else
         {

diff --git a/src/Paillave.Etl.Bloomberg/BloombergValuesProvider.cs b/src/Paillave.Etl.Bloomberg/BloombergValuesProvider.cs
@@ -109,7 +109,7 @@ public override void PushValues(IFileValue input, Action<BloombergResult<TParsed
                                     var splitted = lineSerializer.Splitter.Split(line);
                                     ticker = splitted[0];
                                     line = string.Join("|", splitted.Skip(3));
-                                    parsed = lineSerializer.Deserialize(line, sourceName, index);
+                                    parsed = lineSerializer.Deserialize(line, sourceName, index, null);
                                 }
                                 catch (Exception ex)
                                 {

diff --git a/src/Paillave.Etl.TextFile/FlatFileDefinition.cs b/src/Paillave.Etl.TextFile/FlatFileDefinition.cs
@@ -24,7 +24,8 @@ public class FlatFileDefinition<T>
         private bool _respectHeaderCase = false;
 
         public int FirstLinesToIgnore { get; private set; }
-        public Func<string,string>? LinePreProcessor { get; set; }
+        public Func<string, string>? LinePreProcessor { get; set; }
+        public Dictionary<string, Func<string, string>>? ValuePreProcessor = null;
 
         private IEnumerable<string> GetDefaultColumnNames()
         {
@@ -53,11 +54,18 @@ public FlatFileDefinition<T> IgnoreFirstLines(int firstLinesToIgnore)
             FirstLinesToIgnore = firstLinesToIgnore;
             return this;
         }
-        public FlatFileDefinition<T>  WithLinePreProcessor(Func<string,string> linePreProcessor)
+        public FlatFileDefinition<T> WithLinePreProcessor(Func<string, string> linePreProcessor)
         {
             LinePreProcessor = linePreProcessor;
             return this;
         }
+        public FlatFileDefinition<T> WithValuePreProcessor(string propertyName, Func<string, string> valuePreProcessor)
+        {
+            if (ValuePreProcessor == null)
+                ValuePreProcessor = new Dictionary<string, Func<string, string>>();
+            ValuePreProcessor[propertyName] = valuePreProcessor;
+            return this;
+        }
         public FlatFileDefinition<T> WithMap(Expression<Func<IFieldMapper, T>> expression)
         {
             MapperVisitor vis = new MapperVisitor();

diff --git a/src/Paillave.Etl.TextFile/FlatFileValuesProvider.cs b/src/Paillave.Etl.TextFile/FlatFileValuesProvider.cs
@@ -46,7 +46,7 @@ public override void PushValues(IFileValue input, Action<TOut> push, Cancellatio
                         TParsed parsed;
                         try
                         {
-                            parsed = lineSerializer.Deserialize(line, sourceName, index);
+                            parsed = lineSerializer.Deserialize(line, sourceName, index, _args.Mapping.ValuePreProcessor);
                         }
                         catch (Exception ex)
                         {
@@ -71,7 +71,7 @@ public override void PushValues(IFileValue input, Action<TOut> push, Cancellatio
                         TParsed parsed;
                         try
                         {
-                            parsed = lineSerializer.Deserialize(line, sourceName, index);
+                            parsed = lineSerializer.Deserialize(line, sourceName, index, _args.Mapping.ValuePreProcessor);
                         }
                         catch (Exception ex)
                         {

diff --git a/src/Paillave.Etl.TextFile/LineSerializer.cs b/src/Paillave.Etl.TextFile/LineSerializer.cs
@@ -23,7 +23,7 @@ public LineSerializer(ILineSplitter splitter, IDictionary<int, FlatFilePropertyS
             this._rowNumberPropertyNames = rowNumberPropertyNames;
             this._rowGuidPropertyNames = rowGuidPropertyNames;
         }
-        public T Deserialize(string line, string sourceName, int rowNumber)
+        public T Deserialize(string line, string sourceName, int rowNumber, Dictionary<string, Func<string, string>>? valuePreProcessor)
         {
             var stringValues = this.Splitter.Split(line);
             var values = this._indexToPropertySerializerDictionary.ToDictionary(i => i.Value.PropertyName, i =>
@@ -40,8 +40,10 @@ public T Deserialize(string line, string sourceName, int rowNumber)
                 // if(string.IsNullOrWhiteSpace(valueToParse) && Nullable.)
                 try
                 {
+                    if (valuePreProcessor != null && valuePreProcessor.TryGetValue(i.Value.PropertyName, out var preProcessor))
+                        valueToParse = preProcessor(valueToParse);
                     return i.Value.Deserialize(valueToParse);
-                } 
+                }
                 catch (Exception ex)
                 {
                     throw new FlatFileFieldDeserializeException(i.Key, i.Value.PropertyName, valueToParse, ex);

diff --git a/src/SharedSettings.props b/src/SharedSettings.props
@@ -2,7 +2,7 @@
   <PropertyGroup>
     <LangVersion>latest</LangVersion>
     <Nullable>enable</Nullable>
-    <Version>2.1.37-beta</Version>
+    <Version>2.1.38-beta</Version>
     <PackageIcon>NugetIcon.png</PackageIcon>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <Authors>Stéphane Royer</Authors>

diff --git a/src/Tutorials/Paillave.Etl.Samples/DataAccess/SimpleTable.cs b/src/Tutorials/Paillave.Etl.Samples/DataAccess/SimpleTable.cs
@@ -2,20 +2,37 @@
 using Microsoft.EntityFrameworkCore.Metadata.Builders;
 using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
 
-namespace Paillave.Etl.Samples.DataAccess
+namespace Paillave.Etl.Samples.DataAccess;
+public class SimpleTable
 {
-    public class SimpleTable
+    public int Id { get; set; }
+    public string Name { get; set; }
+    public List<SimpleTableRelated> Relateds { get; set; }
+}
+public class SimpleTableConfiguration : IEntityTypeConfiguration<SimpleTable>
+{
+    public void Configure(EntityTypeBuilder<SimpleTable> builder)
     {
-        public int Id { get; set; }
-        public string Name { get; set; }
+        builder.ToTable(nameof(SimpleTable));
+        builder.HasKey(i => i.Id);
+        builder.Property(i => i.Id).UseIdentityColumn();
+        builder.HasMany(i => i.Relateds).WithOne().OnDelete(DeleteBehavior.Cascade).HasForeignKey(i => i.SimpleTableId);
     }
-    public class SimpleTableConfiguration : IEntityTypeConfiguration<SimpleTable>
+}
+
+
+public class SimpleTableRelated
+{
+    public int Id { get; set; }
+    public string Name { get; set; }
+    public int SimpleTableId { get; set; }
+}
+public class SimpleTableRelatedConfiguration : IEntityTypeConfiguration<SimpleTableRelated>
+{
+    public void Configure(EntityTypeBuilder<SimpleTableRelated> builder)
     {
-        public void Configure(EntityTypeBuilder<SimpleTable> builder)
-        {
-            builder.ToTable(nameof(SimpleTable));
-            builder.HasKey(i => i.Id);
-            builder.Property(i => i.Id).UseIdentityColumn();
-        }
+        builder.ToTable(nameof(SimpleTableRelated));
+        builder.HasKey(i => i.Id);
+        builder.Property(i => i.Id).UseIdentityColumn();
     }
 }
diff --git a/src/Tutorials/Paillave.Etl.Samples/InputFiles/Input01.Portfolios.csv b/src/Tutorials/Paillave.Etl.Samples/InputFiles/Input01.Portfolios.csv
@@ -1,4 +1,4 @@
 SicavCode,SicavName,SicavType,PortfolioCode,PortfolioName
-S1,Sicav1AA,UCITS,P1,Portfolio1
-S2,Sicav2AA,AIFM,P2,Portfolio2
-S1,Sicav1AA,UCITS,P3,Portfolio3
+S1,Sicav1,UCITS,P1,Portfolio1
+S2,Sicav2,AIFM,P2,Portfolio2
+S1,Sicav1,UCITS,P3,Portfolio3
diff --git a/...ations/20241127170031_Initial.Designer.cs → ...ations/20241128161257_Initial.Designer.cs b/...ations/20241127170031_Initial.Designer.cs → ...ations/20241128161257_Initial.Designer.cs
diff --git a/...ples/Migrations/20241127170031_Initial.cs → ...ples/Migrations/20241128161257_Initial.cs b/...ples/Migrations/20241127170031_Initial.cs → ...ples/Migrations/20241128161257_Initial.cs
@@ -81,6 +81,26 @@ protected override void Up(MigrationBuilder migrationBuilder)
                         onDelete: ReferentialAction.Cascade);
                 });
 
+            migrationBuilder.CreateTable(
+                name: "SimpleTableRelated",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "int", nullable: false)
+                        .Annotation("SqlServer:Identity", "1, 1"),
+                    Name = table.Column<string>(type: "nvarchar(max)", nullable: false),
+                    SimpleTableId = table.Column<int>(type: "int", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_SimpleTableRelated", x => x.Id);
+                    table.ForeignKey(
+                        name: "FK_SimpleTableRelated_SimpleTable_SimpleTableId",
+                        column: x => x.SimpleTableId,
+                        principalTable: "SimpleTable",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
             migrationBuilder.CreateTable(
                 name: "Composition",
                 columns: table => new
@@ -141,6 +161,11 @@ protected override void Up(MigrationBuilder migrationBuilder)
                 name: "IX_Position_SecurityId",
                 table: "Position",
                 column: "SecurityId");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_SimpleTableRelated_SimpleTableId",
+                table: "SimpleTableRelated",
+                column: "SimpleTableId");
         }
 
         /// <inheritdoc />
@@ -150,14 +175,17 @@ protected override void Down(MigrationBuilder migrationBuilder)
                 name: "Position");
 
             migrationBuilder.DropTable(
-                name: "SimpleTable");
+                name: "SimpleTableRelated");
 
             migrationBuilder.DropTable(
                 name: "Composition");
 
             migrationBuilder.DropTable(
                 name: "Security");
 
+            migrationBuilder.DropTable(
+                name: "SimpleTable");
+
             migrationBuilder.DropTable(
                 name: "Portfolio");
 

diff --git a/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs b/src/Tutorials/Paillave.Etl.Samples/Migrations/TestDbContextModelSnapshot.cs
@@ -169,6 +169,28 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.ToTable("SimpleTable", (string)null);
                 });
 
+            modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTableRelated", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("int");
+
+                    SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<int>("SimpleTableId")
+                        .HasColumnType("int");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("SimpleTableId");
+
+                    b.ToTable("SimpleTableRelated", (string)null);
+                });
+
             modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Equity", b =>
                 {
                     b.HasBaseType("Paillave.Etl.Samples.DataAccess.Security");
@@ -232,10 +254,24 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.Navigation("Security");
                 });
 
+            modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTableRelated", b =>
+                {
+                    b.HasOne("Paillave.Etl.Samples.DataAccess.SimpleTable", null)
+                        .WithMany("Relateds")
+                        .HasForeignKey("SimpleTableId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+                });
+
             modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.Composition", b =>
                 {
                     b.Navigation("Positions");
                 });
+
+            modelBuilder.Entity("Paillave.Etl.Samples.DataAccess.SimpleTable", b =>
+                {
+                    b.Navigation("Relateds");
+                });
 #pragma warning restore 612, 618
         }
     }

diff --git a/src/Tutorials/Paillave.Etl.Samples/Program2.cs b/src/Tutorials/Paillave.Etl.Samples/Program2.cs
@@ -55,7 +55,7 @@ static async Task SimplyImportAsync(string[] args)
         /// <returns></returns>
         static async Task ImportAndCreateFileAsync(string[] args)
         {
-            var processRunner = StreamProcessRunner.Create<string[]>(TestImport2.Import);
+            var processRunner = StreamProcessRunner.Create<string[]>(TestImport3.Import);
             var structure = processRunner.GetDefinitionStructure();
             // structure.OpenEstimatedExecutionPlan();
 

diff --git a/src/Tutorials/Paillave.Etl.Samples/TestImport3.cs b/src/Tutorials/Paillave.Etl.Samples/TestImport3.cs
@@ -11,9 +11,23 @@ public static void Import(ISingleStream<string[]> contextStream)
         var portfolioFileStream = contextStream
             .FromConnector("Get portfolio files", "PTF")
             .CrossApplyTextFile("Parse portfolio file", o => o
-                .UseMap(i => new DataAccess.SimpleTable { Name = i.ToColumn("SicavName") }).IsColumnSeparated(','));
+                .UseMap(i => new DataAccess.SimpleTable
+                {
+                    Name = i.ToColumn("SicavName")
+                }).IsColumnSeparated(','));
         var sicavStream = portfolioFileStream
             .Distinct("Distinct sicav", i => i.Name, true)
+            .Select("Create sicav", i => new DataAccess.SimpleTable
+            {
+                Name = i.Name,
+                Relateds = new List<DataAccess.SimpleTableRelated>
+                {
+                    new DataAccess.SimpleTableRelated
+                    {
+                        Name = "Related"
+                    }
+                }
+            })
             .EfCoreSave("Save sicav", o => o.WithMode(SaveMode.EntityFrameworkCore));
     }
 }