diff --git a/FreeSql.Tests/FreeSql.Tests/ClickHouse/ClickHouseTest3.cs b/FreeSql.Tests/FreeSql.Tests/ClickHouse/ClickHouseTest3.cs index b3fbd72db..3ae9c22c4 100644 --- a/FreeSql.Tests/FreeSql.Tests/ClickHouse/ClickHouseTest3.cs +++ b/FreeSql.Tests/FreeSql.Tests/ClickHouse/ClickHouseTest3.cs @@ -282,7 +282,7 @@ public async Task UriStringIsTooLongTest() //单个插入报错 await _fsql.Insert(t).ExecuteAffrowsAsync(); - // await _fsql.Insert(t).ExecuteBulkCopyAsync(); + // await _fsql.Insert(t).ExecuteQuestBulkCopyAsync(); } @@ -304,9 +304,9 @@ public async Task TestBulkCopySingle() //单个插入报错 await _fsql.Insert(t).ExecuteAffrowsAsync(); - await _fsql.Insert(t).ExecuteBulkCopyAsync(); + await _fsql.Insert(t).ExecuteClickHouseBulkCopyAsync(); - _fsql.Insert(t).ExecuteBulkCopy(); + _fsql.Insert(t).ExecuteClickHouseBulkCopy(); } /// @@ -333,9 +333,9 @@ public async Task TestBulkCopyMany() await _fsql.Insert(t).ExecuteAffrowsAsync(); //BulkCopy不会报错 - await _fsql.Insert(t).ExecuteBulkCopyAsync(); + await _fsql.Insert(t).ExecuteClickHouseBulkCopyAsync(); - _fsql.Insert(t).ExecuteBulkCopy(); + _fsql.Insert(t).ExecuteClickHouseBulkCopy(); } } diff --git a/FreeSql.Tests/FreeSql.Tests/QuestDb/Crud/QuestDbTestInsertAndUpdate.cs b/FreeSql.Tests/FreeSql.Tests/QuestDb/Crud/QuestDbTestInsertAndUpdate.cs index cde036015..15cebad3e 100644 --- a/FreeSql.Tests/FreeSql.Tests/QuestDb/Crud/QuestDbTestInsertAndUpdate.cs +++ b/FreeSql.Tests/FreeSql.Tests/QuestDb/Crud/QuestDbTestInsertAndUpdate.cs @@ -223,7 +223,8 @@ public async Task TestSqlBulkCopy() NameUpdate = "NameUpdate" }); } - var result = await restFsql.Insert(list).ExecuteBulkCopyAsync(); + + var result = await restFsql.Insert(list).ExecuteQuestDbBulkCopyAsync(); Assert.True(result > 0); } diff --git a/FreeSql.Tests/FreeSql.Tests/QuestDb/QuestDbIssue/QuestDbIssue.cs b/FreeSql.Tests/FreeSql.Tests/QuestDb/QuestDbIssue/QuestDbIssue.cs new file mode 100644 index 000000000..68a0ba0aa --- /dev/null +++ b/FreeSql.Tests/FreeSql.Tests/QuestDb/QuestDbIssue/QuestDbIssue.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Xunit; + +namespace FreeSql.Tests.QuestDb.QuestDbIssue +{ + public class QuestDbIssue : QuestDbTest + { + [Fact] + public void Issue1757() + { + restFsql.CodeFirst.SyncStructure(); + var count = fsql.Insert(new List() + { + new() + { + CreateTime = DateTime.Now, + CustomId = 3, Name = "test333", + Price = 3, + Value = 3 + } + }).ExecuteQuestDbBulkCopy(); + + Assert.True(count > 0); + + var list = fsql.Select().ToList(); + } + + + [Fact] + public void Issue1757Many() + { + restFsql.CodeFirst.SyncStructure(); + var count = fsql.Insert(new List() + { + new() + { + CreateTime = DateTime.Now, + CustomId = 4, Name = "test444", + Price = 4, + Value = 4 + }, + new() + { + CreateTime = DateTime.Now, + CustomId = 5, Name = "test555", + Price = 5, + Value = 5 + }, + new() + { + CreateTime = DateTime.Now, + CustomId = 6, Name = "test666", + Price = 6, + Value = 6 + } + }).ExecuteQuestDbBulkCopy(); + + Assert.True(count > 0); + + var list = fsql.Select().ToList(); + } + } + + public class Test0111 + { + public long Id { get; set; } + + public string Name { get; set; } + public decimal Price { get; set; } + public DateTime CreateTime { get; set; } + public long CustomId { get; set; } + + public double Value { get; set; } + } +} \ No newline at end of file diff --git a/FreeSql.Tests/FreeSql.Tests/QuestDb/QuestDbTest.cs b/FreeSql.Tests/FreeSql.Tests/QuestDb/QuestDbTest.cs index 69072f412..a1fb83c80 100644 --- a/FreeSql.Tests/FreeSql.Tests/QuestDb/QuestDbTest.cs +++ b/FreeSql.Tests/FreeSql.Tests/QuestDb/QuestDbTest.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Xunit; namespace FreeSql.Tests.QuestDb { @@ -11,16 +12,19 @@ public class QuestDbTest { public static IFreeSql fsql = new FreeSql.FreeSqlBuilder() .UseConnectionString(FreeSql.DataType.QuestDb, - @"host=192.168.0.36;port=8812;username=admin;password=quest;database=qdb;ServerCompatibilityMode=NoTypeLoading;") + @"host=192.168.1.114;port=8812;username=admin;password=quest;database=qdb;ServerCompatibilityMode=NoTypeLoading;") .UseMonitorCommand(cmd => Debug.WriteLine($"Sql：{cmd.CommandText}")) //监听SQL语句 .UseNoneCommandParameter(true) .Build(); public static IFreeSql restFsql = new FreeSql.FreeSqlBuilder() .UseConnectionString(FreeSql.DataType.QuestDb, - @"host=192.168.0.36;port=8812;username=admin;password=quest;database=qdb;ServerCompatibilityMode=NoTypeLoading;") + @"host=192.168.1.114;port=8812;username=admin;password=quest;database=qdb;ServerCompatibilityMode=NoTypeLoading;") .UseMonitorCommand(cmd => Debug.WriteLine($"Sql：{cmd.CommandText}")) //监听SQL语句 - .UseQuestDbRestAPI("192.168.0.36:9001", "admin", "ushahL(aer2r") + .UseQuestDbRestAPI("192.168.1.114:9000") .Build(); + } + + } \ No newline at end of file diff --git a/Providers/FreeSql.Provider.ClickHouse/ClickHouseExtensions.cs b/Providers/FreeSql.Provider.ClickHouse/ClickHouseExtensions.cs index 36428c072..2e4d0a7fc 100644 --- a/Providers/FreeSql.Provider.ClickHouse/ClickHouseExtensions.cs +++ b/Providers/FreeSql.Provider.ClickHouse/ClickHouseExtensions.cs @@ -66,7 +66,7 @@ public static ISelect Sample(this ISelect that, decimal k, int n, decim /// /// /// - public static async Task ExecuteBulkCopyAsync(this IInsert that) where T : class + public static async Task ExecuteClickHouseBulkCopyAsync(this IInsert that) where T : class { try { @@ -87,8 +87,8 @@ public static async Task ExecuteBulkCopyAsync(this IInsert that) wher /// /// /// - public static int ExecuteBulkCopy(this IInsert insert) where T : class + public static int ExecuteClickHouseBulkCopy(this IInsert insert) where T : class { - return ExecuteBulkCopyAsync(insert).ConfigureAwait(false).GetAwaiter().GetResult(); + return ExecuteClickHouseBulkCopyAsync(insert).ConfigureAwait(false).GetAwaiter().GetResult(); } } diff --git a/Providers/FreeSql.Provider.QuestDb/QuestDbGlobalExtensions.cs b/Providers/FreeSql.Provider.QuestDb/QuestDbGlobalExtensions.cs index d47197c9a..a047356a0 100644 --- a/Providers/FreeSql.Provider.QuestDb/QuestDbGlobalExtensions.cs +++ b/Providers/FreeSql.Provider.QuestDb/QuestDbGlobalExtensions.cs @@ -31,12 +31,12 @@ public static partial class QuestDbGlobalExtensions /// /// public static string FormatQuestDb(this string that, params object[] args) => - _QuestDbAdo.Addslashes(that, args); + _questDbAdo.Addslashes(that, args); - static QuestDbAdo _QuestDbAdo = new QuestDbAdo(); + private static readonly QuestDbAdo _questDbAdo = new QuestDbAdo(); - public static FreeSqlBuilder UseQuestDbRestAPI(this FreeSqlBuilder buider, string host, string username = "", - string password = "") => RestAPIExtension.UseQuestDbRestAPI(buider, host, username, password); + public static FreeSqlBuilder UseQuestDbRestAPI(this FreeSqlBuilder build, string host, string username = "", + string password = "") => RestAPIExtension.UseQuestDbRestAPI(build, host, username, password); /// /// 对于多个时间序列存储在同一个表中的场景，根据时间戳检索给定键或键组合的最新项。 @@ -53,6 +53,7 @@ public static ISelect LatestOn(this ISelect select, Expression LatestOnExtension.InternelImpl(timestamp, partition); return select; } + /// /// 对于多个时间序列存储在同一个表中的场景，根据时间戳检索给定键或键组合的最新项。 /// @@ -113,13 +114,15 @@ public static ISelect LatestOn(this ISelec /// 单位 /// 对准日历 /// - public static ISelect SampleBy(this ISelect select, double time, SampleUnit unit, bool alignToCalendar = false) + public static ISelect SampleBy(this ISelect select, double time, SampleUnit unit, + bool alignToCalendar = false) { SampleByExtension.IsExistence.Value = true; var samoleByTemple = $"{Environment.NewLine}SAMPLE BY {{0}}{{1}} {{2}}"; string alignToCalendarTemple = ""; if (alignToCalendar) alignToCalendarTemple = "ALIGN TO CALENDAR "; - SampleByExtension.SamoleByString.Value = string.Format(samoleByTemple, time.ToString(), (char)unit, alignToCalendarTemple); + SampleByExtension.SamoleByString.Value = + string.Format(samoleByTemple, time.ToString(), (char)unit, alignToCalendarTemple); return select; } @@ -156,23 +159,25 @@ private static List SplitByLine(string text) /// /// 导入时，时间格式 默认:yyyy/M/d H:mm:ss /// - public static async Task ExecuteBulkCopyAsync(this IInsert that,string dateFormat = "yyyy/M/d H:mm:ss") where T : class + public static async Task ExecuteQuestDbBulkCopyAsync(this IInsert that, + string dateFormat = "yyyy/M/d H:mm:ss") where T : class { //思路：通过提供的RestAPI imp，实现快速复制 if (string.IsNullOrWhiteSpace(RestAPIExtension.BaseUrl)) { - throw new Exception("BulkCopy功能需要启用RestAPI，启用方式：new FreeSqlBuilder().UseQuestDbRestAPI(\"localhost:9000\", \"username\", \"password\")"); + throw new Exception( + "BulkCopy功能需要启用RestAPI，启用方式：new FreeSqlBuilder().UseQuestDbRestAPI(\"localhost:9000\", \"username\", \"password\")"); } + var result = 0; - var fileName = $"{Guid.NewGuid()}.csv"; - var filePath = Path.Combine(AppContext.BaseDirectory, fileName); + try { var client = QuestDbContainer.GetService().CreateClient(); var boundary = "---------------" + DateTime.Now.Ticks.ToString("x"); var list = new List(); var insert = that as QuestDbInsert; - var name = insert.InternalTableRuleInvoke(); //获取表名 + var name = insert.InternalTableRuleInvoke(); //获取表名 insert.InternalOrm.DbFirst.GetTableByName(name).Columns.ForEach(d => { if (d.DbTypeText == "TIMESTAMP") @@ -181,7 +186,7 @@ public static async Task ExecuteBulkCopyAsync(this IInsert that,strin { { "name", d.Name }, { "type", d.DbTypeText }, - { "pattern", dateFormat} + { "pattern", dateFormat } }); } else @@ -194,34 +199,37 @@ public static async Task ExecuteBulkCopyAsync(this IInsert that,strin } }); var schema = JsonConvert.SerializeObject(list); - //写入CSV文件 - using (var writer = new StreamWriter(filePath)) - using (var csv = new CsvWriter(writer, CultureInfo.CurrentCulture)) + using (MemoryStream stream = new MemoryStream()) { - await csv.WriteRecordsAsync(insert._source); - } + //写入CSV文件 + using (var writer = new StreamWriter(stream)) + using (var csv = new CsvWriter(writer, CultureInfo.CurrentCulture)) + { + await csv.WriteRecordsAsync(insert._source); + } - var httpContent = new MultipartFormDataContent(boundary); - if (!string.IsNullOrWhiteSpace(RestAPIExtension.authorization)) - client.DefaultRequestHeaders.Add("Authorization", RestAPIExtension.authorization); - httpContent.Add(new StringContent(schema), "schema"); - httpContent.Add(new ByteArrayContent(File.ReadAllBytes(filePath)), "data"); - //boundary带双引号 可能导致服务器错误情况 - httpContent.Headers.Remove("Content-Type"); - httpContent.Headers.TryAddWithoutValidation("Content-Type", - "multipart/form-data; boundary=" + boundary); - var httpResponseMessage = - await client.PostAsync($"{RestAPIExtension.BaseUrl}/imp?name={name}", httpContent); - var readAsStringAsync = await httpResponseMessage.Content.ReadAsStringAsync(); - var splitByLine = SplitByLine(readAsStringAsync); - foreach (var s in splitByLine) - { - if (s.Contains("Rows")) + var httpContent = new MultipartFormDataContent(boundary); + if (!string.IsNullOrWhiteSpace(RestAPIExtension.authorization)) + client.DefaultRequestHeaders.Add("Authorization", RestAPIExtension.authorization); + httpContent.Add(new StringContent(schema), "schema"); + httpContent.Add(new ByteArrayContent(stream.ToArray()), "data"); + //boundary带双引号 可能导致服务器错误情况 + httpContent.Headers.Remove("Content-Type"); + httpContent.Headers.TryAddWithoutValidation("Content-Type", + "multipart/form-data; boundary=" + boundary); + var httpResponseMessage = + await client.PostAsync($"{RestAPIExtension.BaseUrl}/imp?name={name}", httpContent); + var readAsStringAsync = await httpResponseMessage.Content.ReadAsStringAsync(); + var splitByLine = SplitByLine(readAsStringAsync); + foreach (var s in splitByLine) { - var strings = s.Split('|'); - if (strings[1].Trim() == "Rows imported") + if (s.Contains("Rows")) { - result = Convert.ToInt32(strings[2].Trim()); + var strings = s.Split('|'); + if (strings[1].Trim() == "Rows imported") + { + result = Convert.ToInt32(strings[2].Trim()); + } } } } @@ -230,17 +238,6 @@ public static async Task ExecuteBulkCopyAsync(this IInsert that,strin { throw e; } - finally - { - try - { - File.Delete(filePath); - } - catch - { - // ignored - } - } return result; } @@ -252,9 +249,9 @@ public static async Task ExecuteBulkCopyAsync(this IInsert that,strin /// /// 导入时，时间格式 默认:yyyy/M/d H:mm:ss /// - public static int ExecuteBulkCopy(this IInsert insert,string dateFormat = "yyyy/M/d H:mm:ss") where T : class + public static int ExecuteQuestDbBulkCopy(this IInsert insert, string dateFormat = "yyyy/M/d H:mm:ss") where T : class { - return ExecuteBulkCopyAsync(insert,dateFormat).ConfigureAwait(false).GetAwaiter().GetResult(); + return ExecuteQuestDbBulkCopyAsync(insert, dateFormat).ConfigureAwait(false).GetAwaiter().GetResult(); } } @@ -344,6 +341,7 @@ internal static FreeSqlBuilder UseQuestDbRestAPI(FreeSqlBuilder buider, string h var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{username}:{password}")); authorization = $"Basic {base64}"; } + //RestApi需要无参数 buider.UseNoneCommandParameter(true); return buider;