feat: taosbenchmark vgid (#505)

* feat: taosbenchmark supports vgid * feat: taosbenchmark support vgid. works first time * feat: use system cores to specify vgroups * fix: command line supplement insert needs to disable nthread auto * test: add sleep in tests/taosbenchmark/commandline.py * test: change test branch * fix: util func refactor * test: upgrade go version to 1.17
taosdata · Dec 3, 2022 · e62c5ea · e62c5ea
1 parent d4b98bd
commit e62c5ea
Show file tree

Hide file tree

Showing 23 changed files with 86 additions and 84 deletions.
diff --git a/.github/workflows/2.x-centos-release.yml b/.github/workflows/2.x-centos-release.yml
@@ -130,7 +130,7 @@ jobs:
           || github.event_name == 'push'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Set up Rust
         if: |

diff --git a/.github/workflows/2.x-macos-debug.yml b/.github/workflows/2.x-macos-debug.yml
@@ -90,7 +90,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Build & Install TDengine
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/2.x-macos-release.yml b/.github/workflows/2.x-macos-release.yml
@@ -90,7 +90,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Build & Install TDengine
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/2.x-taosbenchmark-debug.yml b/.github/workflows/2.x-taosbenchmark-debug.yml
@@ -82,7 +82,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Build & Install TDengine
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/2.x-taosbenchmark-release.yml b/.github/workflows/2.x-taosbenchmark-release.yml
@@ -82,7 +82,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Build & Install TDengine
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/2.x-taosdump-debug.yml b/.github/workflows/2.x-taosdump-debug.yml
@@ -83,7 +83,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Build & Install TDengine
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/2.x-taosdump-release.yml b/.github/workflows/2.x-taosdump-release.yml
@@ -92,7 +92,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Build & Install TDengine
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/3.0-coveralls.yml b/.github/workflows/3.0-coveralls.yml
@@ -85,7 +85,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Set up Rust
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/3.0-macos-release.yml b/.github/workflows/3.0-macos-release.yml
@@ -93,7 +93,7 @@ jobs:
           || github.event_name == 'push'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Set up Rust
         if: |

diff --git a/.github/workflows/3.0-taosBenchmark-debug.yml b/.github/workflows/3.0-taosBenchmark-debug.yml
@@ -79,7 +79,7 @@ jobs:
           || github.event_name == 'push'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Set up Rust
         if: |

diff --git a/.github/workflows/3.0-taosBenchmark-release.yml b/.github/workflows/3.0-taosBenchmark-release.yml
@@ -79,7 +79,7 @@ jobs:
           || github.event_name == 'push'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Set up Rust
         if: |

diff --git a/.github/workflows/3.0-taosdump-debug.yml b/.github/workflows/3.0-taosdump-debug.yml
@@ -83,7 +83,7 @@ jobs:
           || github.event_name == 'push'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: first build TDengine
         if: |

diff --git a/.github/workflows/3.0-taosdump-release-ws.yml b/.github/workflows/3.0-taosdump-release-ws.yml
@@ -83,7 +83,7 @@ jobs:
           || github.event_name == 'push'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Set up Rust
         if:

diff --git a/.github/workflows/3.0-taosdump-release.yml b/.github/workflows/3.0-taosdump-release.yml
@@ -83,7 +83,7 @@ jobs:
           || github.event_name == 'push'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: first build TDengine
         if: |

diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml
@@ -97,7 +97,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Set up Rust
         if:

diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml
@@ -111,7 +111,7 @@ jobs:
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         uses: actions/setup-go@v3
         with:
-          go-version: 1.14
+          go-version: 1.17
 
       - name: Set up Rust
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/inc/bench.h b/inc/bench.h
@@ -50,6 +50,8 @@
 #include <netinet/in.h>
 #include <sys/time.h>
 #include <netdb.h>
+#else
+#include <winsock2.h>
 #endif
 
 #include <regex.h>

diff --git a/inc/toolsdef.h b/inc/toolsdef.h
@@ -17,14 +17,6 @@
 #define __TOOLSDEF_H_
 
 #include <stdbool.h>
-#include <unistd.h>
-
-#ifdef WINDOWS
-#include <sysinfoapi.h>
-#include <winsock2.h>
-#endif
-
-#include <time.h>
 
 // max file name length on Linux is 255
 #define MAX_FILE_NAME_LEN 256  // max file name length on linux is 255.
@@ -168,6 +160,9 @@ int32_t toolsParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePr
 struct tm* toolsLocalTime(const time_t *timep, struct tm *result);
 int32_t toolsGetTimeOfDay(struct timeval *tv);
 int32_t toolsClockGetTime(int clock_id, struct timespec *pTS);
+int64_t toolsGetTimestampMs();
+int64_t toolsGetTimestampUs();
+int64_t toolsGetTimestampNs();
 
 #ifdef WINDOWS
 typedef struct {
@@ -224,40 +219,7 @@ int32_t       toolsCloseDir(TdDirPtr *ppDir);
 #endif
 
 int64_t atomic_add_fetch_64(int64_t volatile* ptr, int64_t val);
-
-#if defined(WINDOWS)
-    #define CLOCK_REALTIME 0
-#endif
-
-static FORCE_INLINE int64_t toolsGetTimestampMs() {
-    struct timeval systemTime;
-    toolsGetTimeOfDay(&systemTime);
-    return (int64_t)systemTime.tv_sec * 1000L +
-        (int64_t)systemTime.tv_usec / 1000;
-}
-
-static FORCE_INLINE int64_t toolsGetTimestampUs() {
-    struct timeval systemTime;
-    toolsGetTimeOfDay(&systemTime);
-    return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec;
-}
-
-static FORCE_INLINE int64_t toolsGetTimestampNs() {
-    struct timespec systemTime = {0};
-    toolsClockGetTime(CLOCK_REALTIME, &systemTime);
-    return (int64_t)systemTime.tv_sec * 1000000000L +
-        (int64_t)systemTime.tv_nsec;
-}
-
-static FORCE_INLINE int32_t toolsGetNumberOfCores() {
-#ifdef WINDOWS
-    SYSTEM_INFO info;
-    GetSystemInfo(&info);
-    return (int32_t)info.dwNumberOfProcessors;
-#else
-    return (int32_t)sysconf(_SC_NPROCESSORS_ONLN);
-#endif
-}
+int32_t toolsGetNumberOfCores();
 
 int64_t toolsGetTimestamp(int32_t precision);
 void    toolsMsleep(int32_t mseconds);

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -201,9 +201,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
         ADD_DEPENDENCIES(taosdump deps-jansson)
         ADD_DEPENDENCIES(taosdump deps-snappy)
         IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
-            ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c)
+            ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c toolsSys.c)
         ELSE()
-            ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c)
+            ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c toolsSys.c)
         ENDIF()
     ELSE ()
         INCLUDE_DIRECTORIES(/usr/local/include)
@@ -212,9 +212,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
         SET(OS_ID "Darwin")
 
         IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
-            ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c)
+            ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c toolsSys.c)
         ELSE()
-            ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c)
+            ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c toolsSys.c)
         ENDIF()
     ENDIF ()
 
@@ -366,9 +366,9 @@ ELSE ()
     set (CMAKE_C_STANDARD 11)
     SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8")
     IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
-        ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c toolsString.c)
+        ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c toolsString.c toolsSys.c)
     ELSE ()
-        ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c)
+        ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c toolstime.c toolsSys.c)
     ENDIF ()
 
     ADD_EXECUTABLE(taosdump taosdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c)

diff --git a/src/benchMain.c b/src/benchMain.c
@@ -11,6 +11,7 @@
  */
 
 #include "bench.h"
+#include "toolsdef.h"
 
 SArguments*    g_arguments;
 SQueryMetaInfo g_queryInfo;

diff --git a/src/benchUtil.c b/src/benchUtil.c
@@ -97,20 +97,6 @@ unsigned int taosRandom() {
 
     return number;
 }
-
-void usleep(__int64 usec)
-{
-    HANDLE timer;
-    LARGE_INTEGER ft;
-
-    ft.QuadPart = -(10*usec); // Convert to 100 nanosecond interval, negative value indicates relative time
-
-    timer = CreateWaitableTimer(NULL, TRUE, NULL);
-    SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0);
-    WaitForSingleObject(timer, INFINITE);
-    CloseHandle(timer);
-}
-
 #else  // Not windows
 void setupForAnsiEscape(void) {}
 
@@ -119,8 +105,7 @@ void resetAfterAnsiEscape(void) {
     printf("\x1b[0m");
 }
 
-unsigned int taosRandom() { return (unsigned int)rand(); }
-
+FORCE_INLINE unsigned int taosRandom() { return (unsigned int)rand(); }
 #endif
 
 int getAllChildNameOfSuperTable(TAOS *taos, char *dbName, char *stbName,
@@ -245,8 +230,6 @@ int64_t toolsGetTimestamp(int32_t precision) {
     }
 }
 
-void toolsMsleep(int32_t mseconds) { usleep(mseconds * 1000); }
-
 int regexMatch(const char *s, const char *reg, int cflags) {
     regex_t regex;
     char    msgbuf[100] = {0};

diff --git a/src/toolsSys.c b/src/toolsSys.c
@@ -16,6 +16,9 @@
 #ifdef WINDOWS
 #include <time.h>
 #include <WinSock2.h>
+#include <sysinfoapi.h>
+#else
+#include <unistd.h>
 #endif
 
 #include <toolsdef.h>
@@ -30,3 +33,13 @@ int64_t atomic_add_fetch_64(int64_t volatile* ptr, int64_t val) {
 #endif
 }
 
+FORCE_INLINE int32_t toolsGetNumberOfCores() {
+#ifdef WINDOWS
+    SYSTEM_INFO info;
+    GetSystemInfo(&info);
+    return (int32_t)info.dwNumberOfProcessors;
+#else
+    return (int32_t)sysconf(_SC_NPROCESSORS_ONLN);
+#endif
+}
+
diff --git a/src/toolstime.c b/src/toolstime.c
@@ -36,11 +36,12 @@
 #include "bench.h"
 #include "toolsdef.h"
 
-#if defined(WIN32) || defined(WIN64)
+#if defined(WINDOWS)
 
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
+#include <winsock2.h>
 //#define TM_YEAR_BASE 1970 //origin
 #define TM_YEAR_BASE 1900  // slguan
 /*
@@ -772,7 +773,8 @@ struct tm* toolsLocalTime(const time_t *timep, struct tm *result) {
 #endif
     return result;
 }
-int32_t toolsGetTimestampSec() { return (int32_t)time(NULL); }
+
+FORCE_INLINE int32_t toolsGetTimestampSec() { return (int32_t)time(NULL); }
 
 FORCE_INLINE int32_t toolsGetTimeOfDay(struct timeval *tv) {
 #if defined(WIN32) || defined(WIN64)
@@ -788,3 +790,42 @@ FORCE_INLINE int32_t toolsGetTimeOfDay(struct timeval *tv) {
     return gettimeofday(tv, NULL);
 #endif
 }
+
+FORCE_INLINE int64_t toolsGetTimestampMs() {
+    struct timeval systemTime;
+    toolsGetTimeOfDay(&systemTime);
+    return (int64_t)systemTime.tv_sec * 1000L +
+        (int64_t)systemTime.tv_usec / 1000;
+}
+
+FORCE_INLINE int64_t toolsGetTimestampUs() {
+    struct timeval systemTime;
+    toolsGetTimeOfDay(&systemTime);
+    return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec;
+}
+
+#if defined(WINDOWS)
+    #define CLOCK_REALTIME 0
+
+void usleep(__int64 usec)
+{
+    HANDLE timer;
+    LARGE_INTEGER ft;
+
+    ft.QuadPart = -(10*usec); // Convert to 100 nanosecond interval, negative value indicates relative time
+
+    timer = CreateWaitableTimer(NULL, TRUE, NULL);
+    SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0);
+    WaitForSingleObject(timer, INFINITE);
+    CloseHandle(timer);
+}
+#endif
+
+FORCE_INLINE int64_t toolsGetTimestampNs() {
+    struct timespec systemTime = {0};
+    toolsClockGetTime(CLOCK_REALTIME, &systemTime);
+    return (int64_t)systemTime.tv_sec * 1000000000L +
+        (int64_t)systemTime.tv_nsec;
+}
+
+FORCE_INLINE void toolsMsleep(int32_t mseconds) { usleep(mseconds * 1000); }