Merge branch 'unstable' into hnsw-indexer

.asf.yaml

@@ -42,6 +42,16 @@ github:
         strict: true
         contexts:
           - Required
+    '1.3': {}
+    '2.0': {}
+    '2.1': {}
+    '2.2': {}
+    '2.3': {}
+    '2.4': {}
+    '2.5': {}
+    '2.6': {}
+    '2.7': {}
+    '2.8': {}
 
 notifications:
   commits:      commits@kvrocks.apache.org

.github/workflows/kvrocks.yaml

@@ -57,10 +57,10 @@ jobs:
       FORCE_COLOR: 1
     steps:
       - uses: actions/checkout@v4
-      - name: Install typos
-        run: curl -LsSf https://github.com/crate-ci/typos/releases/download/v1.18.2/typos-v1.18.2-x86_64-unknown-linux-musl.tar.gz | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
-      - name: Run typos check
-        run: typos --config .github/config/typos.toml
+      - name: Check typos
+        uses: crate-ci/typos@v1.22.9
+        with:
+          config: .github/config/typos.toml
 
   check-and-lint:
     name: Lint and check code
@@ -110,24 +110,25 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Darwin Clang
-            os: macos-11
-            compiler: auto
+          # FIXME: update macos-11 to macos-12/13
+          # - name: Darwin Clang
+          #   os: macos-11
+          #   compiler: auto
           - name: Darwin Clang arm64
             os: macos-14
             compiler: auto
-          - name: Darwin Clang without Jemalloc
-            os: macos-11
-            compiler: auto
-            without_jemalloc: -DDISABLE_JEMALLOC=ON
-          - name: Darwin Clang with OpenSSL
-            os: macos-11
-            compiler: auto
-            with_openssl: -DENABLE_OPENSSL=ON
-          - name: Darwin Clang without luaJIT
-            os: macos-11
-            compiler: auto
-            without_luajit: -DENABLE_LUAJIT=OFF
+          # - name: Darwin Clang without Jemalloc
+          #   os: macos-11
+          #   compiler: auto
+          #   without_jemalloc: -DDISABLE_JEMALLOC=ON
+          # - name: Darwin Clang with OpenSSL
+          #   os: macos-11
+          #   compiler: auto
+          #   with_openssl: -DENABLE_OPENSSL=ON
+          # - name: Darwin Clang without luaJIT
+          #   os: macos-11
+          #   compiler: auto
+          #   without_luajit: -DENABLE_LUAJIT=OFF
           - name: Ubuntu GCC
             os: ubuntu-20.04
             compiler: gcc
@@ -389,7 +390,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Get core numbers
         run: echo "NPROC=$(nproc)" >> $GITHUB_ENV
-      - uses: docker/build-push-action@v5
+      - uses: docker/build-push-action@v6
         with:
           context: .
           build-args: MORE_BUILD_ARGS=-j${{ env.NPROC }}
@@ -419,9 +420,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: CentOS 7
-            image: centos:7
-            compiler: gcc
           - name: openSUSE Leap 15
             image: opensuse/leap:15
             compiler: gcc
@@ -433,17 +431,6 @@ jobs:
     container:
       image: ${{ matrix.image }}
     steps:
-      - name: Setup CentOS
-        if: ${{ startsWith(matrix.image, 'centos') }}
-        run: |
-          yum install -y centos-release-scl-rh
-          yum install -y devtoolset-11 python3 python3-pip autoconf automake wget git gcc gcc-c++
-          echo "NPROC=$(nproc)" >> $GITHUB_ENV
-          mv /usr/bin/gcc /usr/bin/gcc-4.8.5
-          ln -s /opt/rh/devtoolset-11/root/bin/gcc /usr/bin/gcc
-          mv /usr/bin/g++ /usr/bin/g++-4.8.5
-          ln -s /opt/rh/devtoolset-11/root/bin/g++ /usr/bin/g++
-
       - name: Setup ArchLinux
         if: ${{ startsWith(matrix.image, 'archlinux') }}
         run: |
@@ -486,15 +473,8 @@ jobs:
           pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-cli && mv src/redis-cli $HOME/local/bin/ && popd
           pushd redis-6.2.14 && USE_JEMALLOC=no make -j$NPROC redis-server && mv src/redis-server $HOME/local/bin/ && popd
 
-      - name: Install cmake
-        if: ${{ startsWith(matrix.image, 'centos') }}
-        run: |
-          VERSION=3.26.4
-          wget https://github.com/Kitware/CMake/releases/download/v$VERSION/cmake-$VERSION-linux-x86_64.sh
-          bash cmake-$VERSION-linux-x86_64.sh --skip-license --prefix=/usr
-
-      - uses: actions/checkout@v3 #v4 use Node 20 and not working at CentOS 7
-      - uses: actions/setup-go@v4 #v5 use Node 20 too
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         if: ${{ !startsWith(matrix.image, 'opensuse') }}
         with:
           go-version-file: 'tests/gocase/go.mod'

.github/workflows/nightly.yaml

@@ -61,7 +61,7 @@ jobs:
             type=sha,prefix=nightly-{{date 'YYYYMMDD'}}-,format=short
             type=raw,value=nightly
 
-      - uses: docker/build-push-action@v5
+      - uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64, linux/arm64

cmake/fmt.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(fmt
-  fmtlib/fmt 10.2.1
-  MD5=1bba4e8bdd7b0fa98f207559ffa380a3
+  fmtlib/fmt 11.0.1
+  MD5=6ca210c0a9e751705bacff468dd55ee1
 )
 
 FetchContent_MakeAvailableWithArgs(fmt)

cmake/rocksdb.cmake

@@ -26,8 +26,8 @@ endif()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(rocksdb
-  facebook/rocksdb v9.2.1
-  MD5=b03db4c602754a2c50aa2967fd07a2a7
+  facebook/rocksdb v9.3.1
+  MD5=129235c789a963c004290d27d09ca48a
 )
 
 FetchContent_GetProperties(jemalloc)

cmake/tbb.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(tbb
-  oneapi-src/oneTBB v2021.12.0
-  MD5=0919a8eda74333e1aafa8d602bb9cc90
+  oneapi-src/oneTBB v2021.13.0
+  MD5=2dd9b7cfa5de5bb3add2f7392e0c9bab
 )
 
 FetchContent_MakeAvailableWithArgs(tbb

kvrocks.conf

@@ -492,15 +492,23 @@ profiling-sample-record-threshold-ms 100
 ################################## CRON ###################################
 
 # Compact Scheduler, auto compact at schedule time
-# Time expression format is the same as crontab (currently only support *, int and */int)
-# e.g. compact-cron 0 3 * * * 0 4 * * *
+# Time expression format is the same as crontab (supported cron syntax: *, n, */n, `1,3-6,9,11`)
+# e.g. compact-cron 0 3,4 * * *
 # would compact the db at 3am and 4am everyday
 # compact-cron 0 3 * * *
 
 # The hour range that compaction checker would be active
 # e.g. compaction-checker-range 0-7 means compaction checker would be worker between
 # 0-7am every day.
-compaction-checker-range 0-7
+# WARNING: this config option is deprecated and will be removed,
+# please use compaction-checker-cron instead
+# compaction-checker-range 0-7
+
+# The time pattern that compaction checker would be active
+# Time expression format is the same as crontab (supported cron syntax: *, n, */n, `1,3-6,9,11`)
+# e.g. compaction-checker-cron * 0-7 * * * means compaction checker would be worker between
+# 0-7am every day.
+compaction-checker-cron * 0-7 * * *
 
 # When the compaction checker is triggered, the db will periodically pick the SST file
 # with the highest "deleted percentage" (i.e. the percentage of deleted keys in the SST 
@@ -515,14 +523,14 @@ compaction-checker-range 0-7
 # force-compact-file-min-deleted-percentage 10
 
 # Bgsave scheduler, auto bgsave at scheduled time
-# Time expression format is the same as crontab (currently only support *, int and */int)
-# e.g. bgsave-cron 0 3 * * * 0 4 * * *
+# Time expression format is the same as crontab (supported cron syntax: *, n, */n, `1,3-6,9,11`)
+# e.g. bgsave-cron 0 3,4 * * *
 # would bgsave the db at 3am and 4am every day
 
 # Kvrocks doesn't store the key number directly. It needs to scan the DB and
 # then retrieve the key number by using the dbsize scan command.
 # The Dbsize scan scheduler auto-recalculates the estimated keys at scheduled time.
-# Time expression format is the same as crontab (currently only support *, int and */int)
+# Time expression format is the same as crontab (supported cron syntax: *, n, */n, `1,3-6,9,11`)
 # e.g. dbsize-scan-cron 0 * * * *
 # would recalculate the keyspace infos of the db every hour.
 

src/cluster/slot_migrate.cc

@@ -382,6 +382,12 @@ Status SlotMigrator::sendSnapshotByCmd() {
     }
   }
 
+  if (auto s = iter->status(); !s.ok()) {
+    auto err_str = s.ToString();
+    LOG(ERROR) << "[migrate] Failed to iterate keys of slot " << slot << ": " << err_str;
+    return {Status::NotOK, fmt::format("failed to iterate keys of slot {}: {}", slot, err_str)};
+  }
+
   // It's necessary to send commands that are still in the pipeline since the final pipeline may not be sent
   // while iterating keys because its size could be less than max_pipeline_size_
   auto s = sendCmdsPipelineIfNeed(&restore_cmds, true);
@@ -820,6 +826,11 @@ Status SlotMigrator::migrateComplexKey(const rocksdb::Slice &key, const Metadata
     }
   }
 
+  if (auto s = iter->status(); !s.ok()) {
+    return {Status::NotOK,
+            fmt::format("failed to iterate values of the complex key {}: {}", key.ToString(), s.ToString())};
+  }
+
   // Have to check the item count of the last command list
   if (item_count % kMaxItemsInCommand != 0) {
     *restore_cmds += redis::ArrayOfBulkStrings(user_cmd);
@@ -880,6 +891,11 @@ Status SlotMigrator::migrateStream(const Slice &key, const StreamMetadata &metad
     }
   }
 
+  if (auto s = iter->status(); !s.ok()) {
+    return {Status::NotOK,
+            fmt::format("failed to iterate values of the stream key {}: {}", key.ToString(), s.ToString())};
+  }
+
   // commands like XTRIM and XDEL affect stream's metadata, but we use only XADD for a slot migration
   // XSETID is used to adjust stream's info on the destination node according to the current values on the source
   *restore_cmds += redis::ArrayOfBulkStrings({"XSETID", key.ToString(), metadata.last_generated_id.ToString(),

src/commands/cmd_json.cc

@@ -423,8 +423,7 @@ class CommandJsonArrTrim : public Commander {
     auto s = json.ArrTrim(args_[1], path_, start_, stop_, &results);
 
     if (s.IsNotFound()) {
-      *output = conn->NilString();
-      return Status::OK();
+      return {Status::RedisExecErr, "could not perform this operation on a key that doesn't exist"};
     }
     if (!s.ok()) return {Status::RedisExecErr, s.ToString()};
 

src/commands/cmd_stream.cc

@@ -18,6 +18,8 @@
  *
  */
 
+#include <algorithm>
+#include <limits>
 #include <memory>
 #include <stdexcept>
 
@@ -358,6 +360,97 @@ class CommandXClaim : public Commander {
   }
 };
 
+class CommandAutoClaim : public Commander {
+ public:
+  Status Parse(const std::vector<std::string> &args) override {
+    CommandParser parser(args, 1);
+    key_name_ = GET_OR_RET(parser.TakeStr());
+    group_name_ = GET_OR_RET(parser.TakeStr());
+    consumer_name_ = GET_OR_RET(parser.TakeStr());
+    if (auto parse_status = parser.TakeInt<uint64_t>(); !parse_status.IsOK()) {
+      return {Status::RedisParseErr, "Invalid min-idle-time argument for XAUTOCLAIM"};
+    } else {
+      options_.min_idle_time_ms = parse_status.GetValue();
+    }
+
+    auto start_str = GET_OR_RET(parser.TakeStr());
+    if (!start_str.empty() && start_str.front() == '(') {
+      options_.exclude_start = true;
+      start_str = start_str.substr(1);
+    }
+    if (!options_.exclude_start && start_str == "-") {
+      options_.start_id = StreamEntryID::Minimum();
+    } else {
+      auto parse_status = ParseRangeStart(start_str, &options_.start_id);
+      if (!parse_status.IsOK()) {
+        return parse_status;
+      }
+    }
+
+    if (parser.EatEqICase("count")) {
+      uint64_t count = GET_OR_RET(parser.TakeInt<uint64_t>());
+      constexpr uint64_t min_count = 1;
+      uint64_t max_count = std::numeric_limits<int64_t>::max() /
+                           (std::max(static_cast<uint64_t>(sizeof(StreamEntryID)), options_.attempts_factors));
+      if (count < min_count || count > max_count) {
+        return {Status::RedisParseErr, "COUNT must be > 0"};
+      }
+      options_.count = count;
+    }
+
+    if (parser.Good() && parser.EatEqICase("justid")) {
+      options_.just_id = true;
+    }
+
+    return Status::OK();
+  }
+
+  Status Execute(Server *srv, Connection *conn, std::string *output) override {
+    redis::Stream stream_db(srv->storage, conn->GetNamespace());
+    StreamAutoClaimResult result;
+    auto s = stream_db.AutoClaim(key_name_, group_name_, consumer_name_, options_, &result);
+    if (!s.ok()) {
+      if (s.IsNotFound()) {
+        return {Status::RedisExecErr,
+                "NOGROUP No such key '" + key_name_ + "' or consumer group '" + group_name_ + "'"};
+      }
+      return {Status::RedisExecErr, s.ToString()};
+    }
+    return sendResults(conn, result, output);
+  }
+
+ private:
+  Status sendResults(Connection *conn, const StreamAutoClaimResult &result, std::string *output) const {
+    output->append(redis::MultiLen(3));
+    output->append(redis::BulkString(result.next_claim_id));
+    output->append(redis::MultiLen(result.entries.size()));
+    for (const auto &item : result.entries) {
+      if (options_.just_id) {
+        output->append(redis::BulkString(item.key));
+      } else {
+        output->append(redis::MultiLen(2));
+        output->append(redis::BulkString(item.key));
+        output->append(redis::MultiLen(item.values.size()));
+        for (const auto &value_item : item.values) {
+          output->append(redis::BulkString(value_item));
+        }
+      }
+    }
+
+    output->append(redis::MultiLen(result.deleted_ids.size()));
+    for (const auto &item : result.deleted_ids) {
+      output->append(redis::BulkString(item));
+    }
+
+    return Status::OK();
+  }
+
+  std::string key_name_;
+  std::string group_name_;
+  std::string consumer_name_;
+  StreamAutoClaimOptions options_;
+};
+
 class CommandXGroup : public Commander {
  public:
   Status Parse(const std::vector<std::string> &args) override {
@@ -1647,6 +1740,7 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr<CommandXAck>("xack", -4, "write no-dbsize-ch
                         MakeCmdAttr<CommandXAdd>("xadd", -5, "write", 1, 1, 1),
                         MakeCmdAttr<CommandXDel>("xdel", -3, "write no-dbsize-check", 1, 1, 1),
                         MakeCmdAttr<CommandXClaim>("xclaim", -6, "write", 1, 1, 1),
+                        MakeCmdAttr<CommandAutoClaim>("xautoclaim", -6, "write", 1, 1, 1),
                         MakeCmdAttr<CommandXGroup>("xgroup", -4, "write", 2, 2, 1),
                         MakeCmdAttr<CommandXLen>("xlen", -2, "read-only", 1, 1, 1),
                         MakeCmdAttr<CommandXInfo>("xinfo", -2, "read-only", 0, 0, 0),