Change: Remove feature flag single-term-leader

In OpenRaft, whether a term supports multiple or single leaders is
determined by the ordering implementation of `LeaderId`. This behavior
is independent of compilation and can be configured at the application
level, removing the need for a compilation flag `single-term-leader`.

### Changes:

- The `single-term-leader` feature flag is removed.
- Leader mode (multi-leader or single-leader per term) is now controlled
  by the `LeaderId` associated type in the application configuration.

### Configuration Guide:

To enable **standard Raft mode (single leader per term)**:
- Set `LeaderId` to `openraft::impls::leader_id_std::LeaderId` in the
  `declare_raft_types!` statement, or within the `RaftTypeConfig`
  implementation.

Example:

```rust
openraft::declare_raft_types!(
    pub TypeConfig:
        D = String,
        LeaderId = openraft::impls::leader_id_std::LeaderId<TypeConfig>,
);
```

To use the **default multi-leader per term mode**, leave `LeaderId`
unset or explicitly set it to
`openraft::impls::leader_id_adv::LeaderId`.

Example:

```rust
openraft::declare_raft_types!(
    pub TypeConfig:
        D = String,
);
// Or:
openraft::declare_raft_types!(
    pub TypeConfig:
        D = String,
        LeaderId = openraft::impls::leader_id_adv::LeaderId<TypeConfig>,
);
```

Upgrade tip:

If the `single-term-leader` flag was previously used, remove it and
configure `LeaderId` as follows:

```rust
openraft::declare_raft_types!(
    pub TypeConfig:
        LeaderId = openraft::impls::leader_id_std::LeaderId<TypeConfig>,
);
```

.github/workflows/ci.yaml

@@ -63,29 +63,20 @@ jobs:
           args: --features bench nothing-to-run --manifest-path openraft/Cargo.toml
 
 
-  # Run openraft unit test `openraft/` and integration test `tests/`.
-  openraft-test:
+  # Run openraft unit test `openraft/`
+  test-crate-openraft:
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
         include:
           - toolchain: "stable"
-            send_delay: "0"
             features: ""
 
-          # With network delay
           - toolchain: "nightly"
-            send_delay: "30"
             features: ""
 
-          # Feature-flag: Standard raft term
-          - toolchain: "nightly"
-            send_delay: "0"
-            features: "single-term-leader"
-
-
     steps:
       - name: Setup | Checkout
         uses: actions/checkout@v4
@@ -98,19 +89,57 @@ jobs:
           override: true
 
 
-      # - A store with defensive checks returns error when unexpected accesses are sent to RaftStore.
-      # - Raft should not depend on defensive error to work correctly.
       - name: Test crate `openraft/`
         uses: actions-rs/cargo@v1
         with:
           command: test
           args: --features "${{ matrix.features }}" --manifest-path openraft/Cargo.toml
         env:
-          # Parallel tests block each other and result in timeout.
-          RUST_TEST_THREADS: 2
           RUST_LOG: debug
           RUST_BACKTRACE: full
-          OPENRAFT_NETWORK_SEND_DELAY: ${{ matrix.send_delay }}
+
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: "ut-openraft-${{ matrix.toolchain }}-${{ matrix.features }}"
+          path: |
+            openraft/_log/
+
+  # Run integration test `tests/`.
+  test-crate-tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - toolchain: "stable"
+            send_delay: "0"
+            features: ""
+
+          # With network delay
+          - toolchain: "nightly"
+            send_delay: "30"
+            features: ""
+
+          # Feature-flag: Standard raft term
+          - toolchain: "nightly"
+            send_delay: "0"
+            features: "single-term-leader"
+
+
+    steps:
+      - name: Setup | Checkout
+        uses: actions/checkout@v4
+
+
+      - name: Setup | Toolchain
+        uses: actions-rs/toolchain@v1.0.6
+        with:
+          toolchain: "${{ matrix.toolchain }}"
+          override: true
 
 
       - name: Test crate `tests/`
@@ -130,9 +159,8 @@ jobs:
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: "ut-${{ matrix.toolchain }}-${{ matrix.features }}-${{ matrix.send_delay }}"
+          name: "ut-tests-${{ matrix.toolchain }}-${{ matrix.features }}-${{ matrix.send_delay }}"
           path: |
-            openraft/_log/
             tests/_log/
 
 
@@ -252,16 +280,8 @@ jobs:
           - toolchain: "nightly"
             features: "serde"
 
-          # Some test requires feature single-term-leader on and serde off.
-          # This can only be tested without building another crate that enables
-          # `serde`.
-          # Move this test to unit-test when the snapshot API is upgraded to
-          # non-serde-dependent.
-          - toolchain: "nightly"
-            features: "single-term-leader"
-
           - toolchain: "nightly"
-            features: "single-term-leader,serde,singlethreaded"
+            features: "serde,singlethreaded"
 
 
     steps:
@@ -364,7 +384,7 @@ jobs:
         shell: bash
         run: |
           cargo clippy --no-deps --workspace --all-targets                -- -D warnings
-          cargo clippy --no-deps --workspace --all-targets --features "bt,serde,bench,single-term-leader,compat" -- -D warnings
+          cargo clippy --no-deps --workspace --all-targets --features "bt,serde,bench,compat" -- -D warnings
 
 
       - name: Build-doc

cluster_benchmark/Cargo.toml

@@ -31,7 +31,7 @@ tracing = "0.1.29"
 [features]
 
 bt = ["openraft/bt"]
-single-term-leader = ["openraft/single-term-leader"]
+single-term-leader = []
 
 [profile.release]
 debug = 2

cluster_benchmark/tests/benchmark/store.rs

@@ -37,11 +37,20 @@ pub struct ClientResponse {}
 
 pub type NodeId = u64;
 
+/// Choose a LeaderId implementation by feature flag.
+mod leader_id_mode {
+    #[cfg(not(feature = "single-term-leader"))]
+    pub use openraft::impls::leader_id_adv::LeaderId;
+    #[cfg(feature = "single-term-leader")]
+    pub use openraft::impls::leader_id_std::LeaderId;
+}
+
 openraft::declare_raft_types!(
     pub TypeConfig:
         D = ClientRequest,
         R = ClientResponse,
         Node = (),
+        LeaderId = leader_id_mode::LeaderId<TypeConfig>,
 );
 
 #[derive(Debug)]

examples/utils/declare_types.rs

@@ -4,7 +4,7 @@ use super::TypeConfig;
 pub type Raft = openraft::Raft<TypeConfig>;
 
 pub type Vote = openraft::Vote<TypeConfig>;
-pub type LeaderId = openraft::LeaderId<TypeConfig>;
+pub type LeaderId = <TypeConfig as openraft::RaftTypeConfig>::LeaderId;
 pub type LogId = openraft::LogId<TypeConfig>;
 pub type Entry = openraft::Entry<TypeConfig>;
 pub type EntryPayload = openraft::EntryPayload<TypeConfig>;

openraft/Cargo.toml

@@ -62,17 +62,9 @@ bt  = ["anyerror/backtrace", "anyhow/backtrace"]
 # If you'd like to use `serde` to serialize messages.
 serde = ["dep:serde"]
 
-# Turn on this feature it allows at most ONE quorum-granted leader for each term.
-# This is the way standard raft does, by making the LeaderId a partial order value.
-#
-# - With this feature on:
-#   It is more likely to conflict during election. But every log only needs to store one `term` in it.
-#
-# - With this feature off:
-#   Election conflict rate will be reduced, but every log has to store a `LeaderId{ term, node_id}`,
-#   which may be costly if an application uses a big NodeId type.
-#
-# This feature is disabled by default.
+# This feature is removed.
+# Use `openraft::impls::leader_id_std::Leader` for `RaftTypeConfig`
+# to enable standard Raft leader election.
 single-term-leader = []
 
 # Enable this feature to use `openraft::alias::*` type shortcuts.
@@ -103,8 +95,6 @@ loosen-follower-log-revert = []
 # See: https://docs.rs/tracing/latest/tracing/#emitting-log-records
 tracing-log = [ "tracing/log" ]
 
-# default = ["single-term-leader"]
-
 [package.metadata.docs.rs]
 
 # Enable these feature flags to show all types/mods,

openraft/src/core/notification.rs

@@ -7,8 +7,8 @@ use crate::raft_state::IOId;
 use crate::replication;
 use crate::replication::ReplicationSessionId;
 use crate::type_config::alias::InstantOf;
-use crate::vote::CommittedVote;
-use crate::vote::NonCommittedVote;
+use crate::vote::committed::CommittedVote;
+use crate::vote::non_committed::NonCommittedVote;
 use crate::RaftTypeConfig;
 use crate::StorageError;
 use crate::Vote;

openraft/src/core/raft_core.rs

@@ -93,9 +93,9 @@ use crate::type_config::alias::ResponderOf;
 use crate::type_config::alias::WatchSenderOf;
 use crate::type_config::async_runtime::MpscUnboundedReceiver;
 use crate::type_config::TypeConfigExt;
+use crate::vote::committed::CommittedVote;
+use crate::vote::non_committed::NonCommittedVote;
 use crate::vote::vote_status::VoteStatus;
-use crate::vote::CommittedVote;
-use crate::vote::NonCommittedVote;
 use crate::vote::RaftLeaderId;
 use crate::ChangeMembers;
 use crate::Instant;

openraft/src/core/tick.rs

@@ -180,7 +180,7 @@ mod tests {
         type NodeId = u64;
         type Node = ();
         type Term = u64;
-        type LeaderId = crate::impls::LeaderId<Self>;
+        type LeaderId = crate::impls::leader_id_adv::LeaderId<Self>;
         type Entry = crate::Entry<TickUTConfig>;
         type SnapshotData = Cursor<Vec<u8>>;
         type AsyncRuntime = TokioRuntime;

openraft/src/docs/data/leader_id.md

@@ -1,8 +1,13 @@
 ## Leader-id in Advanced mode and Standard mode
 
-Openraft provides two `LeaderId` types to switch between these two modes with feature [`single-term-leader`] :
+Openraft provides two `LeaderId` types to switch between these two modes:
+- the default mode: every term may have more than one leader
+  (enabled by default, or explicitly by setting [`RaftTypeConfig::LeaderId`] to [`leader_id_adv::LeaderId`]).
+- and the standard Raft mode: every term has only one leader
+  (enabled by setting [`RaftTypeConfig::LeaderId`] to [`leader_id_std::LeaderId`]).
 
-The feature [`single-term-leader`] affects the `PartialOrd` implementation of `LeaderId`, where `LeaderId` is defined as a tuple `(term, node_id)`.
+[`leader_id_adv::LeaderId`] is totally ordered.
+[`leader_id_std::LeaderId`] is `PartialOrd`.
 
 ### Definition of `LeaderId`
 
@@ -15,27 +20,26 @@ Within Openraft, and also implicitly in the standard Raft, `LeaderId` is utilize
   `A.term > B.term ↔ A > B`.
   <br/><br/>
 
-- Conversely, in Openraft, with the [`single-term-leader`] feature disabled by default, `LeaderId` follows a `total order` based on lexicographical comparison:
+- Conversely, in Openraft, with [`leader_id_adv::LeaderId`], `LeaderId` follows a `total order` based on lexicographical comparison:
 
   `A.term > B.term || (A.term == B.term && A.node_id > B.node_id) ↔ A > B`.
 
-Activating the `single-term-leader` feature makes `LeaderId` conform to the `partial order` seen in standard Raft.
+Using [`leader_id_std::LeaderId`] makes `LeaderId` conform to the `partial order` seen in standard Raft.
 
 ### Usage of `LeaderId`
 
 When handling `VoteRequest`, both Openraft and standard Raft (though not explicitly detailed) rely on the ordering of `LeaderId` to decide whether to grant a vote:
 **a node will grant a vote with a `LeaderId` that is greater than any it has previously granted**.
 
-Consequently, by default in Openraft (with `single-term-leader` disabled), it is possible to elect multiple `Leader`s within the same term, with the last elected `Leader` being recognized as valid. In contrast, under standard Raft protocol, only a single `Leader` is elected per `term`.
+Consequently, by default in Openraft (with [`leader_id_adv::LeaderId`]), it is possible to elect multiple `Leader`s within the same term, with the last elected `Leader` being recognized as valid. In contrast, under standard Raft protocol, only a single `Leader` is elected per `term`.
 
 ### Default: advanced mode
 
-`cargo build` without [`single-term-leader`][], is the advanced mode, the default mode:
+Use `openraft::impls::leader_id_adv::LeaderId` for [`RaftTypeConfig::LeaderId`] or leave it to default to switch to advanced mode.
 `LeaderId` is defined as the following, and it is a **totally ordered** value(two or more leaders can be granted in the same term):
 
 ```ignore
 // Advanced mode(default):
-#[cfg(not(feature = "single-term-leader"))]
 #[derive(PartialOrd, Ord)]
 pub struct LeaderId<NID: NodeId>
 {
@@ -57,12 +61,11 @@ elected(although only the last is valid and can commit logs).
 
 #### Standard mode
 
-`cargo build --features "single-term-leader"` builds openraft in standard raft mode.
+Use `openraft::impls::leader_id_std::LeaderId` for [`RaftTypeConfig::LeaderId`] to switch to standard mode.
 In the standard mode, `LeaderId` is defined as the following, and it is a **partially ordered** value(no two leaders can be granted in the same term):
 
 ```ignore
 // Standard raft mode:
-#[cfg(feature = "single-term-leader")]
 pub struct LeaderId<NID: NodeId>
 {
   pub term: u64,
@@ -125,4 +128,6 @@ So let a committed `Vote` override a incomparable non-committed is safe.
 - Cons: election conflicting rate may increase.
 
 
-[`single-term-leader`]: crate::docs::feature_flags
+[`RaftTypeConfig::LeaderId`]: crate::RaftTypeConfig::LeaderId
+[`leader_id_adv::LeaderId`]: `crate::impls::leader_id_adv::LeaderId`
+[`leader_id_std::LeaderId`]: `crate::impls::leader_id_std::LeaderId`

openraft/src/docs/data/vote.md

@@ -34,8 +34,10 @@ i.e., it is legal that `!(vote_a => vote_b) && !(vote_a <= vote_b)`.
 Because `Vote.leader_id` may be a partial order value:
 
 Openraft provides two election modes.
-- the default mode: every term may have more than one leader.
-- and the standard Raft mode: every term has only one leader(enabled by [`single-term-leader`]),
+- the default mode: every term may have more than one leader
+  (enabled by default, or explicitly by setting [`RaftTypeConfig::LeaderId`] to [`leader_id_adv::LeaderId`]).
+- and the standard Raft mode: every term has only one leader
+  (enabled by setting [`RaftTypeConfig::LeaderId`] to [`leader_id_std::LeaderId`]),
 
 The only difference between these two modes is the definition of `LeaderId`, and the `PartialOrd` implementation of it.
 See: [`leader-id`].
@@ -79,5 +81,7 @@ For node-2:
 
 
 [`Vote`]: `crate::vote::Vote`
-[`single-term-leader`]: `crate::docs::feature_flags`
+[`RaftTypeConfig::LeaderId`]: `crate::RaftTypeConfig::LeaderId`
+[`leader_id_adv::LeaderId`]: `crate::impls::leader_id_adv::LeaderId`
+[`leader_id_std::LeaderId`]: `crate::impls::leader_id_std::LeaderId`
 [`leader-id`]: `crate::docs::data::leader_id`

openraft/src/docs/faq/faq.md

@@ -237,8 +237,6 @@ pub(crate) fn following_handler(&mut self) -> FollowingHandler<C> {
 ```
 
 
-[`single-term-leader`]:         `crate::docs::feature_flags#single_term_leader`
-
 [`Linearizable Read`]: `crate::docs::protocol::read`
 [`leader_id`]:         `crate::docs::data::leader_id`
 

openraft/src/docs/feature_flags/feature-flags.md

@@ -24,6 +24,9 @@ in storage and network, such as `Vote` or `AppendEntriesRequest`.
 
 ## feature-flag `single-term-leader`
 
+**This feature flag is removed**.
+User [`leader_id_std::LeaderId`] in [`RaftTypeConfig`] instead.
+
 Allows only one leader to be elected in each `term`.
 This is the standard raft policy, which increases election conflict rate
 but reduce `LogId` size(`(term, node_id, index)` to `(term, index)`).
@@ -72,3 +75,6 @@ let snapshot_data: <MyTypeConfig as RaftTypeConfig>::SnapshotData;
 Note that the type shortcuts are not stable and may be changed in the future.
 It is also a good idea to copy the type shortcuts to your own codebase if you
 want to use them.
+
+[`RaftTypeConfig`]: crate::RaftTypeConfig
+[`leader_id_std::LeaderId`]: crate::impls::leader_id_std::LeaderId

openraft/src/engine/command.rs

@@ -21,7 +21,7 @@ use crate::raft_state::IOId;
 use crate::replication::request::Replicate;
 use crate::replication::ReplicationSessionId;
 use crate::type_config::alias::OneshotSenderOf;
-use crate::vote::CommittedVote;
+use crate::vote::committed::CommittedVote;
 use crate::LogId;
 use crate::OptionalSend;
 use crate::RaftTypeConfig;

openraft/src/engine/engine_impl.rs

@@ -47,8 +47,8 @@ use crate::storage::SnapshotMeta;
 use crate::type_config::alias::ResponderOf;
 use crate::type_config::alias::SnapshotDataOf;
 use crate::type_config::TypeConfigExt;
-use crate::vote::raft_term::RaftTerm;
 use crate::vote::RaftLeaderId;
+use crate::vote::RaftTerm;
 use crate::LogId;
 use crate::LogIdOptionExt;
 use crate::Membership;

openraft/src/engine/handler/following_handler/mod.rs

@@ -16,7 +16,7 @@ use crate::error::RejectAppendEntries;
 use crate::raft_state::IOId;
 use crate::raft_state::LogStateReader;
 use crate::storage::Snapshot;
-use crate::vote::CommittedVote;
+use crate::vote::committed::CommittedVote;
 use crate::EffectiveMembership;
 use crate::LogId;
 use crate::LogIdOptionExt;

openraft/src/engine/testing.rs

@@ -37,7 +37,7 @@ where N: Node + Ord
     type Node = N;
     type Entry = crate::Entry<Self>;
     type Term = u64;
-    type LeaderId = crate::impls::LeaderId<Self>;
+    type LeaderId = crate::impls::leader_id_adv::LeaderId<Self>;
     type SnapshotData = Cursor<Vec<u8>>;
     type AsyncRuntime = TokioRuntime;
     type Responder = crate::impls::OneshotResponder<Self>;