From c13e14c8cde8e17a0d43b54951fa4a9876a4c494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Wed, 21 Feb 2024 16:31:06 +0800 Subject: [PATCH] Doc: Update link in doc to docs.rs The outdated online documentation at https://datafuselabs.github.io is no longer maintained due to being out-of-date and lacking version control. In this commit, we have updated the links that previously directed users to the old site, to now point to `docs.rs`, which naturally supports documentation versioning. And replace list with head for `feature-flags.md`, so that it's able to link to a certain feature-flag. --- Makefile | 1 + openraft/src/compat/compat07.rs | 16 +- openraft/src/compat/testing.rs | 5 +- openraft/src/core/raft_core.rs | 4 +- openraft/src/docs/feature_flags/Makefile | 6 + .../docs/feature_flags/feature-flags-toc.md | 10 + .../src/docs/feature_flags/feature-flags.md | 177 ++++++++++-------- openraft/src/docs/feature_flags/mod.rs | 4 + .../docs/getting_started/getting-started.md | 134 +++++++------ openraft/src/engine/engine_impl.rs | 5 +- openraft/src/network/factory.rs | 2 +- openraft/src/network/network.rs | 2 +- openraft/src/quorum/coherent_impl.rs | 2 +- openraft/src/raft/mod.rs | 19 +- .../src/raft_state/membership_state/mod.rs | 2 +- openraft/src/storage/helper.rs | 4 +- openraft/src/storage/mod.rs | 5 +- openraft/src/storage/v2.rs | 5 +- openraft/src/type_config.rs | 6 +- 19 files changed, 233 insertions(+), 176 deletions(-) create mode 100644 openraft/src/docs/feature_flags/Makefile create mode 100644 openraft/src/docs/feature_flags/feature-flags-toc.md diff --git a/Makefile b/Makefile index dbff136b3..fbc324882 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ fix: doc: make -C openraft/src/docs/faq + make -C openraft/src/docs/feature_flags RUSTDOCFLAGS="-D warnings" cargo doc --document-private-items --all --no-deps check_missing_doc: diff --git a/openraft/src/compat/compat07.rs b/openraft/src/compat/compat07.rs index 2ac6dd3b4..21a3feedf 100644 --- a/openraft/src/compat/compat07.rs +++ b/openraft/src/compat/compat07.rs @@ -1,9 +1,10 @@ //! This mod provides types that can be deserialized from data written by either v0.7 or //! the latest openraft. //! -//! This mod is enabled by feature flag `compat-07`. See: [feature-flag-compat-07](https://datafuselabs.github.io/openraft/feature-flags) -//! Ap application does not needs to enable this feature if it chooses to manually upgrade v0.7 -//! on-disk data. +//! This mod is enabled by feature flag `compat-07`. +//! See: [feature-flag-compat-07](`crate::docs::feature_flags#feature-flag-compat-07`). +//! Ap application does not needs to enable this feature if it chooses +//! to manually upgrade v0.7 on-disk data. //! //! In v0.7 compatible mode, openraft enables feature flags: //! - `serde`: it adds `serde` implementation to types such as `LogId`. @@ -15,10 +16,15 @@ //! //! An application that tries to upgrade from v0.7 can use types in this mod to replace the //! corresponding ones in a `RaftStorage` implementation, so that v0.7 data and v0.8 data can both -//! be read. [rocksstore-compat07](https://github.com/datafuselabs/openraft/tree/main/rocksstore-compat07) is an example using these type to implement an upgraded RaftStorage +//! be read. +//! [rocksstore-compat07](https://github.com/datafuselabs/openraft/tree/main/rocksstore-compat07) +//! is an example using these type to implement an upgraded RaftStorage //! //! This mod also provides a testing suite [`testing::Suite07`] to ensure old data will be correctly -//! read. An application should ensure its storage to pass test suite like [rocksstore-compat07/compatibility_test.rs](https://github.com/datafuselabs/openraft/blob/main/rocksstore-compat07/src/compatibility_test.rs) does: +//! read. +//! An application should ensure its storage +//! to pass test suite like [rocksstore-compat07/compatibility_test.rs](https://github.com/datafuselabs/openraft/blob/main/rocksstore-compat07/src/compatibility_test.rs) +//! does: //! ```ignore //! use openraft::compat; //! diff --git a/openraft/src/compat/testing.rs b/openraft/src/compat/testing.rs index efdbf5b49..394b448f8 100644 --- a/openraft/src/compat/testing.rs +++ b/openraft/src/compat/testing.rs @@ -1,6 +1,9 @@ //! This mod provides supporting utilities for compatibility testing. //! -//! An application that tries to be compatible with an older format data must ensure its `RaftStorage` implementation to pass test suite, just like [rocksstore-compat07/compatibility_test.rs](https://github.com/datafuselabs/openraft/blob/main/rocksstore-compat07/src/compatibility_test.rs) does +//! An application that tries to be compatible with an older format data +//! must ensure its `RaftStorage` implementation to pass test suite, +//! just like [rocksstore-compat07/compatibility_test.rs](https://github.com/datafuselabs/openraft/blob/main/rocksstore-compat07/src/compatibility_test.rs) +//! does use std::path::Path; diff --git a/openraft/src/core/raft_core.rs b/openraft/src/core/raft_core.rs index b46675965..c93d6592d 100644 --- a/openraft/src/core/raft_core.rs +++ b/openraft/src/core/raft_core.rs @@ -586,7 +586,9 @@ where /// Handle the admin command `initialize`. /// /// It is allowed to initialize only when `last_log_id.is_none()` and `vote==(0,0)`. - /// See: [Conditions for initialization](https://datafuselabs.github.io/openraft/cluster-formation.html#conditions-for-initialization) + /// See: [Conditions for initialization][precondition] + /// + /// [precondition]: crate::docs::cluster_control::cluster_formation#preconditions-for-initialization #[tracing::instrument(level = "debug", skip(self, tx))] pub(crate) fn handle_initialize( &mut self, diff --git a/openraft/src/docs/feature_flags/Makefile b/openraft/src/docs/feature_flags/Makefile new file mode 100644 index 000000000..633bb604c --- /dev/null +++ b/openraft/src/docs/feature_flags/Makefile @@ -0,0 +1,6 @@ +all: + # dependency: + # https://github.com/jonschlinkert/markdown-toc#cli + # brew install markdown-toc + markdown-toc feature-flags.md > feature-flags-toc.md + echo "" >> feature-flags-toc.md diff --git a/openraft/src/docs/feature_flags/feature-flags-toc.md b/openraft/src/docs/feature_flags/feature-flags-toc.md new file mode 100644 index 000000000..c36ce6bca --- /dev/null +++ b/openraft/src/docs/feature_flags/feature-flags-toc.md @@ -0,0 +1,10 @@ +- [feature-flag `bench`](#feature-flag-bench) +- [feature-flag `bt`](#feature-flag-bt) +- [feature-flag `compat-07`](#feature-flag-compat-07) +- [feature-flag `generic-snapshot-data`](#feature-flag-generic-snapshot-data) +- [feature-flag `loosen-follower-log-revert`](#feature-flag-loosen-follower-log-revert) +- [feature-flag `serde`](#feature-flag-serde) +- [feature-flag `single-term-leader`](#feature-flag-single-term-leader) +- [feature-flag `singlethreaded`](#feature-flag-singlethreaded) +- [feature-flag `storage-v2`](#feature-flag-storage-v2) +- [feature-flag `tracing-log`](#feature-flag-tracing-log) diff --git a/openraft/src/docs/feature_flags/feature-flags.md b/openraft/src/docs/feature_flags/feature-flags.md index a4eca5358..21aa533d4 100644 --- a/openraft/src/docs/feature_flags/feature-flags.md +++ b/openraft/src/docs/feature_flags/feature-flags.md @@ -1,89 +1,100 @@ -# Feature flags By default openraft enables no features. -- `bench`: Enables benchmarks in unittest. Benchmark in openraft depends on the unstable feature - `test` thus it can not be used with stable rust. In order to run the benchmark with stable - toolchain, the unstable features have to be enabled explicitly with environment variable - `RUSTC_BOOTSTRAP=1`. -

- -- `bt`: - attaches backtrace to generated errors. This feature works ONLY with nightly rust, because it requires unstable feature `error_generic_member_access`. -

- -- `loosen-follower-log-revert`: - Permit the follower's log to roll back to an earlier state without causing the leader to panic. - Although log state reversion is typically seen as a bug, enabling it can be useful for testing or other special scenarios. - For instance, in an even number nodes cluster, erasing a node's data and then rebooting it(log reverts to empty) will not result in data loss. - - **Do not use it unless you know what you are doing**. -

- -- `serde`: derives `serde::Serialize, serde::Deserialize` for type that are used - in storage and network, such as `Vote` or `AppendEntriesRequest`. -

- -- `single-term-leader`: allows only one leader to be elected in each `term`. - This is the standard raft policy, which increases election conflict rate - but reduce `LogId` size(`(term, node_id, index)` to `(term, index)`). - - Read more about how it is implemented in: - [`leader_id`](crate::docs::data::leader_id) - and [`vote`](crate::docs::data::vote). -

- -- `compat-07`: provides additional data types to build v0.7 compatible RaftStorage. - - ```toml - compat-07 = ["compat", "single-term-leader", "serde", "dep:or07", "compat-07-testing"] - compat-07-testing = ["dep:tempdir", "anyhow", "dep:serde_json"] - ``` -

- -- `storage-v2`: enables `RaftLogStorage` and `RaftStateMachine` as the v2 storage - This is a temporary feature flag, and will be removed in the future, when v2 storage is stable. - This feature disables `Adapter`, which is for v1 storage to be used as v2. - V2 storage separates log store and state machine store so that log IO and state machine IO can be parallelized naturally. -

- -- `singlethreaded`: removes `Send` and `Sync` bounds from `AppData`, `AppDataResponse`, `RaftEntry`, `SnapshotData` - and other types to force the asynchronous runtime to spawn any tasks in the current thread. - This is for any single-threaded application that never allows a raft instance to be shared among multiple threads. - This feature relies on the `async_fn_in_trait` language feature that is officially supported from Rust 1.75.0. - If the feature is enabled, affected asynchronous trait methods require `Send` bounds. - In order to use the feature, `AsyncRuntime::spawn` should invoke `tokio::task::spawn_local` or equivalents. -

- -- `generic-snapshot-data`: Enable this feature flag - to eliminate the `AsyncRead + AsyncWrite + AsyncSeek + Unpin` bound - from [`RaftTypeConfig::SnapshotData`](crate::RaftTypeConfig::SnapshotData) - Enabling this feature allows applications to use a custom snapshot data format and transport fragmentation, - diverging from the default implementation which typically relies on a single-file structure. - - By default, it is off. - This feature is introduced in 0.9.0 - - On the sending end (leader that sends snapshot to follower): - - - Without `generic-snapshot-data`: [`RaftNetwork::snapshot()`] - provides a default implementation that invokes the chunk-based API - [`RaftNetwork::install_snapshot()`] for transmit. - - - With `generic-snapshot-data` enabled: [`RaftNetwork::snapshot()`] - must be implemented to provide application customized snapshot transmission. - Application does not need to implement [`RaftNetwork::install_snapshot()`]. - - On the receiving end(follower): - - - `Raft::install_snapshot()` is available only when `generic-snapshot-data` is disabled. - - Refer to example `examples/raft-kv-memstore-generic-snapshot-data` with `generic-snapshot-data` enabled. -

- -- `tracing-log`: enables "log" feature in `tracing` crate, to let tracing events - emit log record. - See: [tracing doc: emitting-log-records](https://docs.rs/tracing/latest/tracing/#emitting-log-records) +## feature-flag `bench` + +Enables benchmarks in unittest. Benchmark in openraft depends on the unstable feature +`test` thus it can not be used with stable rust. In order to run the benchmark with stable +toolchain, the unstable features have to be enabled explicitly with environment variable +`RUSTC_BOOTSTRAP=1`. + +## feature-flag `bt` + +attaches backtrace to generated errors. +This feature works ONLY with nightly rust, because it requires unstable feature `error_generic_member_access`. + +## feature-flag `compat-07` + +Provides additional data types to build v0.7 compatible RaftStorage. + + ```toml + compat-07 = ["compat", "single-term-leader", "serde", "dep:or07", "compat-07-testing"] + compat-07-testing = ["dep:tempdir", "anyhow", "dep:serde_json"] + ``` + +## feature-flag `generic-snapshot-data` + +Enable this feature flag +to eliminate the `AsyncRead + AsyncWrite + AsyncSeek + Unpin` bound +from [`RaftTypeConfig::SnapshotData`](crate::RaftTypeConfig::SnapshotData) +Enabling this feature allows applications to use a custom snapshot data format and transport fragmentation, +diverging from the default implementation which typically relies on a single-file structure. + +By default, it is off. +This feature is introduced in 0.9.0 + +On the sending end (leader that sends snapshot to follower): + +- Without `generic-snapshot-data`: [`RaftNetwork::snapshot()`] + provides a default implementation that invokes the chunk-based API + [`RaftNetwork::install_snapshot()`] for transmit. + +- With `generic-snapshot-data` enabled: [`RaftNetwork::snapshot()`] + must be implemented to provide application customized snapshot transmission. + Application does not need to implement [`RaftNetwork::install_snapshot()`]. + +On the receiving end(follower): + +- `Raft::install_snapshot()` is available only when `generic-snapshot-data` is disabled. + +Refer to example `examples/raft-kv-memstore-generic-snapshot-data` with `generic-snapshot-data` enabled. + +## feature-flag `loosen-follower-log-revert` + +Permit the follower's log to roll back to an earlier state without causing the leader to panic. +Although log state reversion is typically seen as a bug, enabling it can be useful for testing or other special scenarios. +For instance, in an even number nodes cluster, +erasing a node's data and then rebooting it(log reverts to empty) will not result in data loss. + +**Do not use it unless you know what you are doing**. + +## feature-flag `serde` + +Derives `serde::Serialize, serde::Deserialize` for type that are used +in storage and network, such as `Vote` or `AppendEntriesRequest`. + +## feature-flag `single-term-leader` + +Allows only one leader to be elected in each `term`. +This is the standard raft policy, which increases election conflict rate +but reduce `LogId` size(`(term, node_id, index)` to `(term, index)`). + +Read more about how it is implemented in: +[`leader_id`](crate::docs::data::leader_id) +and [`vote`](crate::docs::data::vote). + +## feature-flag `singlethreaded` + +Removes `Send` and `Sync` bounds from `AppData`, `AppDataResponse`, `RaftEntry`, `SnapshotData` +and other types to force the asynchronous runtime to spawn any tasks in the current thread. +This is for any single-threaded application that never allows a raft instance to be shared among multiple threads. +This feature relies on the `async_fn_in_trait` language feature that is officially supported from Rust 1.75.0. +If the feature is enabled, affected asynchronous trait methods require `Send` bounds. +In order to use the feature, `AsyncRuntime::spawn` should invoke `tokio::task::spawn_local` or equivalents. + +## feature-flag `storage-v2` + +Enables `RaftLogStorage` and `RaftStateMachine` as the v2 storage +This is a temporary feature flag, and will be removed in the future, when v2 storage is stable. +This feature disables `Adapter`, which is for v1 storage to be used as v2. +V2 storage separates log store and state machine store so that log IO and state machine IO can be parallelized naturally. + +## feature-flag `tracing-log` + +Enables "log" feature in `tracing` crate, to let tracing events +emit log record. +See: [tracing doc: emitting-log-records](https://docs.rs/tracing/latest/tracing/#emitting-log-records) + [`RaftNetwork::snapshot()`]: crate::network::RaftNetwork::snapshot [`RaftNetwork::install_snapshot()`]: crate::network::RaftNetwork::install_snapshot \ No newline at end of file diff --git a/openraft/src/docs/feature_flags/mod.rs b/openraft/src/docs/feature_flags/mod.rs index 3f6b24a47..5e09ab0a0 100644 --- a/openraft/src/docs/feature_flags/mod.rs +++ b/openraft/src/docs/feature_flags/mod.rs @@ -1 +1,5 @@ +//! # Feature flags + +#![doc = include_str!("feature-flags-toc.md")] +// #![doc = include_str!("feature-flags.md")] diff --git a/openraft/src/docs/getting_started/getting-started.md b/openraft/src/docs/getting_started/getting-started.md index 53874a306..1d519d6bb 100644 --- a/openraft/src/docs/getting_started/getting-started.md +++ b/openraft/src/docs/getting_started/getting-started.md @@ -42,7 +42,7 @@ pub struct Response(Result, ClientError>); ``` These two types are entirely application-specific and are mainly related to the -state machine implementation in [`RaftStorage`]. +state machine implementation in [`RaftStateMachine`]. ## 2. Define types for the application @@ -71,50 +71,56 @@ pub struct Raft {} Openraft provides default implementations for `Node` ([`EmptyNode`] and [`BasicNode`]) and log `Entry` ([`Entry`]). You can use these implementations directly or define your own custom types. -A [`RaftTypeConfig`] is also used by other components such as [`RaftStorage`], [`RaftNetworkFactory`] and [`RaftNetwork`]. +A [`RaftTypeConfig`] is also used by other components such as [`RaftLogStorage`], [`RaftStateMachine`], +[`RaftNetworkFactory`] and [`RaftNetwork`]. -## 3. Implement [`RaftStorage`] +## 3. Implement [`RaftLogStorage`] and [`RaftStateMachine`] -The trait [`RaftStorage`] defines how data is stored and consumed. +The trait [`RaftLogStorage`] defines how log data is stored and consumed. It could be a wrapper for a local key-value store like [RocksDB](https://docs.rs/rocksdb/latest/rocksdb/). +The trait [`RaftStateMachine`] defines how log is interpreted. Usually it is an in memory state machine with or without on-disk data backed. + There is a good example, [`Mem KV Store`](https://github.com/datafuselabs/openraft/blob/main/examples/raft-kv-memstore/src/store/mod.rs), -that demonstrates what should be done when a method is called. The list of [`RaftStorage`] methods is shown below. +that demonstrates what should be done when a method is called. The storage methods are listed as the below. Follow the link to method document to see the details. -| Kind | [`RaftStorage`] method | Return value | Description | -|------------|----------------------------------|------------------------------|---------------------------------------| -| Read log: | [`get_log_reader()`] | impl [`RaftLogReader`] | get a read-only log reader | -| | | ↳ [`try_get_log_entries()`] | get a range of logs | -| | [`get_log_state()`] | [`LogState`] | get first/last log id | -| Write log: | [`append_to_log()`] | () | append logs | -| Write log: | [`delete_conflict_logs_since()`] | () | delete logs `[index, +oo)` | -| Write log: | [`purge_logs_upto()`] | () | purge logs `(-oo, index]` | -| Vote: | [`save_vote()`] | () | save vote | -| Vote: | [`read_vote()`] | [`Vote`] | read vote | -| SM: | [`last_applied_state()`] | [`LogId`], [`Membership`] | get last applied log id, membership | -| SM: | [`apply_to_state_machine()`] | Vec of [`AppDataResponse`] | apply logs to state machine | -| Snapshot: | [`begin_receiving_snapshot()`] | `SnapshotData` | begin to install snapshot | -| Snapshot: | [`install_snapshot()`] | () | install snapshot | -| Snapshot: | [`get_current_snapshot()`] | [`Snapshot`] | get current snapshot | -| Snapshot: | [`get_snapshot_builder()`] | impl [`RaftSnapshotBuilder`] | get a snapshot builder | -| | | ↳ [`build_snapshot()`] | build a snapshot from state machine | +| Kind | [`RaftLogStorage`] method | Return value | Description | +|------------|---------------------------|------------------------------|---------------------------------------| +| Read log: | [`get_log_reader()`] | impl [`RaftLogReader`] | get a read-only log reader | +| | | ↳ [`try_get_log_entries()`] | get a range of logs | +| | [`get_log_state()`] | [`LogState`] | get first/last log id | +| Write log: | [`append()`] | () | append logs | +| Write log: | [`truncate()`] | () | delete logs `[index, +oo)` | +| Write log: | [`purge()`] | () | purge logs `(-oo, index]` | +| Vote: | [`save_vote()`] | () | save vote | +| Vote: | [`read_vote()`] | [`Vote`] | read vote | + +| Kind | [`RaftStateMachine`] method | Return value | Description | +|------------|--------------------------------|------------------------------|---------------------------------------| +| SM: | [`applied_state()`] | [`LogId`], [`Membership`] | get last applied log id, membership | +| SM: | [`apply()`] | Vec of [`AppDataResponse`] | apply logs to state machine | +| Snapshot: | [`begin_receiving_snapshot()`] | `SnapshotData` | begin to install snapshot | +| Snapshot: | [`install_snapshot()`] | () | install snapshot | +| Snapshot: | [`get_current_snapshot()`] | [`Snapshot`] | get current snapshot | +| Snapshot: | [`get_snapshot_builder()`] | impl [`RaftSnapshotBuilder`] | get a snapshot builder | +| | | ↳ [`build_snapshot()`] | build a snapshot from state machine | Most of the APIs are quite straightforward, except two indirect APIs: - Read logs: - [`RaftStorage`] defines a method [`get_log_reader()`] to get log reader [`RaftLogReader`] : + [`RaftLogStorage`] defines a method [`get_log_reader()`] to get log reader [`RaftLogReader`] : ```ignore - trait RaftStorage { + trait RaftLogStorage { type LogReader: RaftLogReader; async fn get_log_reader(&mut self) -> Self::LogReader; } ``` - [`RaftLogReader`] defines the APIs to read logs, and is an also super trait of [`RaftStorage`] : + [`RaftLogReader`] defines the APIs to read logs, and is an also super trait of [`RaftLogStorage`] : - [`try_get_log_entries()`] get log entries in a range; ```ignore @@ -123,18 +129,21 @@ Most of the APIs are quite straightforward, except two indirect APIs: } ``` - And [`RaftStorage::get_log_state()`][`get_log_state()`] get latest log state from the storage; + And [`RaftLogStorage::get_log_state()`][`get_log_state()`] get latest log state from the storage; - Build a snapshot from the local state machine needs to be done in two steps: - - [`RaftStorage::get_snapshot_builder() -> Self::SnapshotBuilder`][`get_snapshot_builder()`], + - [`RaftLogStorage::get_snapshot_builder() -> Self::SnapshotBuilder`][`get_snapshot_builder()`], - [`RaftSnapshotBuilder::build_snapshot() -> Result`][`build_snapshot()`], -### Ensure the implementation of RaftStorage is correct +### Ensure the storage implementation is correct -There is a [Test suite for RaftStorage][`Suite`] available in Openraft. If your implementation passes the tests, Openraft should work well with it. To test your implementation, you have two options: +There is a [Test suite for RaftLogStorage and RaftStateMachine][`Suite`] available in Openraft. +If your implementation passes the tests, Openraft should work well with it. +To test your implementation, you have two options: -1. Run `Suite::test_all()` with an `async fn()` that creates a new [`RaftStorage`], as shown in the [`MemStore` test](https://github.com/datafuselabs/openraft/blob/main/memstore/src/test.rs): +1. Run `Suite::test_all()` with an `async fn()` that creates a new pair of [`RaftLogStorage`] and [`RaftStateMachine`], + as shown in the [`MemStore` test](https://github.com/datafuselabs/openraft/blob/main/memstore/src/test.rs): ```ignore #[test] @@ -143,14 +152,15 @@ There is a [Test suite for RaftStorage][`Suite`] available in Openraft. If your } ``` -2. Alternatively, run `Suite::test_all()` with a [`StoreBuilder`] implementation, as shown in the [`RocksStore` test](https://github.com/datafuselabs/openraft/blob/main/rocksstore/src/test.rs). +2. Alternatively, run `Suite::test_all()` with a [`StoreBuilder`] implementation, + as shown in the [`RocksStore` test](https://github.com/datafuselabs/openraft/blob/main/rocksstore/src/test.rs). By following either of these approaches, you can ensure that your custom storage implementation can work correctly in a distributed system. ### An implementation has to guarantee data durability. -The caller always assumes a completed write is persistent. +The caller always assumes a completed writing is persistent. The raft correctness highly depends on a reliable store. @@ -161,9 +171,9 @@ The trait [`RaftNetwork`] defines the data transmission requirements. ```ignore pub trait RaftNetwork: Send + Sync + 'static { - async fn send_append_entries(&mut self, rpc: AppendEntriesRequest) -> Result<...>; - async fn send_install_snapshot(&mut self, rpc: InstallSnapshotRequest) -> Result<...>; - async fn send_vote(&mut self, rpc: VoteRequest) -> Result<...>; + async fn vote(&mut self, rpc: VoteRequest) -> Result<...>; + async fn append_entries(&mut self, rpc: AppendEntriesRequest) -> Result<...>; + async fn snapshot(&mut self, vote: Vote, snapshot: Snapshot) -> Result<...>; } ``` @@ -174,11 +184,11 @@ and receiving messages between Raft nodes. Here is the list of methods that need to be implemented for the [`RaftNetwork`] trait: -| [`RaftNetwork`] method | forward request | to target | -|------------------------------|----------------------------|------------------------------------------| -| [`send_append_entries()`] | [`AppendEntriesRequest`] | remote node [`Raft::append_entries()`] | -| [`send_install_snapshot()`] | [`InstallSnapshotRequest`] | remote node [`Raft::install_snapshot()`] | -| [`send_vote()`] | [`VoteRequest`] | remote node [`Raft::vote()`] | +| [`RaftNetwork`] method | forward request | to target | +|------------------------|--------------------------|---------------------------------------------------| +| [`append_entries()`] | [`AppendEntriesRequest`] | remote node [`Raft::append_entries()`] | +| [`snapshot()`] | [`Snapshot`] | remote node [`Raft::install_complete_snapshot()`] | +| [`vote()`] | [`VoteRequest`] | remote node [`Raft::vote()`] | [Mem KV Network](https://github.com/datafuselabs/openraft/blob/main/examples/raft-kv-memstore/src/network/raft_network_impl.rs) demonstrates how to forward messages to other Raft nodes using [`reqwest`](https://docs.rs/reqwest/latest/reqwest/) as network transport layer. @@ -341,7 +351,7 @@ Additionally, two test scripts for setting up a cluster are available: [`Raft`]: `crate::Raft` [`Raft::append_entries()`]: `crate::Raft::append_entries` [`Raft::vote()`]: `crate::Raft::vote` -[`Raft::install_snapshot()`]: `crate::Raft::install_snapshot` +[`Raft::install_complete_snapshot()`]: `crate::Raft::install_complete_snapshot` [`AppendEntriesRequest`]: `crate::raft::AppendEntriesRequest` [`VoteRequest`]: `crate::raft::VoteRequest` @@ -362,29 +372,33 @@ Additionally, two test scripts for setting up a cluster are available: [`RaftLogReader`]: `crate::storage::RaftLogReader` [`try_get_log_entries()`]: `crate::storage::RaftLogReader::try_get_log_entries` -[`RaftStorage`]: `crate::storage::RaftStorage` -[`get_log_state()`]: `crate::storage::RaftStorage::get_log_state` -[`RaftStorage::LogReader`]: `crate::storage::RaftStorage::LogReader` -[`RaftStorage::SnapshotBuilder`]: `crate::storage::RaftStorage::SnapshotBuilder` -[`get_log_reader()`]: `crate::storage::RaftStorage::get_log_reader` -[`save_vote()`]: `crate::storage::RaftStorage::save_vote` -[`read_vote()`]: `crate::storage::RaftStorage::read_vote` -[`append_to_log()`]: `crate::storage::RaftStorage::append_to_log` -[`delete_conflict_logs_since()`]: `crate::storage::RaftStorage::delete_conflict_logs_since` -[`purge_logs_upto()`]: `crate::storage::RaftStorage::purge_logs_upto` -[`last_applied_state()`]: `crate::storage::RaftStorage::last_applied_state` -[`apply_to_state_machine()`]: `crate::storage::RaftStorage::apply_to_state_machine` -[`get_current_snapshot()`]: `crate::storage::RaftStorage::get_current_snapshot` -[`begin_receiving_snapshot()`]: `crate::storage::RaftStorage::begin_receiving_snapshot` -[`install_snapshot()`]: `crate::storage::RaftStorage::install_snapshot` -[`get_snapshot_builder()`]: `crate::storage::RaftStorage::get_snapshot_builder` + +[`RaftLogStorage::SnapshotBuilder`]: `crate::storage::RaftLogStorage::SnapshotBuilder` + +[`RaftLogStorage`]: `crate::storage::RaftLogStorage` +[`RaftLogStorage::LogReader`]: `crate::storage::RaftLogStorage::LogReader` +[`append()`]: `crate::storage::RaftLogStorage::append` +[`truncate()`]: `crate::storage::RaftLogStorage::truncate` +[`purge()`]: `crate::storage::RaftLogStorage::purge` +[`save_vote()`]: `crate::storage::RaftLogStorage::save_vote` +[`read_vote()`]: `crate::storage::RaftLogStorage::read_vote` +[`get_log_state()`]: `crate::storage::RaftLogStorage::get_log_state` +[`get_log_reader()`]: `crate::storage::RaftLogStorage::get_log_reader` + +[`RaftStateMachine`]: `crate::storage::RaftStateMachine` +[`applied_state()`]: `crate::storage::RaftStateMachine::applied_state` +[`apply()`]: `crate::storage::RaftStateMachine::apply` +[`get_current_snapshot()`]: `crate::storage::RaftStateMachine::get_current_snapshot` +[`begin_receiving_snapshot()`]: `crate::storage::RaftStateMachine::begin_receiving_snapshot` +[`install_snapshot()`]: `crate::storage::RaftStateMachine::install_snapshot` +[`get_snapshot_builder()`]: `crate::storage::RaftStateMachine::get_snapshot_builder` [`RaftNetworkFactory`]: `crate::network::RaftNetworkFactory` [`RaftNetworkFactory::new_client()`]: `crate::network::RaftNetworkFactory::new_client` [`RaftNetwork`]: `crate::network::RaftNetwork` -[`send_append_entries()`]: `crate::RaftNetwork::send_append_entries` -[`send_vote()`]: `crate::RaftNetwork::send_vote` -[`send_install_snapshot()`]: `crate::RaftNetwork::send_install_snapshot` +[`append_entries()`]: `crate::RaftNetwork::append_entries` +[`vote()`]: `crate::RaftNetwork::vote` +[`snapshot()`]: `crate::RaftNetwork::snapshot` [`RaftSnapshotBuilder`]: `crate::storage::RaftSnapshotBuilder` diff --git a/openraft/src/engine/engine_impl.rs b/openraft/src/engine/engine_impl.rs index 29d0702c1..b5db75f52 100644 --- a/openraft/src/engine/engine_impl.rs +++ b/openraft/src/engine/engine_impl.rs @@ -143,11 +143,14 @@ where C: RaftTypeConfig /// /// - The first log has to be membership config log. /// - The node has to contain no logs at all and the vote is the minimal value. See: [Conditions - /// for initialization](https://datafuselabs.github.io/openraft/cluster-formation.html#conditions-for-initialization) + /// for initialization][precondition]. + /// /// /// Appending the very first log is slightly different from appending log by a leader or /// follower. This step is not confined by the consensus protocol and has to be dealt with /// differently. + /// + /// [precondition]: crate::docs::cluster_control::cluster_formation#preconditions-for-initialization #[tracing::instrument(level = "debug", skip_all)] pub(crate) fn initialize(&mut self, mut entry: C::Entry) -> Result<(), InitializeError> { self.check_initialize()?; diff --git a/openraft/src/network/factory.rs b/openraft/src/network/factory.rs index 29ef48b2a..e781ecdf4 100644 --- a/openraft/src/network/factory.rs +++ b/openraft/src/network/factory.rs @@ -8,7 +8,7 @@ use crate::RaftTypeConfig; /// A trait defining the interface for a Raft network factory to create connections between cluster /// members. /// -/// See the [network chapter of the guide](https://datafuselabs.github.io/openraft/getting-started.html#3-impl-raftnetwork) +/// See the [network chapter of the guide](crate::docs::getting_started#4-implement-raftnetwork) /// for details and discussion on this trait and how to implement it. /// /// Typically, the network implementation as such will be hidden behind a `Box` or `Arc` and diff --git a/openraft/src/network/network.rs b/openraft/src/network/network.rs index de3dd5f3a..c019c5e8e 100644 --- a/openraft/src/network/network.rs +++ b/openraft/src/network/network.rs @@ -26,7 +26,7 @@ use crate::Vote; /// A trait defining the interface for a Raft network between cluster members. /// -/// See the [network chapter of the guide](https://datafuselabs.github.io/openraft/getting-started.html#3-impl-raftnetwork) +/// See the [network chapter of the guide](crate::docs::getting_started#4-implement-raftnetwork) /// for details and discussion on this trait and how to implement it. /// /// A single network instance is used to connect to a single target node. The network instance is diff --git a/openraft/src/quorum/coherent_impl.rs b/openraft/src/quorum/coherent_impl.rs index 0084a9860..3964aacb9 100644 --- a/openraft/src/quorum/coherent_impl.rs +++ b/openraft/src/quorum/coherent_impl.rs @@ -11,7 +11,7 @@ where /// Check if two `joint` are coherent. /// /// Read more about: - /// [safe-membership-change](https://datafuselabs.github.io/openraft/dynamic-membership.html#the-safe-to-relation) + /// [Extended membership change](crate::docs::data::extended_membership) fn is_coherent_with(&self, other: &Joint>) -> bool { for a in self.children() { for b in other.children() { diff --git a/openraft/src/raft/mod.rs b/openraft/src/raft/mod.rs index 5972bdf55..c1d403239 100644 --- a/openraft/src/raft/mod.rs +++ b/openraft/src/raft/mod.rs @@ -123,19 +123,16 @@ macro_rules! declare_raft_types { /// For more information on the Raft protocol, see /// [the specification here](https://raft.github.io/raft.pdf) (**pdf warning**). /// -/// For details and discussion on this API, see the -/// [Raft API](https://datafuselabs.github.io/openraft/raft.html) section of the guide. +/// ### Clone /// -/// ### clone -/// This type implements `Clone`, and should be cloned liberally. The clone itself is very cheap -/// and helps to facilitate use with async workflows. +/// This type implements `Clone`, and cloning itself is very cheap and helps to facilitate use with +/// async workflows. /// -/// ### shutting down -/// If any of the interfaces returns a `RaftError::ShuttingDown`, this indicates that the Raft node -/// is shutting down (potentially for data safety reasons due to a storage error), and the -/// `shutdown` method should be called on this type to await the shutdown of the node. If the parent -/// application needs to shutdown the Raft node for any reason, calling `shutdown` will do the -/// trick. +/// ### Shutting down +/// +/// If any of the interfaces returns a `RaftError::Fatal`, this indicates that the Raft node +/// is shutting down. If the parent application needs to shutdown the Raft node for any reason, +/// calling `shutdown` will do the trick. #[derive(Clone)] pub struct Raft where C: RaftTypeConfig diff --git a/openraft/src/raft_state/membership_state/mod.rs b/openraft/src/raft_state/membership_state/mod.rs index 835977aa5..8f6faad03 100644 --- a/openraft/src/raft_state/membership_state/mod.rs +++ b/openraft/src/raft_state/membership_state/mod.rs @@ -170,7 +170,7 @@ where /// /// If the effective membership is from a conflicting log, /// the membership state has to revert to the last committed membership config. - /// See: [Effective-membership](https://datafuselabs.github.io/openraft/effective-membership.html) + /// See: [Effective-membership](crate::docs::data::effective_membership) /// /// ```text /// committed_membership, ... since, ... effective_membership // log diff --git a/openraft/src/storage/helper.rs b/openraft/src/storage/helper.rs index 6c43b0280..6b60b5bce 100644 --- a/openraft/src/storage/helper.rs +++ b/openraft/src/storage/helper.rs @@ -157,7 +157,9 @@ where Ok(RaftState { committed: last_applied, // The initial value for `vote` is the minimal possible value. - // See: [Conditions for initialization](https://datafuselabs.github.io/openraft/cluster-formation.html#conditions-for-initialization) + // See: [Conditions for initialization][precondition] + // + // [precondition]: crate::docs::cluster_control::cluster_formation#preconditions-for-initialization vote: UTime::new(now, vote), purged_next: last_purged_log_id.next_index(), log_ids, diff --git a/openraft/src/storage/mod.rs b/openraft/src/storage/mod.rs index 834d8568a..1f0c2e1f8 100644 --- a/openraft/src/storage/mod.rs +++ b/openraft/src/storage/mod.rs @@ -359,10 +359,9 @@ where C: RaftTypeConfig /// /// Raft will use this handle to receive snapshot data. /// - /// ### implementation guide + /// See the [storage chapter of the guide][sto] for details on log compaction / snapshotting. /// - /// See the [storage chapter of the guide](https://datafuselabs.github.io/openraft/storage.html) - /// for details on log compaction / snapshotting. + /// [sto]: crate::docs::getting_started#3-implement-raftlogstorage-and-raftstatemachine async fn begin_receiving_snapshot(&mut self) -> Result, StorageError>; /// Install a snapshot which has finished streaming from the leader. diff --git a/openraft/src/storage/v2.rs b/openraft/src/storage/v2.rs index dff25fc37..77cbf28fb 100644 --- a/openraft/src/storage/v2.rs +++ b/openraft/src/storage/v2.rs @@ -212,10 +212,9 @@ where C: RaftTypeConfig /// /// Openraft will use this handle to receive snapshot data. /// - /// ### implementation guide + /// See the [storage chapter of the guide][sto] for details on log compaction / snapshotting. /// - /// See the [storage chapter of the guide](https://datafuselabs.github.io/openraft/storage.html) - /// for details on snapshot streaming. + /// [sto]: crate::docs::getting_started#3-implement-raftlogstorage-and-raftstatemachine async fn begin_receiving_snapshot(&mut self) -> Result, StorageError>; /// Install a snapshot which has finished streaming from the leader. diff --git a/openraft/src/type_config.rs b/openraft/src/type_config.rs index 7271d4662..023c37cad 100644 --- a/openraft/src/type_config.rs +++ b/openraft/src/type_config.rs @@ -54,11 +54,11 @@ pub trait RaftTypeConfig: /// Raft log entry, which can be built from an AppData. type Entry: RaftEntry + FromAppData; - // TODO: fix the doc address /// Snapshot data for exposing a snapshot for reading & writing. /// - /// See the [storage chapter of the guide](https://datafuselabs.github.io/openraft/getting-started.html#implement-raftstorage) - /// for details on where and how this is used. + /// See the [storage chapter of the guide][sto] for details on log compaction / snapshotting. + /// + /// [sto]: crate::docs::getting_started#3-implement-raftlogstorage-and-raftstatemachine #[cfg(not(feature = "generic-snapshot-data"))] type SnapshotData: tokio::io::AsyncRead + tokio::io::AsyncWrite