From 5a4b1c6eca4e8e01522fb8075cfcd9f8bcee5a0f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 13 Nov 2025 18:38:31 +0800 Subject: [PATCH 01/10] docs: Add RFC for iceberg-kernel Signed-off-by: Xuanwo --- docs/rfcs/0001_kernel.md | 178 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 docs/rfcs/0001_kernel.md diff --git a/docs/rfcs/0001_kernel.md b/docs/rfcs/0001_kernel.md new file mode 100644 index 0000000000..a1bb7b103a --- /dev/null +++ b/docs/rfcs/0001_kernel.md @@ -0,0 +1,178 @@ +# RFC: Extract `iceberg-kernel` for Pluggable Execution Layers + +## Background + +Issue #1819 proposes decoupling the protocol/metadata/plan logic that currently lives inside the `iceberg` crate so that it can serve as a reusable “kernel,” similar to the approach taken by delta-kernel-rs. Today the `iceberg` crate simultaneously exposes the public trait surface and the default engine (Tokio runtime, opendal-backed FileIO, Arrow readers, etc.). This tight coupling makes it difficult for downstream projects to embed Iceberg metadata while providing their own storage, runtime, or execution stack. + +## Goals and Scope + +- **Full read & write coverage**: the kernel must contain every protocol component required for both scan planning and transactional writes (append, rewrite, commit, etc.). +- **No default runtime dependency**: the kernel defines a `Runtime` trait instead of depending on Tokio or Smol. +- **No default storage dependency**: the kernel defines `FileIO` traits only; concrete implementations (for example `iceberg-fileio-opendal`) live in dedicated crates. +- **Stable facade for existing users**: the top-level `iceberg` crate continues to expose the familiar API by re-exporting the kernel plus a default engine feature. + +Out of scope: changes to the Iceberg table specification or rewriting catalog adapters. + +## Architecture Overview + +### Workspace Layout + +``` +crates/ + kernel/ # new: pure protocols & planning logic + spec/ expr/ catalog/ table/ transaction/ scan/ runtime_api + io/traits.rs # FileIO traits (no opendal) + fileio/ + opendal/ # e.g. `iceberg-fileio-opendal` + fs/ # other FileIO implementations + runtime/ + tokio/ # e.g. `iceberg-runtime-tokio` + smol/ + iceberg/ # facade re-exporting kernel + default engine + catalog/* # depend on kernel (+ chosen FileIO/Runtime crates) + integrations/* # e.g. datafusion using facade or composing crates +``` + +### Trait Surfaces + +#### FileIO + +```rust +pub struct FileMetadata { + pub size: u64, + ... +} + +pub type FileReader = Box; + +#[async_trait::async_trait] +pub trait FileRead: Send + Sync + 'static { + async fn read(&self, range: Range) -> Result; +} + +pub type FileWriter = Box; + +#[async_trait::async_trait] +pub trait FileWrite: Send + Unpin + 'static { + async fn write(&mut self, bs: Bytes) -> Result<()>; + async fn close(&mut self) -> Result; +} + +pub type StorageFactory = fn(attrs: HashMap -> Result>); + +#[async_trait::async_trait] +pub trait Storage: Clone + Send + Sync { + async fn reader(&self, path: &str) -> Result; + async fn writer(&self, path: &str) -> Result; + async fn delete(&self, path: &str) -> Result<()>; + async fn exists(&self, path: &str) -> Result; + + ... +} + +pub struct FileIO { + registry: DashMap, +} + +impl FileIO { + fn register(scheme: &str, factory: StorageFactory); + + async fn read(path: &str) -> Result; + async fn reader(path: &str) -> Result; + async fn write(path: &str, bs: Bytes) -> Result; + async fn writer(path: &str) -> Result; + + async fn delete(&self, path: &str) -> Result<()>; + ... +} +``` + +- The kernel only defines the trait and error types. +- `iceberg-fileio-opendal` (new crate) ships an opendal-based implementation; other backends can publish their own crates. + +#### Runtime + +```rust +pub trait Runtime: Send + Sync + 'static { + type JoinHandle: Future + Send + 'static; + + fn spawn(&self, fut: F) -> Self::JoinHandle + where + F: Future + Send + 'static, + T: Send + 'static; + + fn spawn_blocking(&self, f: F) -> Self::JoinHandle + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static; + + fn sleep(&self, dur: Duration) -> Pin + Send>>; +} +``` + +- `TableScan` planning, metadata refresh, and `Transaction::commit` depend only on this trait. +- Crates such as `iceberg-runtime-tokio` provide concrete schedulers; consumers pick whichever runtime crate fits their stack. + +#### Catalog / Table / Transaction + +- The `Catalog` trait moves into the kernel and returns lightweight `TableHandle` objects (metadata + FileIO + Runtime). +- `TableHandle` no longer embeds Arrow helpers; Arrow-specific logic lives in engine crates. +- Transactions and their actions remain in the kernel, but rely on injected `Runtime` for retries/backoff. + +#### Scan / Planner + +- The kernel produces pure `TableScanPlan` descriptions (manifests, data-files, predicates, task graph). +- Engines provide executors (e.g., `ArrowExecutor`) that transform plans into record batches or other runtime-specific artifacts. + +### Facade Behavior + +- The top-level `iceberg` crate becomes a facade (`pub use iceberg_kernel::*`) that enables a *composition* of default crates (e.g. `iceberg-runtime-tokio`, `iceberg-fileio-opendal`, and a reference executor) behind feature flags. +- Existing convenience APIs (`Table::scan().to_arrow()`, `MemoryCatalog`, etc.) stay available but internally assemble the kernel with those default building blocks. + +## Migration Plan + +1. **Phase 1 – Create the kernel crate** + - Add `crates/kernel` and move `spec`, `expr`, `catalog`, `table`, `transaction`, `scan`, and supporting modules. + - Introduce temporary shim modules in the facade so existing imports keep working (mark them deprecated). + +2. **Phase 2 – Abstract runtime & IO** + - Define the `Runtime` and `FileIO` traits inside the kernel. + - Remove direct `tokio`/`opendal` dependencies from kernel modules. + - Introduce standalone crates (`iceberg-runtime-tokio`, `iceberg-fileio-opendal`, etc.) that implement the new traits. + +3. **Phase 3 – Detach Arrow/execution** + - Move `arrow` helpers and `ArrowReaderBuilder` into a reference executor crate (e.g. `iceberg-engine-arrow`). + - Update the DataFusion integration to depend on the facade or directly compose kernel + runtime + fileio + executor crates. + +4. **Phase 4 – Catalog & integration updates** + - Point catalog crates and other integrations to the kernel interfaces; depend on specific FileIO/Runtime crates only when required. + - Keep `iceberg-catalog-loader` kernel-only so users can inject their preferred combinations. + +5. **Phase 5 – Release & documentation** + - Finish the split within the 0.y.z series, provide an upgrade guide, and add kernel acceptance tests to guarantee trait stability. + +## Compatibility + +- Users who stick with the `iceberg` facade keep their existing API surface; the facade simply composes kernel + default runtime + default FileIO + reference executor under the hood. +- Advanced integrators can depend solely on `iceberg-kernel` and mix in whichever `FileIO`, `Runtime`, and executor crates they need (or author their own). +- CI keeps running the current integration tests and adds kernel-specific acceptance suites. + +## Risks and Mitigations + +| Risk | Description | Mitigation | +| ---- | ----------- | ---------- | +| Trait churn | Updating catalog/scan traits could break downstream crates | Maintain shim modules, use `#[deprecated]` windows, and document migration steps | +| Generic complexity | New traits may introduce complicated type bounds | Prefer `Arc` and `BoxFuture` to keep signatures manageable | +| Documentation gap | Users may not know which engine to pick | Publish new docs, diagrams, and “custom engine” tutorials alongside the split | + +## Open Questions + +1. Should the kernel expose any Arrow helpers, or should every Arrow-specific function live exclusively in engine crates? +2. Do we need a `ScanExecutor` trait inside the kernel for non-Arrow consumers? + +## Conclusion + +Extracting `iceberg-kernel` plus a pluggable engine layer lets the project: + +- Offer a lightweight, embeddable implementation of the Iceberg protocol, +- Enable external engines (DataFusion, Spark Connect, custom services) to reuse Iceberg metadata without inheriting specific runtime/storage dependencies, From 891fd75af080c272542ecc9b165c754c1dcfc3ca Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 13 Nov 2025 18:45:32 +0800 Subject: [PATCH 02/10] Add license: Signed-off-by: Xuanwo --- docs/rfcs/0001_kernel.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/rfcs/0001_kernel.md b/docs/rfcs/0001_kernel.md index a1bb7b103a..cca233908e 100644 --- a/docs/rfcs/0001_kernel.md +++ b/docs/rfcs/0001_kernel.md @@ -1,3 +1,22 @@ + + # RFC: Extract `iceberg-kernel` for Pluggable Execution Layers ## Background From 5cb95ad0dc211e8de21f1ce1021245210f16aa4b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 17 Nov 2025 17:32:46 +0800 Subject: [PATCH 03/10] Update docs/rfcs/0001_kernel.md Co-authored-by: Andrew Lamb --- docs/rfcs/0001_kernel.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rfcs/0001_kernel.md b/docs/rfcs/0001_kernel.md index cca233908e..2fade54b24 100644 --- a/docs/rfcs/0001_kernel.md +++ b/docs/rfcs/0001_kernel.md @@ -21,7 +21,7 @@ ## Background -Issue #1819 proposes decoupling the protocol/metadata/plan logic that currently lives inside the `iceberg` crate so that it can serve as a reusable “kernel,” similar to the approach taken by delta-kernel-rs. Today the `iceberg` crate simultaneously exposes the public trait surface and the default engine (Tokio runtime, opendal-backed FileIO, Arrow readers, etc.). This tight coupling makes it difficult for downstream projects to embed Iceberg metadata while providing their own storage, runtime, or execution stack. +Issue #1819 proposes decoupling the protocol/metadata/plan logic that currently lives inside the `iceberg` crate so that it can serve as a reusable “kernel,” similar to the approach taken by [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs). Today the `iceberg` crate simultaneously exposes the public trait surface and the default engine (Tokio runtime, opendal-backed FileIO, Arrow readers, etc.). This tight coupling makes it difficult for downstream projects to embed Iceberg metadata while providing their own storage, runtime, or execution stack. ## Goals and Scope From f81703442c655610cff5a1535088417161625565 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 17 Nov 2025 19:49:26 +0800 Subject: [PATCH 04/10] Remove spawn_blocking Signed-off-by: Xuanwo --- docs/rfcs/0001_kernel.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/rfcs/0001_kernel.md b/docs/rfcs/0001_kernel.md index 2fade54b24..fd711b0a54 100644 --- a/docs/rfcs/0001_kernel.md +++ b/docs/rfcs/0001_kernel.md @@ -120,11 +120,6 @@ pub trait Runtime: Send + Sync + 'static { F: Future + Send + 'static, T: Send + 'static; - fn spawn_blocking(&self, f: F) -> Self::JoinHandle - where - F: FnOnce() -> T + Send + 'static, - T: Send + 'static; - fn sleep(&self, dur: Duration) -> Pin + Send>>; } ``` From eaf52685dd4f55a88891b2767f5390466dbf9b6e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 19 Nov 2025 18:07:49 +0800 Subject: [PATCH 05/10] Refactor Signed-off-by: Xuanwo --- docs/rfcs/0001_kernel.md | 192 ------------------ ...0001_modularize_iceberg_implementations.md | 177 ++++++++++++++++ 2 files changed, 177 insertions(+), 192 deletions(-) delete mode 100644 docs/rfcs/0001_kernel.md create mode 100644 docs/rfcs/0001_modularize_iceberg_implementations.md diff --git a/docs/rfcs/0001_kernel.md b/docs/rfcs/0001_kernel.md deleted file mode 100644 index fd711b0a54..0000000000 --- a/docs/rfcs/0001_kernel.md +++ /dev/null @@ -1,192 +0,0 @@ - - -# RFC: Extract `iceberg-kernel` for Pluggable Execution Layers - -## Background - -Issue #1819 proposes decoupling the protocol/metadata/plan logic that currently lives inside the `iceberg` crate so that it can serve as a reusable “kernel,” similar to the approach taken by [delta-kernel-rs](https://github.com/delta-io/delta-kernel-rs). Today the `iceberg` crate simultaneously exposes the public trait surface and the default engine (Tokio runtime, opendal-backed FileIO, Arrow readers, etc.). This tight coupling makes it difficult for downstream projects to embed Iceberg metadata while providing their own storage, runtime, or execution stack. - -## Goals and Scope - -- **Full read & write coverage**: the kernel must contain every protocol component required for both scan planning and transactional writes (append, rewrite, commit, etc.). -- **No default runtime dependency**: the kernel defines a `Runtime` trait instead of depending on Tokio or Smol. -- **No default storage dependency**: the kernel defines `FileIO` traits only; concrete implementations (for example `iceberg-fileio-opendal`) live in dedicated crates. -- **Stable facade for existing users**: the top-level `iceberg` crate continues to expose the familiar API by re-exporting the kernel plus a default engine feature. - -Out of scope: changes to the Iceberg table specification or rewriting catalog adapters. - -## Architecture Overview - -### Workspace Layout - -``` -crates/ - kernel/ # new: pure protocols & planning logic - spec/ expr/ catalog/ table/ transaction/ scan/ runtime_api - io/traits.rs # FileIO traits (no opendal) - fileio/ - opendal/ # e.g. `iceberg-fileio-opendal` - fs/ # other FileIO implementations - runtime/ - tokio/ # e.g. `iceberg-runtime-tokio` - smol/ - iceberg/ # facade re-exporting kernel + default engine - catalog/* # depend on kernel (+ chosen FileIO/Runtime crates) - integrations/* # e.g. datafusion using facade or composing crates -``` - -### Trait Surfaces - -#### FileIO - -```rust -pub struct FileMetadata { - pub size: u64, - ... -} - -pub type FileReader = Box; - -#[async_trait::async_trait] -pub trait FileRead: Send + Sync + 'static { - async fn read(&self, range: Range) -> Result; -} - -pub type FileWriter = Box; - -#[async_trait::async_trait] -pub trait FileWrite: Send + Unpin + 'static { - async fn write(&mut self, bs: Bytes) -> Result<()>; - async fn close(&mut self) -> Result; -} - -pub type StorageFactory = fn(attrs: HashMap -> Result>); - -#[async_trait::async_trait] -pub trait Storage: Clone + Send + Sync { - async fn reader(&self, path: &str) -> Result; - async fn writer(&self, path: &str) -> Result; - async fn delete(&self, path: &str) -> Result<()>; - async fn exists(&self, path: &str) -> Result; - - ... -} - -pub struct FileIO { - registry: DashMap, -} - -impl FileIO { - fn register(scheme: &str, factory: StorageFactory); - - async fn read(path: &str) -> Result; - async fn reader(path: &str) -> Result; - async fn write(path: &str, bs: Bytes) -> Result; - async fn writer(path: &str) -> Result; - - async fn delete(&self, path: &str) -> Result<()>; - ... -} -``` - -- The kernel only defines the trait and error types. -- `iceberg-fileio-opendal` (new crate) ships an opendal-based implementation; other backends can publish their own crates. - -#### Runtime - -```rust -pub trait Runtime: Send + Sync + 'static { - type JoinHandle: Future + Send + 'static; - - fn spawn(&self, fut: F) -> Self::JoinHandle - where - F: Future + Send + 'static, - T: Send + 'static; - - fn sleep(&self, dur: Duration) -> Pin + Send>>; -} -``` - -- `TableScan` planning, metadata refresh, and `Transaction::commit` depend only on this trait. -- Crates such as `iceberg-runtime-tokio` provide concrete schedulers; consumers pick whichever runtime crate fits their stack. - -#### Catalog / Table / Transaction - -- The `Catalog` trait moves into the kernel and returns lightweight `TableHandle` objects (metadata + FileIO + Runtime). -- `TableHandle` no longer embeds Arrow helpers; Arrow-specific logic lives in engine crates. -- Transactions and their actions remain in the kernel, but rely on injected `Runtime` for retries/backoff. - -#### Scan / Planner - -- The kernel produces pure `TableScanPlan` descriptions (manifests, data-files, predicates, task graph). -- Engines provide executors (e.g., `ArrowExecutor`) that transform plans into record batches or other runtime-specific artifacts. - -### Facade Behavior - -- The top-level `iceberg` crate becomes a facade (`pub use iceberg_kernel::*`) that enables a *composition* of default crates (e.g. `iceberg-runtime-tokio`, `iceberg-fileio-opendal`, and a reference executor) behind feature flags. -- Existing convenience APIs (`Table::scan().to_arrow()`, `MemoryCatalog`, etc.) stay available but internally assemble the kernel with those default building blocks. - -## Migration Plan - -1. **Phase 1 – Create the kernel crate** - - Add `crates/kernel` and move `spec`, `expr`, `catalog`, `table`, `transaction`, `scan`, and supporting modules. - - Introduce temporary shim modules in the facade so existing imports keep working (mark them deprecated). - -2. **Phase 2 – Abstract runtime & IO** - - Define the `Runtime` and `FileIO` traits inside the kernel. - - Remove direct `tokio`/`opendal` dependencies from kernel modules. - - Introduce standalone crates (`iceberg-runtime-tokio`, `iceberg-fileio-opendal`, etc.) that implement the new traits. - -3. **Phase 3 – Detach Arrow/execution** - - Move `arrow` helpers and `ArrowReaderBuilder` into a reference executor crate (e.g. `iceberg-engine-arrow`). - - Update the DataFusion integration to depend on the facade or directly compose kernel + runtime + fileio + executor crates. - -4. **Phase 4 – Catalog & integration updates** - - Point catalog crates and other integrations to the kernel interfaces; depend on specific FileIO/Runtime crates only when required. - - Keep `iceberg-catalog-loader` kernel-only so users can inject their preferred combinations. - -5. **Phase 5 – Release & documentation** - - Finish the split within the 0.y.z series, provide an upgrade guide, and add kernel acceptance tests to guarantee trait stability. - -## Compatibility - -- Users who stick with the `iceberg` facade keep their existing API surface; the facade simply composes kernel + default runtime + default FileIO + reference executor under the hood. -- Advanced integrators can depend solely on `iceberg-kernel` and mix in whichever `FileIO`, `Runtime`, and executor crates they need (or author their own). -- CI keeps running the current integration tests and adds kernel-specific acceptance suites. - -## Risks and Mitigations - -| Risk | Description | Mitigation | -| ---- | ----------- | ---------- | -| Trait churn | Updating catalog/scan traits could break downstream crates | Maintain shim modules, use `#[deprecated]` windows, and document migration steps | -| Generic complexity | New traits may introduce complicated type bounds | Prefer `Arc` and `BoxFuture` to keep signatures manageable | -| Documentation gap | Users may not know which engine to pick | Publish new docs, diagrams, and “custom engine” tutorials alongside the split | - -## Open Questions - -1. Should the kernel expose any Arrow helpers, or should every Arrow-specific function live exclusively in engine crates? -2. Do we need a `ScanExecutor` trait inside the kernel for non-Arrow consumers? - -## Conclusion - -Extracting `iceberg-kernel` plus a pluggable engine layer lets the project: - -- Offer a lightweight, embeddable implementation of the Iceberg protocol, -- Enable external engines (DataFusion, Spark Connect, custom services) to reuse Iceberg metadata without inheriting specific runtime/storage dependencies, diff --git a/docs/rfcs/0001_modularize_iceberg_implementations.md b/docs/rfcs/0001_modularize_iceberg_implementations.md new file mode 100644 index 0000000000..4fad520f4d --- /dev/null +++ b/docs/rfcs/0001_modularize_iceberg_implementations.md @@ -0,0 +1,177 @@ +# RFC: Modularize `iceberg` Implementations + +## Background + +Issue #1819 highlighted that the current `iceberg` crate mixes the Iceberg protocol abstractions (catalog/table/plan/transaction) with concrete runtime, storage, and execution implementations (Tokio runtime wrappers, opendal-based `FileIO`, Arrow readers, DataFusion helpers, etc.). This makes the crate heavy, couples unrelated dependencies, and prevents users from bringing their own engines or storage stacks. + +After recent maintainer discussions we agreed on two principles: +1. The `iceberg` crate itself remains the single source of truth for all protocol traits and data structures. +2. All concrete integrations (Tokio runtime, opendal `FileIO`, Arrow/DataFusion executors, catalog adapters, etc.) move out of `iceberg` into dedicated companion crates. Users who need a ready-made execution path can depend on those crates (for example `iceberg-datafusion`) while users building custom stacks can depend solely on `iceberg`. + +This RFC describes the plan to slim down `iceberg` into a pure protocol crate and to reorganize the workspace around pluggable companion crates. + +## Goals and Scope + +- **Keep `iceberg` as the protocol crate**: it exposes all traits (`Catalog`, `Table`, `Transaction`, `FileIO`, `Runtime`, `ScanPlan`, etc.) plus metadata/plan logic, but no longer ships concrete runtimes or storage adapters. +- **Detach embedded implementations**: move opendal-based IO, Tokio runtime helpers, Arrow converters, and similar code into separate crates under `crates/fileio/*`, `crates/runtime/*`, `crates/engine/*`, or existing integration crates. +- **Enable composable combinations**: users assemble the stack they need by combining `iceberg` with specific implementation crates (e.g., `iceberg-fileio-opendal`, `iceberg-runtime-tokio`, `iceberg-engine-arrow`, `iceberg-datafusion`). +- **Minimize breaking surfaces**: trait APIs stay in `iceberg`; downstream crates only adjust their dependency graph. + +Out of scope: changing the Iceberg table specification or rewriting catalog adapters’ external behavior. + +## Architecture Overview + +### Workspace Layout + +``` +crates/ + iceberg/ # core traits, metadata, planning, transactions + fileio/ + opendal/ # e.g. `iceberg-fileio-opendal` + fs/ # other FileIO implementations + runtime/ + tokio/ # e.g. `iceberg-runtime-tokio` + smol/ + engine/ + arrow/ # Arrow executor & schema helpers + catalog/* # catalog adapters (REST, HMS, Glue, etc.) + integrations/ + datafusion/ # combines core + implementations for DF + cache-moka/ + playground/ +``` + +- `crates/iceberg` no longer depends on opendal, Tokio, Arrow, or DataFusion. +- Implementation crates depend on `iceberg` to get the trait surfaces they implement. +- Higher-level crates (e.g., `iceberg-datafusion`) pull in the required runtime/FileIO/executor crates and expose an opinionated combination. + +### Core Trait Surfaces (within `iceberg`) + +#### FileIO + +```rust +pub struct FileMetadata { + pub size: u64, + ... +} + +pub type FileReader = Box; + +#[async_trait::async_trait] +pub trait FileRead: Send + Sync + 'static { + async fn read(&self, range: Range) -> Result; +} + +pub type FileWriter = Box; + +#[async_trait::async_trait] +pub trait FileWrite: Send + Unpin + 'static { + async fn write(&mut self, bs: Bytes) -> Result<()>; + async fn close(&mut self) -> Result; +} + +pub type StorageFactory = fn(attrs: HashMap -> Result>); + +#[async_trait::async_trait] +pub trait Storage: Clone + Send + Sync { + async fn reader(&self, path: &str) -> Result; + async fn writer(&self, path: &str) -> Result; + async fn delete(&self, path: &str) -> Result<()>; + async fn exists(&self, path: &str) -> Result; + + ... +} + +pub struct FileIO { + registry: DashMap, +} + +impl FileIO { + fn register(scheme: &str, factory: StorageFactory); + + async fn read(path: &str) -> Result; + async fn reader(path: &str) -> Result; + async fn write(path: &str, bs: Bytes) -> Result; + async fn writer(path: &str) -> Result; + + async fn delete(&self, path: &str) -> Result<()>; + ... +} +``` + +- `FileRead` / `FileWrite` remain Iceberg-specific traits (range reads, metrics hooks, abort/commit) and live inside `iceberg`. +- Concrete implementations (opendal, local FS, custom stores) live in companion crates and return trait objects. + +#### Runtime + +```rust +pub trait Runtime: Send + Sync + 'static { + type JoinHandle: Future + Send + 'static; + + fn spawn(&self, fut: F) -> Self::JoinHandle + where + F: Future + Send + 'static, + T: Send + 'static; + + fn sleep(&self, dur: Duration) -> Pin + Send>>; +} +``` + +- The trait lives in `iceberg`; crates like `iceberg-runtime-tokio` implement it and expose constructors. + +#### Catalog / Table / Transaction / Scan + +- All existing traits and data structures remain in `crates/iceberg`. +- `TableScan` continues to emit pure plan descriptors; executors interpret them. +- `Transaction` uses injected `Runtime` for retry/backoff but otherwise stays unchanged. + +### Usage Modes + +- **Custom stacks**: depend solely on `iceberg` plus self-authored implementations that satisfy the traits. +- **Pre-built stacks**: depend on crates such as `iceberg-datafusion` that bundle `iceberg` with `iceberg-runtime-tokio`, `iceberg-fileio-opendal`, and `iceberg-engine-arrow` (and expose higher-level APIs). +- `iceberg` itself does not re-export any of the companion crates; users compose them explicitly. + +## Migration Plan + +1. **Phase 1 – Slim down `crates/iceberg`** + - Remove direct dependencies on opendal, Tokio, Arrow, and DataFusion from `iceberg`. + - Move the concrete implementations into new crates (while keeping the same code initially). + +2. **Phase 2 – Stabilize trait surfaces** + - Finalize dyn-friendly `FileIO`, `FileRead`, `FileWrite`, and `Runtime` traits inside `iceberg`. + - Provide shims (deprecation warnings) for any APIs that previously returned concrete types (e.g., `InputFile`, `OutputFile`) so downstream integrations can migrate. + +3. **Phase 3 – Arrow/execution extraction** + - Relocate `crates/iceberg/src/arrow/*` and related helpers into `crates/engine/arrow`. + - Update `iceberg-datafusion` to consume the new executor crate plus whichever runtime/fileio implementations it needs. + +4. **Phase 4 – Catalog and integration updates** + - Ensure catalog crates compile against the new dependency graph (they now depend on `iceberg` plus whichever FileIO/runtime crates they require). + - Provide documentation/examples showing how to assemble `iceberg` with the desired implementations. + +5. **Phase 5 – Documentation & release** + - Publish a migration guide explaining the new crate layout and how to replace previous helper APIs with the new building blocks. + - Tag a breaking-release (e.g., 0.8.0) and coordinate with downstream projects (`iceberg-datafusion`, Python bindings, etc.). + +## Compatibility + +- Existing users who depended on `iceberg`’s built-in Arrow/Tokio/FileIO helpers must now add explicit dependencies on the relevant implementation crates (`iceberg-fileio-opendal`, `iceberg-runtime-tokio`, `iceberg-engine-arrow`, or `iceberg-datafusion`). +- Users implementing custom stacks continue to depend on `iceberg` only; they implement the required traits themselves. +- Tests and examples that previously lived inside `crates/iceberg` move to whichever crate now hosts the implementation. + +## Risks and Mitigations + +| Risk | Description | Mitigation | +| ---- | ----------- | ---------- | +| Discoverability | Users may not know which combination of crates to pick | Provide clear docs linking to `iceberg-datafusion`, `iceberg-fileio-opendal`, etc., plus template examples | +| Trait churn | Adjusting `FileIO`/`Runtime` APIs could break downstream code | Introduce deprecation shims and publish migration notes before removal | +| Duplicate dependencies | Some crates might accidentally pull multiple implementations | Document recommended combos; enforce mutually exclusive features where practical | + +## Open Questions + +1. How do we version the companion crates relative to `iceberg` to signal compatibility (same version numbers vs. independent)? +2. What is the timeline for removing deprecated APIs (e.g., `Table::scan().to_arrow()`), and do we provide temporary re-exports via `iceberg-datafusion`? + +## Conclusion + +By keeping `iceberg` focused on core traits and moving concrete implementations into companion crates, we reduce unnecessary coupling, make it easier for the community to plug in custom runtimes and storage layers, and still provide ready-to-use stacks such as `iceberg-datafusion`. This RFC outlines the restructuring needed to accomplish that while keeping the trait surfaces centralized in `iceberg`. From 09317fc48d055f4e7582837e21aff0a54380eb57 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 19 Nov 2025 18:13:26 +0800 Subject: [PATCH 06/10] Update docs/rfcs/0001_modularize_iceberg_implementations.md Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- ...0001_modularize_iceberg_implementations.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/rfcs/0001_modularize_iceberg_implementations.md b/docs/rfcs/0001_modularize_iceberg_implementations.md index 4fad520f4d..73a19c69df 100644 --- a/docs/rfcs/0001_modularize_iceberg_implementations.md +++ b/docs/rfcs/0001_modularize_iceberg_implementations.md @@ -1,3 +1,22 @@ + + # RFC: Modularize `iceberg` Implementations ## Background From 621ecba12e8e9c0d74eda7e92c9de8283299766e Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 24 Nov 2025 16:09:03 +0800 Subject: [PATCH 07/10] Remove detailed traits out of this RFC Signed-off-by: Xuanwo --- ...0001_modularize_iceberg_implementations.md | 81 ++----------------- 1 file changed, 5 insertions(+), 76 deletions(-) diff --git a/docs/rfcs/0001_modularize_iceberg_implementations.md b/docs/rfcs/0001_modularize_iceberg_implementations.md index 4fad520f4d..f886ecd8c0 100644 --- a/docs/rfcs/0001_modularize_iceberg_implementations.md +++ b/docs/rfcs/0001_modularize_iceberg_implementations.md @@ -47,83 +47,12 @@ crates/ ### Core Trait Surfaces (within `iceberg`) -#### FileIO - -```rust -pub struct FileMetadata { - pub size: u64, - ... -} - -pub type FileReader = Box; - -#[async_trait::async_trait] -pub trait FileRead: Send + Sync + 'static { - async fn read(&self, range: Range) -> Result; -} - -pub type FileWriter = Box; - -#[async_trait::async_trait] -pub trait FileWrite: Send + Unpin + 'static { - async fn write(&mut self, bs: Bytes) -> Result<()>; - async fn close(&mut self) -> Result; -} - -pub type StorageFactory = fn(attrs: HashMap -> Result>); - -#[async_trait::async_trait] -pub trait Storage: Clone + Send + Sync { - async fn reader(&self, path: &str) -> Result; - async fn writer(&self, path: &str) -> Result; - async fn delete(&self, path: &str) -> Result<()>; - async fn exists(&self, path: &str) -> Result; - - ... -} - -pub struct FileIO { - registry: DashMap, -} - -impl FileIO { - fn register(scheme: &str, factory: StorageFactory); - - async fn read(path: &str) -> Result; - async fn reader(path: &str) -> Result; - async fn write(path: &str, bs: Bytes) -> Result; - async fn writer(path: &str) -> Result; - - async fn delete(&self, path: &str) -> Result<()>; - ... -} -``` - -- `FileRead` / `FileWrite` remain Iceberg-specific traits (range reads, metrics hooks, abort/commit) and live inside `iceberg`. -- Concrete implementations (opendal, local FS, custom stores) live in companion crates and return trait objects. - -#### Runtime - -```rust -pub trait Runtime: Send + Sync + 'static { - type JoinHandle: Future + Send + 'static; - - fn spawn(&self, fut: F) -> Self::JoinHandle - where - F: Future + Send + 'static, - T: Send + 'static; - - fn sleep(&self, dur: Duration) -> Pin + Send>>; -} -``` - -- The trait lives in `iceberg`; crates like `iceberg-runtime-tokio` implement it and expose constructors. - -#### Catalog / Table / Transaction / Scan - -- All existing traits and data structures remain in `crates/iceberg`. +- `FileIO` +- `Runtime` +- `Catalog` - `TableScan` continues to emit pure plan descriptors; executors interpret them. -- `Transaction` uses injected `Runtime` for retry/backoff but otherwise stays unchanged. + +Those traits will be discussed separately outside of this RFC. ### Usage Modes From 62e1fc06c35f0c25b32596daf68e1596bab62f6b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 24 Nov 2025 16:13:20 +0800 Subject: [PATCH 08/10] polish details Signed-off-by: Xuanwo --- .../0001_modularize_iceberg_implementations.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/rfcs/0001_modularize_iceberg_implementations.md b/docs/rfcs/0001_modularize_iceberg_implementations.md index f886ecd8c0..13f3db3898 100644 --- a/docs/rfcs/0001_modularize_iceberg_implementations.md +++ b/docs/rfcs/0001_modularize_iceberg_implementations.md @@ -32,10 +32,9 @@ crates/ runtime/ tokio/ # e.g. `iceberg-runtime-tokio` smol/ - engine/ - arrow/ # Arrow executor & schema helpers catalog/* # catalog adapters (REST, HMS, Glue, etc.) integrations/ + local/ # local engine for iceberg. datafusion/ # combines core + implementations for DF cache-moka/ playground/ @@ -57,19 +56,19 @@ Those traits will be discussed separately outside of this RFC. ### Usage Modes - **Custom stacks**: depend solely on `iceberg` plus self-authored implementations that satisfy the traits. -- **Pre-built stacks**: depend on crates such as `iceberg-datafusion` that bundle `iceberg` with `iceberg-runtime-tokio`, `iceberg-fileio-opendal`, and `iceberg-engine-arrow` (and expose higher-level APIs). +- **Pre-built stacks**: depend on crates such as `iceberg-datafusion` that bundle `iceberg` with `iceberg-runtime-tokio`, `iceberg-fileio-opendal` (and expose higher-level APIs). - `iceberg` itself does not re-export any of the companion crates; users compose them explicitly. ## Migration Plan -1. **Phase 1 – Slim down `crates/iceberg`** - - Remove direct dependencies on opendal, Tokio, Arrow, and DataFusion from `iceberg`. - - Move the concrete implementations into new crates (while keeping the same code initially). - -2. **Phase 2 – Stabilize trait surfaces** +1. **Phase 1 – Stabilize trait surfaces** - Finalize dyn-friendly `FileIO`, `FileRead`, `FileWrite`, and `Runtime` traits inside `iceberg`. - Provide shims (deprecation warnings) for any APIs that previously returned concrete types (e.g., `InputFile`, `OutputFile`) so downstream integrations can migrate. +2. **Phase 2 – Slim down `crates/iceberg`** + - Remove direct dependencies on opendal, Tokio, Arrow, and DataFusion from `iceberg`. + - Move the concrete implementations into new crates (while keeping the same code initially). + 3. **Phase 3 – Arrow/execution extraction** - Relocate `crates/iceberg/src/arrow/*` and related helpers into `crates/engine/arrow`. - Update `iceberg-datafusion` to consume the new executor crate plus whichever runtime/fileio implementations it needs. From 54d4ddec6b41305cb2448995427b254712e840d4 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 24 Nov 2025 16:15:34 +0800 Subject: [PATCH 09/10] Add notes for arrow Signed-off-by: Xuanwo --- docs/rfcs/0001_modularize_iceberg_implementations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/rfcs/0001_modularize_iceberg_implementations.md b/docs/rfcs/0001_modularize_iceberg_implementations.md index 3dc8d8c22e..3103c392de 100644 --- a/docs/rfcs/0001_modularize_iceberg_implementations.md +++ b/docs/rfcs/0001_modularize_iceberg_implementations.md @@ -86,6 +86,7 @@ Those traits will be discussed separately outside of this RFC. 2. **Phase 2 – Slim down `crates/iceberg`** - Remove direct dependencies on opendal, Tokio, Arrow, and DataFusion from `iceberg`. + - notes: removing the arrow entirely could be complex, so we will do this gradually. - Move the concrete implementations into new crates (while keeping the same code initially). 3. **Phase 3 – Arrow/execution extraction** From a65640a3499db0dab05b33e1098f8f6b6c6256fe Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 24 Nov 2025 16:44:31 +0800 Subject: [PATCH 10/10] polish Signed-off-by: Xuanwo --- ...0001_modularize_iceberg_implementations.md | 97 +++++++++---------- 1 file changed, 46 insertions(+), 51 deletions(-) diff --git a/docs/rfcs/0001_modularize_iceberg_implementations.md b/docs/rfcs/0001_modularize_iceberg_implementations.md index 3103c392de..14bd478270 100644 --- a/docs/rfcs/0001_modularize_iceberg_implementations.md +++ b/docs/rfcs/0001_modularize_iceberg_implementations.md @@ -21,26 +21,26 @@ ## Background -Issue #1819 highlighted that the current `iceberg` crate mixes the Iceberg protocol abstractions (catalog/table/plan/transaction) with concrete runtime, storage, and execution implementations (Tokio runtime wrappers, opendal-based `FileIO`, Arrow readers, DataFusion helpers, etc.). This makes the crate heavy, couples unrelated dependencies, and prevents users from bringing their own engines or storage stacks. +Issue #1819 highlighted that the current `iceberg` crate mixes the Iceberg protocol abstractions (catalog/table/plan/transaction) with concrete runtime, storage, and execution code (Tokio runtime wrappers, opendal-based `FileIO`, Arrow helpers, DataFusion glue, etc.). This coupling makes the crate heavy and blocks users from composing their own storage or execution stacks. -After recent maintainer discussions we agreed on two principles: -1. The `iceberg` crate itself remains the single source of truth for all protocol traits and data structures. -2. All concrete integrations (Tokio runtime, opendal `FileIO`, Arrow/DataFusion executors, catalog adapters, etc.) move out of `iceberg` into dedicated companion crates. Users who need a ready-made execution path can depend on those crates (for example `iceberg-datafusion`) while users building custom stacks can depend solely on `iceberg`. +Two principles have been agreed: +1. The `iceberg` crate remains the single source of truth for all protocol traits and data structures. We will not create a separate “kernel” crate or facade layer. +2. Concrete integrations (Tokio runtime, opendal `FileIO`, Arrow/DataFusion glue, catalog adapters, etc.) move out into dedicated companion crates. Users needing a ready path can depend on those crates (e.g., `iceberg-datafusion` or `integrations/local`), while custom stacks depend only on `iceberg`. -This RFC describes the plan to slim down `iceberg` into a pure protocol crate and to reorganize the workspace around pluggable companion crates. +This RFC focuses on modularizing implementations; detailed trait signatures (e.g., `FileIO`, `Runtime`) will be handled in separate RFCs. ## Goals and Scope -- **Keep `iceberg` as the protocol crate**: it exposes all traits (`Catalog`, `Table`, `Transaction`, `FileIO`, `Runtime`, `ScanPlan`, etc.) plus metadata/plan logic, but no longer ships concrete runtimes or storage adapters. -- **Detach embedded implementations**: move opendal-based IO, Tokio runtime helpers, Arrow converters, and similar code into separate crates under `crates/fileio/*`, `crates/runtime/*`, `crates/engine/*`, or existing integration crates. -- **Enable composable combinations**: users assemble the stack they need by combining `iceberg` with specific implementation crates (e.g., `iceberg-fileio-opendal`, `iceberg-runtime-tokio`, `iceberg-engine-arrow`, `iceberg-datafusion`). -- **Minimize breaking surfaces**: trait APIs stay in `iceberg`; downstream crates only adjust their dependency graph. +- Keep `iceberg` as the protocol crate (traits + metadata + planning), without bundling runtimes, storage adapters, or execution glue. +- Relocate concrete code into companion crates under `crates/fileio/*`, `crates/runtime/*`, and `crates/integrations/*`. +- Provide a staged plan for extracting Arrow-dependent APIs to avoid destabilizing file-format code. +- Minimize breaking surfaces: traits stay in `iceberg`; downstream crates mainly adjust dependencies. -Out of scope: changing the Iceberg table specification or rewriting catalog adapters’ external behavior. +Out of scope: changes to the Iceberg table specification or catalog adapter external behavior; detailed trait method design (covered by follow-up RFCs). ## Architecture Overview -### Workspace Layout +### Workspace Layout (target) ``` crates/ @@ -53,73 +53,68 @@ crates/ smol/ catalog/* # catalog adapters (REST, HMS, Glue, etc.) integrations/ - local/ # local engine for iceberg. + local/ # simple local/arrow-based helper crate datafusion/ # combines core + implementations for DF cache-moka/ playground/ ``` -- `crates/iceberg` no longer depends on opendal, Tokio, Arrow, or DataFusion. -- Implementation crates depend on `iceberg` to get the trait surfaces they implement. -- Higher-level crates (e.g., `iceberg-datafusion`) pull in the required runtime/FileIO/executor crates and expose an opinionated combination. +- `crates/iceberg` drops direct deps on opendal, Tokio, Arrow, and DataFusion. +- Implementation crates depend on `iceberg` to implement the traits. +- Higher-level crates (`integrations/local`, `iceberg-datafusion`) assemble the pieces for ready-to-use scenarios. -### Core Trait Surfaces (within `iceberg`) +### Core Trait Surfaces -- `FileIO` -- `Runtime` -- `Catalog` -- `TableScan` continues to emit pure plan descriptors; executors interpret them. - -Those traits will be discussed separately outside of this RFC. +`FileIO`, `Runtime`, `Catalog`, `Table`, `Transaction`, `TableScan` (plan descriptors) all remain hosted in `iceberg`. Precise method signatures are deferred to dedicated RFCs to avoid locking details prematurely. ### Usage Modes -- **Custom stacks**: depend solely on `iceberg` plus self-authored implementations that satisfy the traits. -- **Pre-built stacks**: depend on crates such as `iceberg-datafusion` that bundle `iceberg` with `iceberg-runtime-tokio`, `iceberg-fileio-opendal` (and expose higher-level APIs). -- `iceberg` itself does not re-export any of the companion crates; users compose them explicitly. +- **Custom stacks**: depend on `iceberg` and provide your own implementations. +- **Pre-built stacks**: depend on `integrations/local` or `iceberg-datafusion`, which bundle `iceberg` with selected runtime/FileIO/Arrow helpers. +- `iceberg` does not re-export companion crates; users compose explicitly. -## Migration Plan +## Migration Plan (staged, with Arrow extraction phased) -1. **Phase 1 – Stabilize trait surfaces** - - Finalize dyn-friendly `FileIO`, `FileRead`, `FileWrite`, and `Runtime` traits inside `iceberg`. - - Provide shims (deprecation warnings) for any APIs that previously returned concrete types (e.g., `InputFile`, `OutputFile`) so downstream integrations can migrate. +1. **Phase 1 – Confirm trait hosting, defer details** + - Keep all protocol traits in `iceberg`; move detailed API design (FileIO, Runtime, etc.) to separate RFCs. + - Add temporary shims/deprecations only when traits are finalized. -2. **Phase 2 – Slim down `crates/iceberg`** - - Remove direct dependencies on opendal, Tokio, Arrow, and DataFusion from `iceberg`. - - notes: removing the arrow entirely could be complex, so we will do this gradually. - - Move the concrete implementations into new crates (while keeping the same code initially). +2. **Phase 2 – First Arrow step: move `to_arrow()` out** + - Relocate the public `to_arrow()` API to `integrations/local` (or another higher-level crate). Core no longer exposes Arrow entry points. + - Keep internal Arrow-dependent helpers (e.g., `ArrowFileReader`) temporarily in `iceberg` to avoid breaking file-format flows. -3. **Phase 3 – Arrow/execution extraction** - - Relocate `crates/iceberg/src/arrow/*` and related helpers into `crates/engine/arrow`. - - Update `iceberg-datafusion` to consume the new executor crate plus whichever runtime/fileio implementations it needs. +3. **Phase 3 – Gradual Arrow dependency removal** + - Incrementally migrate/replace Arrow-dependent internals (`ArrowFileReader`, format-specific readers) into `integrations/local` or other helper crates. + - Adjust file-format APIs as needed; expect this to be multi-release work. -4. **Phase 4 – Catalog and integration updates** - - Ensure catalog crates compile against the new dependency graph (they now depend on `iceberg` plus whichever FileIO/runtime crates they require). - - Provide documentation/examples showing how to assemble `iceberg` with the desired implementations. +4. **Phase 4 – Dependency cleanup** + - Ensure catalog and integration crates depend only on `iceberg` plus the specific runtime/FileIO/helper crates they need. + - Verify build/test pipelines against the new dependency graph. -5. **Phase 5 – Documentation & release** - - Publish a migration guide explaining the new crate layout and how to replace previous helper APIs with the new building blocks. - - Tag a breaking-release (e.g., 0.8.0) and coordinate with downstream projects (`iceberg-datafusion`, Python bindings, etc.). +5. **Phase 5 – Docs & release** + - Publish migration guides: where `to_arrow()` moved, how to assemble local/DataFusion stacks. + - Schedule deprecation windows for remaining Arrow helpers; target a breaking release once Arrow is fully removed from `iceberg`. ## Compatibility -- Existing users who depended on `iceberg`’s built-in Arrow/Tokio/FileIO helpers must now add explicit dependencies on the relevant implementation crates (`iceberg-fileio-opendal`, `iceberg-runtime-tokio`, `iceberg-engine-arrow`, or `iceberg-datafusion`). -- Users implementing custom stacks continue to depend on `iceberg` only; they implement the required traits themselves. -- Tests and examples that previously lived inside `crates/iceberg` move to whichever crate now hosts the implementation. +- Short term: users of `Table::scan().to_arrow()` must switch to `integrations/local` (or another crate that rehosts that API). Other Arrow types stay temporarily but will migrate in later phases. +- Long term: `iceberg` will be Arrow-free; companion crates provide Arrow-based helpers. +- Tests/examples move alongside the implementations they exercise. ## Risks and Mitigations | Risk | Description | Mitigation | | ---- | ----------- | ---------- | -| Discoverability | Users may not know which combination of crates to pick | Provide clear docs linking to `iceberg-datafusion`, `iceberg-fileio-opendal`, etc., plus template examples | -| Trait churn | Adjusting `FileIO`/`Runtime` APIs could break downstream code | Introduce deprecation shims and publish migration notes before removal | -| Duplicate dependencies | Some crates might accidentally pull multiple implementations | Document recommended combos; enforce mutually exclusive features where practical | +| Arrow dependency unwinding is complex | File-format readers may rely on Arrow types | Phase the work; move `to_arrow()` first, then refactor readers; document interim state | +| Discoverability | Users may not know where Arrow helpers went | Clear docs pointing to `integrations/local` and `iceberg-datafusion`; migration guide | +| Trait churn | Future trait RFCs may break early adopters | Use deprecation shims and communicate timelines | +| Duplicate impls | Multiple helper crates could overlap | Provide recommended combinations and feature guidance | ## Open Questions -1. How do we version the companion crates relative to `iceberg` to signal compatibility (same version numbers vs. independent)? -2. What is the timeline for removing deprecated APIs (e.g., `Table::scan().to_arrow()`), and do we provide temporary re-exports via `iceberg-datafusion`? +1. Versioning: align companion crate versions with `iceberg`, or allow independent versions plus compatibility matrix? +2. Deprecation schedule: how long do we keep interim Arrow helpers before full removal from `iceberg`? ## Conclusion -By keeping `iceberg` focused on core traits and moving concrete implementations into companion crates, we reduce unnecessary coupling, make it easier for the community to plug in custom runtimes and storage layers, and still provide ready-to-use stacks such as `iceberg-datafusion`. This RFC outlines the restructuring needed to accomplish that while keeping the trait surfaces centralized in `iceberg`. +We will keep `iceberg` as the protocol crate while modularizing concrete implementations. Arrow removal will be phased: first relocating `to_arrow()` to `integrations/local`, then gradually moving Arrow-dependent readers and helpers. This keeps the core lean, lets users compose their preferred runtime/FileIO stacks, and still offers ready-to-use combinations via companion crates.