From 769c9171bd7e57d81faea0dbbb765faf33d6cbb6 Mon Sep 17 00:00:00 2001 From: xaneets Date: Fri, 2 Jan 2026 13:49:28 +0300 Subject: [PATCH 1/9] add some tests, add check limit for bincode decode --- .github/workflows/tests.yml | 14 ++++- README.md | 5 +- pg_debyte_core/src/codec.rs | 7 +++ pg_debyte_core/tests/decode.rs | 55 +++++++++++++++++ pg_debyte_core/tests/encode.rs | 59 ++++++++++++++++++- pg_debyte_core/tests/envelope_errors.rs | 49 +++++++++++++++ pg_debyte_core/tests/registry.rs | 15 +++++ ...test.sh => docker-ci-test-pg-extension.sh} | 0 scripts/docker-ci-test-workspace.sh | 6 ++ scripts/docker-test.sh | 5 -- scripts/host-tests.sh | 5 ++ 11 files changed, 210 insertions(+), 10 deletions(-) create mode 100644 pg_debyte_core/tests/envelope_errors.rs create mode 100644 pg_debyte_core/tests/registry.rs rename scripts/{docker-ci-test.sh => docker-ci-test-pg-extension.sh} (100%) create mode 100755 scripts/docker-ci-test-workspace.sh delete mode 100755 scripts/docker-test.sh create mode 100755 scripts/host-tests.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a8d9751..39b3b81 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,17 @@ on: pull_request: jobs: - tests: + core-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build docker image + run: ./scripts/docker-ci-build-image.sh + - name: Workspace tests (exclude pg_debyte_ext) + run: ./scripts/docker-ci-test-workspace.sh + pg-extension-tests: runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -14,4 +24,4 @@ jobs: - name: Build docker image run: ./scripts/docker-ci-build-image.sh - name: Docker tests - run: ./scripts/docker-ci-test.sh + run: ./scripts/docker-ci-test-pg-extension.sh diff --git a/README.md b/README.md index 0f11e52..c051964 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # pg-debyte [![Build](https://github.com/Xaneets/pg-debyte/actions/workflows/build.yml/badge.svg)](https://github.com/Xaneets/pg-debyte/actions/workflows/build.yml) -[![tests](https://github.com/Xaneets/pg-debyte/actions/workflows/tests.yml/badge.svg)](https://github.com/Xaneets/pg-debyte/actions/workflows/tests.yml) +[![Workspace tests](https://img.shields.io/github/actions/workflow/status/Xaneets/pg-debyte/tests.yml?branch=main&job=core-tests&label=workspace%20tests)](https://github.com/Xaneets/pg-debyte/actions/workflows/tests.yml) +[![PG extension tests](https://img.shields.io/github/actions/workflow/status/Xaneets/pg-debyte/tests.yml?branch=main&job=pg-extension-tests&label=pg%20extension%20tests)](https://github.com/Xaneets/pg-debyte/actions/workflows/tests.yml) [![Crates.io](https://img.shields.io/crates/v/pg_debyte_core.svg)](https://crates.io/crates/pg_debyte_core) Core building blocks for PostgreSQL extensions that decode `bytea` into JSON. @@ -84,5 +85,5 @@ cargo run -p pg_debyte_tools --bin demo_auto_sql If host permissions make `cargo pgrx test` difficult, use the Docker runner: ```bash -./scripts/docker-test.sh +./scripts/docker-ci-test-pg-extension.sh ``` diff --git a/pg_debyte_core/src/codec.rs b/pg_debyte_core/src/codec.rs index bd74613..141e5e8 100644 --- a/pg_debyte_core/src/codec.rs +++ b/pg_debyte_core/src/codec.rs @@ -41,6 +41,13 @@ impl Codec for BincodeCodec { limits: &DecodeLimits, ) -> Result { let limit = self.byte_limit.min(limits.max_output_bytes as u64); + if bytes.len() as u64 > limit { + return Err(DecodeError::LimitExceeded { + context: "codec_input_bytes", + limit: limit as usize, + actual: bytes.len(), + }); + } bincode::DefaultOptions::new() .with_limit(limit) .deserialize(bytes) diff --git a/pg_debyte_core/tests/decode.rs b/pg_debyte_core/tests/decode.rs index d0be9ec..f0d0928 100644 --- a/pg_debyte_core/tests/decode.rs +++ b/pg_debyte_core/tests/decode.rs @@ -3,6 +3,7 @@ use pg_debyte_core::action::ZstdAction; use pg_debyte_core::codec::{BincodeCodec, Codec}; use pg_debyte_core::error::DecodeError; use pg_debyte_core::types::DecodeLimits; +use pg_debyte_core::types::EncodeLimits; use pg_debyte_core::ByteAction; use serde::{Deserialize, Serialize}; @@ -44,3 +45,57 @@ fn zstd_decode_respects_limit() { other => panic!("unexpected error: {other:?}"), } } + +#[test] +fn zstd_encode_respects_limit() { + let payload = b"hello hello hello"; + let action = ZstdAction::new(1); + let limits = EncodeLimits::new(1); + let err = action + .encode(payload, &limits, &[]) + .expect_err("expected limit error"); + match err { + DecodeError::LimitExceeded { context, .. } => { + assert_eq!(context, "action_output_bytes"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn bincode_decode_respects_limits() { + let demo: Vec = vec![0u8; 1024]; + let bytes = bincode::DefaultOptions::new() + .with_limit(2048) + .serialize(&demo) + .expect("serialize"); + assert!(bytes.len() > 64); + let codec = BincodeCodec::new(1, 1024); + let limits = DecodeLimits::new(1024, 64, 1024); + let err: DecodeError = codec + .decode::>(&bytes, &limits) + .expect_err("expected limit error"); + match err { + DecodeError::LimitExceeded { context, .. } => { + assert_eq!(context, "codec_input_bytes"); + } + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn bincode_encode_respects_limits() { + let demo = Demo { + id: 1, + name: "demo".repeat(64), + }; + let codec = BincodeCodec::new(1, 16); + let limits = pg_debyte_core::types::EncodeLimits::new(16); + let err = codec + .encode(&demo, &limits) + .expect_err("expected limit error"); + match err { + DecodeError::Bincode(_) => {} + other => panic!("unexpected error: {other:?}"), + } +} diff --git a/pg_debyte_core/tests/encode.rs b/pg_debyte_core/tests/encode.rs index b79b483..ea84e4e 100644 --- a/pg_debyte_core/tests/encode.rs +++ b/pg_debyte_core/tests/encode.rs @@ -1,4 +1,4 @@ -use pg_debyte_core::action::ActionSpec; +use pg_debyte_core::action::{ActionSpec, ZstdAction}; use pg_debyte_core::codec::{BincodeCodec, Codec}; use pg_debyte_core::encode::encode_to_envelope; use pg_debyte_core::envelope::{try_parse, ParsedEnvelope}; @@ -49,3 +49,60 @@ fn encode_builds_envelope() { .expect("decode payload"); assert_eq!(decoded, demo); } + +#[test] +fn encode_roundtrip_with_actions() { + let demo = Demo { + id: 9, + name: "actions".to_string(), + }; + let key = TypeKey { + type_id: Uuid::from_bytes([3; 16]), + schema_version: 2, + }; + let codec = BincodeCodec::new(2, 1024); + let limits = EncodeLimits::new(1024); + static ACTION: ZstdAction = ZstdAction::new(7); + static ACTIONS: [&'static dyn pg_debyte_core::action::ByteAction; 1] = [&ACTION]; + let registry = StaticRegistry::new(&[], &ACTIONS); + let actions = vec![ActionSpec::new(7, 1, vec![1])]; + + let encoded = + encode_to_envelope(&demo, &codec, key, &actions, ®istry, &limits).expect("encode"); + + let parsed = try_parse(&encoded).expect("parse"); + let view = match parsed { + ParsedEnvelope::Envelope(view) => view, + ParsedEnvelope::None => panic!("expected envelope"), + }; + + assert_eq!(view.key, key); + assert_eq!(view.codec_id, codec.id()); + assert_eq!(view.actions.len(), 1); + assert_eq!(view.actions[0].id, 7); + assert_eq!(view.actions[0].flags, 1); + assert_eq!(view.actions[0].params, [1]); +} + +#[test] +fn encode_rejects_unknown_action() { + let demo = Demo { + id: 10, + name: "unknown".to_string(), + }; + let key = TypeKey { + type_id: Uuid::from_bytes([4; 16]), + schema_version: 1, + }; + let codec = BincodeCodec::new(3, 1024); + let limits = EncodeLimits::new(1024); + let registry = StaticRegistry::new(&[], &[]); + let actions = vec![ActionSpec::new(99, 0, vec![0])]; + + let err = encode_to_envelope(&demo, &codec, key, &actions, ®istry, &limits) + .expect_err("expected unknown action"); + match err { + pg_debyte_core::error::DecodeError::UnknownAction(id) => assert_eq!(id, 99), + other => panic!("unexpected error: {other:?}"), + } +} diff --git a/pg_debyte_core/tests/envelope_errors.rs b/pg_debyte_core/tests/envelope_errors.rs new file mode 100644 index 0000000..764ddbb --- /dev/null +++ b/pg_debyte_core/tests/envelope_errors.rs @@ -0,0 +1,49 @@ +use pg_debyte_core::envelope::try_parse; +use pg_debyte_core::error::DecodeError; +use uuid::Uuid; + +fn base_header(version: u8, actions_count: u8) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"PGDEBYTE"); + bytes.push(version); + bytes.extend_from_slice(Uuid::from_bytes([0x11; 16]).as_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&2u16.to_le_bytes()); + bytes.push(actions_count); + bytes +} + +#[test] +fn envelope_unsupported_version() { + let bytes = base_header(2, 0); + let err = try_parse(&bytes).expect_err("expected error"); + match err { + DecodeError::BadEnvelope(msg) => assert_eq!(msg, "unsupported envelope version"), + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn envelope_action_header_out_of_bounds() { + let bytes = base_header(1, 1); + let err = try_parse(&bytes).expect_err("expected error"); + match err { + DecodeError::BadEnvelope(msg) => assert_eq!(msg, "action header out of bounds"), + other => panic!("unexpected error: {other:?}"), + } +} + +#[test] +fn envelope_params_out_of_bounds() { + let mut bytes = base_header(1, 1); + bytes.extend_from_slice(&9u16.to_le_bytes()); + bytes.push(0); + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(b"abc"); + + let err = try_parse(&bytes).expect_err("expected error"); + match err { + DecodeError::BadEnvelope(msg) => assert_eq!(msg, "params out of bounds"), + other => panic!("unexpected error: {other:?}"), + } +} diff --git a/pg_debyte_core/tests/registry.rs b/pg_debyte_core/tests/registry.rs new file mode 100644 index 0000000..6ec0a02 --- /dev/null +++ b/pg_debyte_core/tests/registry.rs @@ -0,0 +1,15 @@ +use pg_debyte_core::registry::{Registry, StaticRegistry}; +use pg_debyte_core::types::TypeKey; +use uuid::Uuid; + +#[test] +fn registry_lookup_missing_entries() { + let registry = StaticRegistry::new(&[], &[]); + let key = TypeKey { + type_id: Uuid::from_bytes([9; 16]), + schema_version: 1, + }; + + assert!(registry.lookup_decoder(key).is_none()); + assert!(registry.lookup_action(42).is_none()); +} diff --git a/scripts/docker-ci-test.sh b/scripts/docker-ci-test-pg-extension.sh similarity index 100% rename from scripts/docker-ci-test.sh rename to scripts/docker-ci-test-pg-extension.sh diff --git a/scripts/docker-ci-test-workspace.sh b/scripts/docker-ci-test-workspace.sh new file mode 100755 index 0000000..1f9a463 --- /dev/null +++ b/scripts/docker-ci-test-workspace.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +docker run --rm -t pg-debyte-ci bash -lc "\ + cargo test --workspace --exclude pg_debyte_ext \ +" diff --git a/scripts/docker-test.sh b/scripts/docker-test.sh deleted file mode 100755 index 021ab9f..0000000 --- a/scripts/docker-test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -./scripts/docker-ci-build-image.sh -./scripts/docker-ci-test.sh diff --git a/scripts/host-tests.sh b/scripts/host-tests.sh new file mode 100755 index 0000000..d6abd2d --- /dev/null +++ b/scripts/host-tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +cargo test --workspace --exclude pg_debyte_ext +cargo pgrx test -p pg_debyte_ext --features pg17 From 2c4480ae9b78fce2a9c5d9036e1e224b17eddd91 Mon Sep 17 00:00:00 2001 From: xaneets Date: Fri, 2 Jan 2026 13:59:02 +0300 Subject: [PATCH 2/9] move core test from docker to runner --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39b3b81..2b6e0d5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,10 +11,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Build docker image - run: ./scripts/docker-ci-build-image.sh + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable - name: Workspace tests (exclude pg_debyte_ext) - run: ./scripts/docker-ci-test-workspace.sh + run: cargo test --workspace --exclude pg_debyte_ext pg-extension-tests: runs-on: ubuntu-latest timeout-minutes: 20 From f6b3435169650822a71c439a88ffb40034b16721 Mon Sep 17 00:00:00 2001 From: xaneets Date: Fri, 2 Jan 2026 14:09:24 +0300 Subject: [PATCH 3/9] split workflow --- .github/workflows/core-tests.yml | 17 +++++++++++++++ .github/workflows/pg-extension-tests.yml | 21 ++++++++++++++++++ .github/workflows/tests.yml | 27 ------------------------ README.md | 4 ++-- scripts/docker-ci-test-workspace.sh | 6 ------ 5 files changed, 40 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/core-tests.yml create mode 100644 .github/workflows/pg-extension-tests.yml delete mode 100644 .github/workflows/tests.yml delete mode 100755 scripts/docker-ci-test-workspace.sh diff --git a/.github/workflows/core-tests.yml b/.github/workflows/core-tests.yml new file mode 100644 index 0000000..c90a1ae --- /dev/null +++ b/.github/workflows/core-tests.yml @@ -0,0 +1,17 @@ +name: core-tests + +on: + push: + pull_request: + +jobs: + core-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + - name: Workspace tests (exclude pgrx/extension) + run: cargo test --workspace --exclude pg_debyte_ext --exclude pg_debyte_pgrx diff --git a/.github/workflows/pg-extension-tests.yml b/.github/workflows/pg-extension-tests.yml new file mode 100644 index 0000000..1f72728 --- /dev/null +++ b/.github/workflows/pg-extension-tests.yml @@ -0,0 +1,21 @@ +name: pg-extension-tests + +on: + push: + pull_request: + +jobs: + pg-extension-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build docker image + run: ./scripts/docker-ci-build-image.sh + - name: PGRX tests (pg_debyte_ext) + run: ./scripts/docker-ci-test-pg-extension.sh + - name: Workspace tests (include pg_debyte_pgrx) + run: docker run --rm -t pg-debyte-ci bash -lc "\ + cargo test --workspace --exclude pg_debyte_ext \ + " diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 2b6e0d5..0000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: tests - -on: - push: - pull_request: - -jobs: - core-tests: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - - name: Workspace tests (exclude pg_debyte_ext) - run: cargo test --workspace --exclude pg_debyte_ext - pg-extension-tests: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build docker image - run: ./scripts/docker-ci-build-image.sh - - name: Docker tests - run: ./scripts/docker-ci-test-pg-extension.sh diff --git a/README.md b/README.md index c051964..fe24697 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # pg-debyte [![Build](https://github.com/Xaneets/pg-debyte/actions/workflows/build.yml/badge.svg)](https://github.com/Xaneets/pg-debyte/actions/workflows/build.yml) -[![Workspace tests](https://img.shields.io/github/actions/workflow/status/Xaneets/pg-debyte/tests.yml?branch=main&job=core-tests&label=workspace%20tests)](https://github.com/Xaneets/pg-debyte/actions/workflows/tests.yml) -[![PG extension tests](https://img.shields.io/github/actions/workflow/status/Xaneets/pg-debyte/tests.yml?branch=main&job=pg-extension-tests&label=pg%20extension%20tests)](https://github.com/Xaneets/pg-debyte/actions/workflows/tests.yml) +[![Workspace tests](https://github.com/Xaneets/pg-debyte/actions/workflows/core-tests.yml/badge.svg)](https://github.com/Xaneets/pg-debyte/actions/workflows/core-tests.yml) +[![PG extension tests](https://github.com/Xaneets/pg-debyte/actions/workflows/pg-extension-tests.yml/badge.svg)](https://github.com/Xaneets/pg-debyte/actions/workflows/pg-extension-tests.yml) [![Crates.io](https://img.shields.io/crates/v/pg_debyte_core.svg)](https://crates.io/crates/pg_debyte_core) Core building blocks for PostgreSQL extensions that decode `bytea` into JSON. diff --git a/scripts/docker-ci-test-workspace.sh b/scripts/docker-ci-test-workspace.sh deleted file mode 100755 index 1f9a463..0000000 --- a/scripts/docker-ci-test-workspace.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -docker run --rm -t pg-debyte-ci bash -lc "\ - cargo test --workspace --exclude pg_debyte_ext \ -" From e81ffd3209e3f5cce3807e54674b71fbb59ae84b Mon Sep 17 00:00:00 2001 From: xaneets Date: Fri, 2 Jan 2026 15:38:38 +0300 Subject: [PATCH 4/9] add test for limits. fix ci --- .github/workflows/pg-extension-tests.yml | 4 +-- pg_debyte_ext/src/lib.rs | 34 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pg-extension-tests.yml b/.github/workflows/pg-extension-tests.yml index 1f72728..ca4a83a 100644 --- a/.github/workflows/pg-extension-tests.yml +++ b/.github/workflows/pg-extension-tests.yml @@ -16,6 +16,4 @@ jobs: - name: PGRX tests (pg_debyte_ext) run: ./scripts/docker-ci-test-pg-extension.sh - name: Workspace tests (include pg_debyte_pgrx) - run: docker run --rm -t pg-debyte-ci bash -lc "\ - cargo test --workspace --exclude pg_debyte_ext \ - " + run: docker run --rm -t pg-debyte-ci bash -lc "/usr/local/cargo/bin/cargo test --workspace --exclude pg_debyte_ext" diff --git a/pg_debyte_ext/src/lib.rs b/pg_debyte_ext/src/lib.rs index 03833f9..7f45d87 100644 --- a/pg_debyte_ext/src/lib.rs +++ b/pg_debyte_ext/src/lib.rs @@ -140,4 +140,38 @@ mod tests { assert!(!ok); } + + #[pg_test] + fn test_guc_max_input_bytes() { + let ok = PgTryBuilder::new(|| { + Spi::run("SET LOCAL pg_debyte.max_input_bytes = 8").expect("set guc"); + let _ = Spi::get_one::( + "SELECT bytea_to_json_by_id(decode('0102030405', 'hex'), \ + '11111111-1111-1111-1111-111111111111'::uuid, 1::smallint)", + ) + .expect("spi"); + true + }) + .catch_others(|_| false) + .execute(); + + assert!(!ok); + } + + #[pg_test] + fn test_guc_max_json_bytes() { + let ok = PgTryBuilder::new(|| { + Spi::run("SET LOCAL pg_debyte.max_json_bytes = 8").expect("set guc"); + let _ = Spi::get_one::( + "SELECT bytea_to_json_by_id(decode('010464656d6f', 'hex'), \ + '11111111-1111-1111-1111-111111111111'::uuid, 1::smallint)", + ) + .expect("spi"); + true + }) + .catch_others(|_| false) + .execute(); + + assert!(!ok); + } } From fe1ea864cfc9f9de62465a37e6cc01e5a08e3b0c Mon Sep 17 00:00:00 2001 From: xaneets Date: Sat, 3 Jan 2026 10:45:50 +0300 Subject: [PATCH 5/9] add know schema fn --- README.md | 271 +++++++++++++++++++++ pg_debyte_ext/src/lib.rs | 81 +++++- pg_debyte_macros/src/lib.rs | 31 +++ pg_debyte_pgrx/src/lib.rs | 18 ++ pg_debyte_tools/Cargo.toml | 5 + pg_debyte_tools/src/demo_second_payload.rs | 25 ++ scripts/host-lints.sh | 6 + 7 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 pg_debyte_tools/src/demo_second_payload.rs create mode 100755 scripts/host-lints.sh diff --git a/README.md b/README.md index fe24697..e7c8031 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,265 @@ This repository provides reusable Rust crates plus a small example extension. - `pg_debyte_ext` is only an example implementation; you will create your own extension crate. - PG15 support will be added later as a separate focus. +## Quick start: build your own extension + +Common setup: + +```bash +cargo new my_pg_debyte_ext --lib +cd my_pg_debyte_ext +``` + +```toml +[dependencies] +pgrx = { version = "0.16.1", default-features = false, features = ["pg17"] } +pg_debyte_core = "0.1.0" +pg_debyte_macros = "0.1.0" +pg_debyte_pgrx = "0.1.0" +serde = { version = "1.0", features = ["derive"] } +uuid = "1.8" +bincode = "1.3" +hex = "0.4" +``` + +Build and install once: + +```bash +cargo pgrx init --pg17 /path/to/pg17 +cargo pgrx install --features pg17 --sudo +``` + +Enable in Postgres: + +```sql +CREATE EXTENSION my_pg_debyte_ext; +``` + +### Known schema (per-type function) + +Use this when the payload schema is fixed and you want a dedicated SQL function per type. +The function is generated by `declare_know_schema!` and decodes raw payload without an envelope. + +`src/lib.rs`: + +```rust +use pgrx::prelude::*; +use pg_debyte_core::{BincodeCodec, StaticRegistry}; +use pg_debyte_macros::declare_know_schema; +use serde::{Deserialize, Serialize}; +use uuid::Uuid as CoreUuid; + +pg_module_magic!(); + +#[derive(Debug, Deserialize, Serialize)] +struct MyRecord { + id: u32, + label: String, +} + +const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x22; 16]); +const MY_SCHEMA_VERSION: u16 = 1; +const MY_CODEC_ID: u16 = 1; +const MY_CODEC: BincodeCodec = BincodeCodec::new(MY_CODEC_ID, 32 * 1024 * 1024); + +declare_know_schema!( + MY_DECODER, + ty = MyRecord, + type_id = MY_TYPE_ID, + schema_version = MY_SCHEMA_VERSION, + codec = MY_CODEC, + codec_ty = BincodeCodec, + actions = [], + fn_name = bytea_to_json_my_record +); + +static REGISTRY: StaticRegistry = StaticRegistry::new(&[&MY_DECODER], &[]); + +#[pg_guard] +pub unsafe extern "C-unwind" fn _PG_init() { + pg_debyte_pgrx::init_gucs(); + pg_debyte_pgrx::set_registry(®ISTRY); +} +``` + +Usage: + + +```sql +SELECT bytea_to_json_my_record(decode('', 'hex')); +``` + +### Registry + by_id + +Use this when you have raw payloads but need flexible routing by `type_id` + `schema_version`. +You register decoders in a registry and call a single SQL function with explicit type info. + +`src/lib.rs`: + +```rust +use pgrx::prelude::*; +use pgrx::JsonB; +use pg_debyte_core::{BincodeCodec, DecodeError, StaticRegistry}; +use pg_debyte_macros::declare_decoder; +use serde::{Deserialize, Serialize}; +use uuid::Uuid as CoreUuid; + +pg_module_magic!(); + +#[derive(Debug, Deserialize, Serialize)] +struct MyRecord { + id: u32, + label: String, +} + +const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x22; 16]); +const MY_SCHEMA_VERSION: u16 = 1; +const MY_CODEC_ID: u16 = 1; +const MY_CODEC: BincodeCodec = BincodeCodec::new(MY_CODEC_ID, 32 * 1024 * 1024); + +declare_decoder!( + MY_DECODER, + ty = MyRecord, + type_id = MY_TYPE_ID, + schema_version = MY_SCHEMA_VERSION, + codec = MY_CODEC, + codec_ty = BincodeCodec, + actions = [] +); + +static REGISTRY: StaticRegistry = StaticRegistry::new(&[&MY_DECODER], &[]); + +#[pg_guard] +pub unsafe extern "C-unwind" fn _PG_init() { + pg_debyte_pgrx::init_gucs(); + pg_debyte_pgrx::set_registry(®ISTRY); +} + +#[pg_extern] +fn bytea_to_json_by_id( + data: Vec, + type_id: pgrx::Uuid, + schema_version: i16, +) -> Result { + let limits = pg_debyte_pgrx::limits(); + let core_uuid = CoreUuid::from_bytes(*type_id.as_bytes()); + let value = pg_debyte_pgrx::decode_by_id(&data, core_uuid, schema_version, &limits)?; + Ok(JsonB(value)) +} +``` + +Usage (reuse `demo_payload.rs` from above): + +```sql +SELECT bytea_to_json_by_id( + decode('', 'hex'), + '22222222-2222-2222-2222-222222222222'::uuid, + 1::smallint +); +``` + +### Envelope (auto) + +Use this when payloads include an envelope with type, version, codec, and actions. +`bytea_to_json_auto` parses the envelope and dispatches to the matching decoder automatically. + +`src/lib.rs`: + +```rust +use pgrx::prelude::*; +use pgrx::JsonB; +use pg_debyte_core::{BincodeCodec, DecodeError, StaticRegistry}; +use pg_debyte_macros::declare_decoder; +use serde::{Deserialize, Serialize}; +use uuid::Uuid as CoreUuid; + +pg_module_magic!(); + +#[derive(Debug, Deserialize, Serialize)] +struct MyRecord { + id: u32, + label: String, +} + +const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x22; 16]); +const MY_SCHEMA_VERSION: u16 = 1; +const MY_CODEC_ID: u16 = 1; +const MY_CODEC: BincodeCodec = BincodeCodec::new(MY_CODEC_ID, 32 * 1024 * 1024); + +declare_decoder!( + MY_DECODER, + ty = MyRecord, + type_id = MY_TYPE_ID, + schema_version = MY_SCHEMA_VERSION, + codec = MY_CODEC, + codec_ty = BincodeCodec, + actions = [] +); + +static REGISTRY: StaticRegistry = StaticRegistry::new(&[&MY_DECODER], &[]); + +#[pg_guard] +pub unsafe extern "C-unwind" fn _PG_init() { + pg_debyte_pgrx::init_gucs(); + pg_debyte_pgrx::set_registry(®ISTRY); +} + +#[pg_extern] +fn bytea_to_json_auto(data: Vec) -> Result { + let limits = pg_debyte_pgrx::limits(); + let value = pg_debyte_pgrx::decode_auto(&data, &limits)?; + Ok(JsonB(value)) +} +``` + +`src/bin/demo_envelope.rs` (example of wrapping your data with an envelope; prints hex-encoded payload): + +```rust +use hex::encode; +use pg_debyte_core::action::ActionSpec; +use pg_debyte_core::codec::BincodeCodec; +use pg_debyte_core::encode::encode_to_envelope; +use pg_debyte_core::registry::StaticRegistry; +use pg_debyte_core::types::{EncodeLimits, TypeKey}; +use serde::Serialize; +use uuid::Uuid as CoreUuid; + +#[derive(Debug, Serialize)] +struct MyRecord { + id: u32, + label: String, +} + +fn main() { + let record = MyRecord { + id: 1, + label: "demo".to_string(), + }; + let key = TypeKey { + type_id: CoreUuid::from_bytes([0x22; 16]), + schema_version: 1, + }; + let codec = BincodeCodec::new(1, 32 * 1024 * 1024); + let limits = EncodeLimits::new(32 * 1024 * 1024); + let registry = StaticRegistry::new(&[], &[]); + let actions: Vec = Vec::new(); + + let envelope = encode_to_envelope(&record, &codec, key, &actions, ®istry, &limits) + .expect("encode envelope"); + println!("{}", encode(envelope)); +} +``` + +Usage: + +```bash +cargo run --bin demo_envelope +``` + +```sql +SELECT bytea_to_json_auto(decode('', 'hex')); +``` + ## Example usage (PG17) Build and install the example extension: @@ -61,6 +320,18 @@ SELECT bytea_to_json_by_id( ); ``` +Generate a demo payload hex for a known schema: + +```bash +cargo run -p pg_debyte_tools --bin demo_second_payload +``` + +Decode in SQL (known schema, per-type function): + +```sql +SELECT bytea_to_json_demo_record_second(decode('', 'hex')); +``` + Generate a demo envelope hex: ```bash diff --git a/pg_debyte_ext/src/lib.rs b/pg_debyte_ext/src/lib.rs index 7f45d87..b6dac6d 100644 --- a/pg_debyte_ext/src/lib.rs +++ b/pg_debyte_ext/src/lib.rs @@ -12,7 +12,7 @@ pub mod pg_test { } } use pg_debyte_core::{BincodeCodec, DecodeError, StaticRegistry, ZstdAction}; -use pg_debyte_macros::declare_decoder; +use pg_debyte_macros::{declare_decoder, declare_know_schema}; use serde::{Deserialize, Serialize}; use uuid::Uuid as CoreUuid; @@ -42,6 +42,27 @@ declare_decoder!( actions = [] ); +#[derive(Debug, Deserialize, Serialize)] +struct DemoRecordSecond { + id: u32, + text: String, + flag: bool, +} + +const DEMO_SECOND_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0xa3; 16]); +const DEMO_SECOND_SCHEMA_VERSION: u16 = 1; + +declare_know_schema!( + DEMO_DECODER_SECOND, + ty = DemoRecordSecond, + type_id = DEMO_SECOND_TYPE_ID, + schema_version = DEMO_SECOND_SCHEMA_VERSION, + codec = DEMO_CODEC, + codec_ty = BincodeCodec, + actions = [], + fn_name = bytea_to_json_demo_record_second +); + static REGISTRY: StaticRegistry = StaticRegistry::new(&[&DEMO_DECODER], &[&ZSTD_ACTION]); #[pg_guard] @@ -174,4 +195,62 @@ mod tests { assert!(!ok); } + + #[pg_test] + fn test_bytea_to_json_know_schema() { + let json = Spi::get_one::( + "SELECT bytea_to_json_demo_record_second(decode('01067365636f6e6401', 'hex'))", + ) + .expect("spi") + .expect("json"); + + assert_eq!(json.0, json!({"id": 1, "text": "second", "flag": true})); + } + + #[pg_test] + fn test_know_schema_guc_max_input_bytes() { + let ok = PgTryBuilder::new(|| { + Spi::run("SET LOCAL pg_debyte.max_input_bytes = 8").expect("set guc"); + let _ = Spi::get_one::( + "SELECT bytea_to_json_demo_record_second(decode('01067365636f6e6401', 'hex'))", + ) + .expect("spi"); + true + }) + .catch_others(|_| false) + .execute(); + + assert!(!ok); + } + + #[pg_test] + fn test_know_schema_guc_max_json_bytes() { + let ok = PgTryBuilder::new(|| { + Spi::run("SET LOCAL pg_debyte.max_json_bytes = 8").expect("set guc"); + let _ = Spi::get_one::( + "SELECT bytea_to_json_demo_record_second(decode('01067365636f6e6401', 'hex'))", + ) + .expect("spi"); + true + }) + .catch_others(|_| false) + .execute(); + + assert!(!ok); + } + + #[pg_test] + fn test_know_schema_decode_error() { + let ok = PgTryBuilder::new(|| { + let _ = Spi::get_one::( + "SELECT bytea_to_json_demo_record_second(decode('010464656d6f', 'hex'))", + ) + .expect("spi"); + true + }) + .catch_others(|_| false) + .execute(); + + assert!(!ok); + } } diff --git a/pg_debyte_macros/src/lib.rs b/pg_debyte_macros/src/lib.rs index b0b636e..ff53575 100644 --- a/pg_debyte_macros/src/lib.rs +++ b/pg_debyte_macros/src/lib.rs @@ -20,3 +20,34 @@ macro_rules! declare_decoder { ); }; } + +#[macro_export] +macro_rules! declare_know_schema { + ( + $name:ident, + ty = $ty:ty, + type_id = $type_id:expr, + schema_version = $schema_version:expr, + codec = $codec:expr, + codec_ty = $codec_ty:ty, + actions = [$($action:expr),* $(,)?], + fn_name = $fn_name:ident + ) => { + pub static $name: pg_debyte_core::TypedDecoderEntry<$ty, $codec_ty> = + pg_debyte_core::TypedDecoderEntry::new( + pg_debyte_core::TypeKey { + type_id: $type_id, + schema_version: $schema_version, + }, + $codec, + &[$($action),*], + ); + + #[pg_extern] + fn $fn_name(data: Vec) -> Result { + let limits = pg_debyte_pgrx::limits(); + let value = pg_debyte_pgrx::decode_know_schema(&data, &$name, &limits)?; + Ok(pgrx::JsonB(value)) + } + }; +} diff --git a/pg_debyte_pgrx/src/lib.rs b/pg_debyte_pgrx/src/lib.rs index ad2841c..2a6ab1f 100644 --- a/pg_debyte_pgrx/src/lib.rs +++ b/pg_debyte_pgrx/src/lib.rs @@ -3,6 +3,7 @@ use pg_debyte_core::envelope::{try_parse, ParsedEnvelope}; use pg_debyte_core::error::DecodeError; use pg_debyte_core::registry::Registry; use pg_debyte_core::types::{DecodeLimits, TypeKey}; +use pg_debyte_core::DecoderEntry; use pgrx::guc::{GucContext, GucFlags, GucRegistry, GucSetting}; use std::sync::OnceLock; use uuid::Uuid; @@ -88,6 +89,23 @@ pub fn decode_by_id( Ok(value) } +pub fn decode_know_schema( + data: &[u8], + decoder: &dyn DecoderEntry, + limits: &DecodeLimits, +) -> Result { + ensure_limit("input_bytes", data.len(), limits.max_input_bytes)?; + let value = if decoder.default_actions().is_empty() { + decoder.decode_payload(data, limits)? + } else { + let reg = registry()?; + let payload = apply_actions_refs(reg, decoder.default_actions(), data, limits)?; + decoder.decode_payload(&payload, limits)? + }; + ensure_json_limit(&value, limits)?; + Ok(value) +} + pub fn decode_auto(data: &[u8], limits: &DecodeLimits) -> Result { ensure_limit("input_bytes", data.len(), limits.max_input_bytes)?; let reg = registry()?; diff --git a/pg_debyte_tools/Cargo.toml b/pg_debyte_tools/Cargo.toml index 293963a..94e2fd3 100644 --- a/pg_debyte_tools/Cargo.toml +++ b/pg_debyte_tools/Cargo.toml @@ -22,6 +22,11 @@ path = "src/demo_envelope.rs" name = "demo_auto_sql" path = "src/demo_auto_sql.rs" +[[bin]] +name = "demo_second_payload" +path = "src/demo_second_payload.rs" + + [dependencies] serde = { version = "1.0", features = ["derive"] } bincode = "1.3" diff --git a/pg_debyte_tools/src/demo_second_payload.rs b/pg_debyte_tools/src/demo_second_payload.rs new file mode 100644 index 0000000..eae04ff --- /dev/null +++ b/pg_debyte_tools/src/demo_second_payload.rs @@ -0,0 +1,25 @@ +use bincode::Options; +use hex::encode; +use serde::Serialize; + +#[derive(Debug, Serialize)] +struct DemoRecord { + id: u32, + text: String, + flag: bool, +} + +fn main() { + let record = DemoRecord { + id: 1, + text: "second".to_string(), + flag: true, + }; + + let bytes = bincode::DefaultOptions::new() + .with_limit(32 * 1024 * 1024) + .serialize(&record) + .expect("serialize demo payload"); + + println!("{}", encode(bytes)); +} diff --git a/scripts/host-lints.sh b/scripts/host-lints.sh new file mode 100755 index 0000000..f2233fd --- /dev/null +++ b/scripts/host-lints.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +cargo +nightly fmt -- --check +cargo clippy --workspace --exclude pg_debyte_ext +cargo clippy -p pg_debyte_ext --features pg17 From fabaefa941c8e8ce6beef26edc79cbb69ca2e7e3 Mon Sep 17 00:00:00 2001 From: xaneets Date: Sat, 3 Jan 2026 11:20:56 +0300 Subject: [PATCH 6/9] 0.2.0 ver --- Cargo.lock | 6 +++--- README.md | 7 ++++--- pg_debyte_core/Cargo.toml | 2 +- pg_debyte_ext/Cargo.toml | 6 +++--- pg_debyte_macros/Cargo.toml | 4 ++-- pg_debyte_pgrx/Cargo.toml | 4 ++-- pg_debyte_tools/Cargo.toml | 2 +- 7 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0c06ce..6213fbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -886,7 +886,7 @@ dependencies = [ [[package]] name = "pg_debyte_core" -version = "0.1.0" +version = "0.2.0" dependencies = [ "bincode", "hex", @@ -914,14 +914,14 @@ dependencies = [ [[package]] name = "pg_debyte_macros" -version = "0.1.0" +version = "0.2.0" dependencies = [ "pg_debyte_core", ] [[package]] name = "pg_debyte_pgrx" -version = "0.1.0" +version = "0.2.0" dependencies = [ "pg_debyte_core", "pgrx", diff --git a/README.md b/README.md index e7c8031..86782b6 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ This repository provides reusable Rust crates plus a small example extension. - Action pipeline (decode in reverse) with bounded zstd decode. - Bincode codec with size limits. - Static registry for decoders/codecs/actions. +- Known schema SQL functions (per-type decoding without envelope). - PG17-only helper for GUC limits and decoding (to be called from extension). ## Notes @@ -41,9 +42,9 @@ cd my_pg_debyte_ext ```toml [dependencies] pgrx = { version = "0.16.1", default-features = false, features = ["pg17"] } -pg_debyte_core = "0.1.0" -pg_debyte_macros = "0.1.0" -pg_debyte_pgrx = "0.1.0" +pg_debyte_core = "0.2.0" +pg_debyte_macros = "0.2.0" +pg_debyte_pgrx = "0.2.0" serde = { version = "1.0", features = ["derive"] } uuid = "1.8" bincode = "1.3" diff --git a/pg_debyte_core/Cargo.toml b/pg_debyte_core/Cargo.toml index ead8abf..003c6da 100644 --- a/pg_debyte_core/Cargo.toml +++ b/pg_debyte_core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pg_debyte_core" -version = "0.1.0" +version = "0.2.0" edition = "2021" authors = ["Dmitriy Sergeev "] description = "Core building blocks for PostgreSQL extensions that decode bytea into JSON" diff --git a/pg_debyte_ext/Cargo.toml b/pg_debyte_ext/Cargo.toml index 5e4c1a2..b80e3cb 100644 --- a/pg_debyte_ext/Cargo.toml +++ b/pg_debyte_ext/Cargo.toml @@ -22,9 +22,9 @@ pg17 = ["pgrx/pg17", "pgrx-tests/pg17"] pg_test = [] [dependencies] -pg_debyte_core = { version = "0.1.0", path = "../pg_debyte_core" } -pg_debyte_pgrx = { version = "0.1.0", path = "../pg_debyte_pgrx" } -pg_debyte_macros = { version = "0.1.0", path = "../pg_debyte_macros" } +pg_debyte_core = { version = "0.2.0", path = "../pg_debyte_core" } +pg_debyte_pgrx = { version = "0.2.0", path = "../pg_debyte_pgrx" } +pg_debyte_macros = { version = "0.2.0", path = "../pg_debyte_macros" } serde = { version = "1.0", features = ["derive"] } uuid = "1.8" hex = "0.4" diff --git a/pg_debyte_macros/Cargo.toml b/pg_debyte_macros/Cargo.toml index 6e009fa..4a663d6 100644 --- a/pg_debyte_macros/Cargo.toml +++ b/pg_debyte_macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pg_debyte_macros" -version = "0.1.0" +version = "0.2.0" edition = "2021" authors = ["Dmitriy Sergeev "] description = "Helper macros for registering typed decoders in pg_debyte" @@ -12,4 +12,4 @@ keywords = ["postgres", "postgresql", "bytea", "json", "pgrx"] categories = ["database", "development-tools"] [dependencies] -pg_debyte_core = { version = "0.1.0", path = "../pg_debyte_core" } +pg_debyte_core = { version = "0.2.0", path = "../pg_debyte_core" } diff --git a/pg_debyte_pgrx/Cargo.toml b/pg_debyte_pgrx/Cargo.toml index 75cd987..200091a 100644 --- a/pg_debyte_pgrx/Cargo.toml +++ b/pg_debyte_pgrx/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pg_debyte_pgrx" -version = "0.1.0" +version = "0.2.0" edition = "2021" authors = ["Dmitriy Sergeev "] description = "PG17-only pgrx glue for pg_debyte (GUCs and decode helpers)" @@ -12,7 +12,7 @@ keywords = ["postgres", "postgresql", "bytea", "json", "pgrx"] categories = ["database"] [dependencies] -pg_debyte_core = { version = "0.1.0", path = "../pg_debyte_core" } +pg_debyte_core = { version = "0.2.0", path = "../pg_debyte_core" } serde_json = "1.0" uuid = "1.8" diff --git a/pg_debyte_tools/Cargo.toml b/pg_debyte_tools/Cargo.toml index 94e2fd3..d4ca12a 100644 --- a/pg_debyte_tools/Cargo.toml +++ b/pg_debyte_tools/Cargo.toml @@ -31,5 +31,5 @@ path = "src/demo_second_payload.rs" serde = { version = "1.0", features = ["derive"] } bincode = "1.3" hex = "0.4" -pg_debyte_core = { version = "0.1.0", path = "../pg_debyte_core" } +pg_debyte_core = { version = "0.2.0", path = "../pg_debyte_core" } uuid = "1.8" From 401bce11696154b1389aa4795c303bac2b3f2e0c Mon Sep 17 00:00:00 2001 From: xaneets Date: Sat, 3 Jan 2026 11:56:49 +0300 Subject: [PATCH 7/9] examples --- .github/workflows/build.yml | 2 + Cargo.lock | 36 ++++++++++++++ Cargo.toml | 3 ++ README.md | 66 +++++++++---------------- examples/readme_by_id/Cargo.toml | 20 ++++++++ examples/readme_by_id/src/lib.rs | 49 ++++++++++++++++++ examples/readme_envelope/Cargo.toml | 20 ++++++++ examples/readme_envelope/src/lib.rs | 44 +++++++++++++++++ examples/readme_known_schema/Cargo.toml | 20 ++++++++ examples/readme_known_schema/src/lib.rs | 37 ++++++++++++++ scripts/docker-ci-build-examples.sh | 8 +++ scripts/docker-ci-build-workspace.sh | 2 +- scripts/host-examples.sh | 6 +++ 13 files changed, 268 insertions(+), 45 deletions(-) create mode 100644 examples/readme_by_id/Cargo.toml create mode 100644 examples/readme_by_id/src/lib.rs create mode 100644 examples/readme_envelope/Cargo.toml create mode 100644 examples/readme_envelope/src/lib.rs create mode 100644 examples/readme_known_schema/Cargo.toml create mode 100644 examples/readme_known_schema/src/lib.rs create mode 100644 scripts/docker-ci-build-examples.sh create mode 100755 scripts/host-examples.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b4bd15f..5c164a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,3 +13,5 @@ jobs: run: ./scripts/docker-ci-build-image.sh - name: Build workspace packages run: ./scripts/docker-ci-build-workspace.sh + - name: Build README examples + run: ./scripts/docker-ci-build-examples.sh diff --git a/Cargo.lock b/Cargo.lock index 6213fbe..37af3ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,6 +1225,42 @@ dependencies = [ "getrandom", ] +[[package]] +name = "readme_by_id" +version = "0.1.0" +dependencies = [ + "pg_debyte_core", + "pg_debyte_macros", + "pg_debyte_pgrx", + "pgrx", + "serde", + "uuid", +] + +[[package]] +name = "readme_envelope" +version = "0.1.0" +dependencies = [ + "pg_debyte_core", + "pg_debyte_macros", + "pg_debyte_pgrx", + "pgrx", + "serde", + "uuid", +] + +[[package]] +name = "readme_known_schema" +version = "0.1.0" +dependencies = [ + "pg_debyte_core", + "pg_debyte_macros", + "pg_debyte_pgrx", + "pgrx", + "serde", + "uuid", +] + [[package]] name = "redox_syscall" version = "0.5.18" diff --git a/Cargo.toml b/Cargo.toml index 110d4f8..dee2013 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,5 +3,8 @@ members = [ "pg_debyte_core", "pg_debyte_ext", "pg_debyte_macros", "pg_debyte_pgrx", "pg_debyte_tools", + "examples/readme_known_schema", + "examples/readme_by_id", + "examples/readme_envelope", ] resolver = "2" diff --git a/README.md b/README.md index 86782b6..f56278a 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,13 @@ This repository provides reusable Rust crates plus a small example extension. - `pg_debyte_ext` is only an example implementation; you will create your own extension crate. - PG15 support will be added later as a separate focus. +## Examples + +The code blocks below are backed by real crates in `examples/`: +- `examples/readme_known_schema` +- `examples/readme_by_id` +- `examples/readme_envelope` + ## Quick start: build your own extension Common setup: @@ -47,8 +54,6 @@ pg_debyte_macros = "0.2.0" pg_debyte_pgrx = "0.2.0" serde = { version = "1.0", features = ["derive"] } uuid = "1.8" -bincode = "1.3" -hex = "0.4" ``` Build and install once: @@ -86,7 +91,7 @@ struct MyRecord { label: String, } -const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x22; 16]); +const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x11; 16]); const MY_SCHEMA_VERSION: u16 = 1; const MY_CODEC_ID: u16 = 1; const MY_CODEC: BincodeCodec = BincodeCodec::new(MY_CODEC_ID, 32 * 1024 * 1024); @@ -113,6 +118,11 @@ pub unsafe extern "C-unwind" fn _PG_init() { Usage: +Generate a raw payload with the helper binary: + +```bash +cargo run -p pg_debyte_tools --bin demo_payload +``` ```sql SELECT bytea_to_json_my_record(decode('', 'hex')); @@ -141,7 +151,7 @@ struct MyRecord { label: String, } -const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x22; 16]); +const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x11; 16]); const MY_SCHEMA_VERSION: u16 = 1; const MY_CODEC_ID: u16 = 1; const MY_CODEC: BincodeCodec = BincodeCodec::new(MY_CODEC_ID, 32 * 1024 * 1024); @@ -177,7 +187,11 @@ fn bytea_to_json_by_id( } ``` -Usage (reuse `demo_payload.rs` from above): +Usage (generate payload with the helper binary): + +```bash +cargo run -p pg_debyte_tools --bin demo_payload +``` ```sql SELECT bytea_to_json_by_id( @@ -241,48 +255,12 @@ fn bytea_to_json_auto(data: Vec) -> Result { } ``` -`src/bin/demo_envelope.rs` (example of wrapping your data with an envelope; prints hex-encoded payload): - -```rust -use hex::encode; -use pg_debyte_core::action::ActionSpec; -use pg_debyte_core::codec::BincodeCodec; -use pg_debyte_core::encode::encode_to_envelope; -use pg_debyte_core::registry::StaticRegistry; -use pg_debyte_core::types::{EncodeLimits, TypeKey}; -use serde::Serialize; -use uuid::Uuid as CoreUuid; - -#[derive(Debug, Serialize)] -struct MyRecord { - id: u32, - label: String, -} - -fn main() { - let record = MyRecord { - id: 1, - label: "demo".to_string(), - }; - let key = TypeKey { - type_id: CoreUuid::from_bytes([0x22; 16]), - schema_version: 1, - }; - let codec = BincodeCodec::new(1, 32 * 1024 * 1024); - let limits = EncodeLimits::new(32 * 1024 * 1024); - let registry = StaticRegistry::new(&[], &[]); - let actions: Vec = Vec::new(); - - let envelope = encode_to_envelope(&record, &codec, key, &actions, ®istry, &limits) - .expect("encode envelope"); - println!("{}", encode(envelope)); -} -``` - Usage: +Generate an envelope with the helper binary (type_id 0x11, schema_version 1): + ```bash -cargo run --bin demo_envelope +cargo run -p pg_debyte_tools --bin demo_envelope ``` ```sql diff --git a/examples/readme_by_id/Cargo.toml b/examples/readme_by_id/Cargo.toml new file mode 100644 index 0000000..b99cc22 --- /dev/null +++ b/examples/readme_by_id/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "readme_by_id" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib", "lib"] + +[features] +default = ["pg17"] +pg17 = ["pgrx/pg17"] + +[dependencies] +pgrx = { version = "0.16.1", default-features = false } +pg_debyte_core = { version = "0.2.0", path = "../../pg_debyte_core" } +pg_debyte_macros = { version = "0.2.0", path = "../../pg_debyte_macros" } +pg_debyte_pgrx = { version = "0.2.0", path = "../../pg_debyte_pgrx" } +serde = { version = "1.0", features = ["derive"] } +uuid = "1.8" diff --git a/examples/readme_by_id/src/lib.rs b/examples/readme_by_id/src/lib.rs new file mode 100644 index 0000000..c0d475f --- /dev/null +++ b/examples/readme_by_id/src/lib.rs @@ -0,0 +1,49 @@ +use pg_debyte_core::{BincodeCodec, DecodeError, StaticRegistry}; +use pg_debyte_macros::declare_decoder; +use pgrx::prelude::*; +use pgrx::JsonB; +use serde::{Deserialize, Serialize}; +use uuid::Uuid as CoreUuid; + +pg_module_magic!(); + +#[derive(Debug, Deserialize, Serialize)] +struct MyRecord { + id: u32, + label: String, +} + +const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x22; 16]); +const MY_SCHEMA_VERSION: u16 = 1; +const MY_CODEC_ID: u16 = 1; +const MY_CODEC: BincodeCodec = BincodeCodec::new(MY_CODEC_ID, 32 * 1024 * 1024); + +declare_decoder!( + MY_DECODER, + ty = MyRecord, + type_id = MY_TYPE_ID, + schema_version = MY_SCHEMA_VERSION, + codec = MY_CODEC, + codec_ty = BincodeCodec, + actions = [] +); + +static REGISTRY: StaticRegistry = StaticRegistry::new(&[&MY_DECODER], &[]); + +#[pg_guard] +pub unsafe extern "C-unwind" fn _PG_init() { + pg_debyte_pgrx::init_gucs(); + pg_debyte_pgrx::set_registry(®ISTRY); +} + +#[pg_extern] +fn bytea_to_json_by_id( + data: Vec, + type_id: pgrx::Uuid, + schema_version: i16, +) -> Result { + let limits = pg_debyte_pgrx::limits(); + let core_uuid = CoreUuid::from_bytes(*type_id.as_bytes()); + let value = pg_debyte_pgrx::decode_by_id(&data, core_uuid, schema_version, &limits)?; + Ok(JsonB(value)) +} diff --git a/examples/readme_envelope/Cargo.toml b/examples/readme_envelope/Cargo.toml new file mode 100644 index 0000000..e2f4453 --- /dev/null +++ b/examples/readme_envelope/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "readme_envelope" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib", "lib"] + +[features] +default = ["pg17"] +pg17 = ["pgrx/pg17"] + +[dependencies] +pgrx = { version = "0.16.1", default-features = false } +pg_debyte_core = { version = "0.2.0", path = "../../pg_debyte_core" } +pg_debyte_macros = { version = "0.2.0", path = "../../pg_debyte_macros" } +pg_debyte_pgrx = { version = "0.2.0", path = "../../pg_debyte_pgrx" } +serde = { version = "1.0", features = ["derive"] } +uuid = "1.8" diff --git a/examples/readme_envelope/src/lib.rs b/examples/readme_envelope/src/lib.rs new file mode 100644 index 0000000..ecc2269 --- /dev/null +++ b/examples/readme_envelope/src/lib.rs @@ -0,0 +1,44 @@ +use pg_debyte_core::{BincodeCodec, DecodeError, StaticRegistry}; +use pg_debyte_macros::declare_decoder; +use pgrx::prelude::*; +use pgrx::JsonB; +use serde::{Deserialize, Serialize}; +use uuid::Uuid as CoreUuid; + +pg_module_magic!(); + +#[derive(Debug, Deserialize, Serialize)] +struct MyRecord { + id: u32, + label: String, +} + +const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x11; 16]); +const MY_SCHEMA_VERSION: u16 = 1; +const MY_CODEC_ID: u16 = 1; +const MY_CODEC: BincodeCodec = BincodeCodec::new(MY_CODEC_ID, 32 * 1024 * 1024); + +declare_decoder!( + MY_DECODER, + ty = MyRecord, + type_id = MY_TYPE_ID, + schema_version = MY_SCHEMA_VERSION, + codec = MY_CODEC, + codec_ty = BincodeCodec, + actions = [] +); + +static REGISTRY: StaticRegistry = StaticRegistry::new(&[&MY_DECODER], &[]); + +#[pg_guard] +pub unsafe extern "C-unwind" fn _PG_init() { + pg_debyte_pgrx::init_gucs(); + pg_debyte_pgrx::set_registry(®ISTRY); +} + +#[pg_extern] +fn bytea_to_json_auto(data: Vec) -> Result { + let limits = pg_debyte_pgrx::limits(); + let value = pg_debyte_pgrx::decode_auto(&data, &limits)?; + Ok(JsonB(value)) +} diff --git a/examples/readme_known_schema/Cargo.toml b/examples/readme_known_schema/Cargo.toml new file mode 100644 index 0000000..72894ac --- /dev/null +++ b/examples/readme_known_schema/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "readme_known_schema" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib", "lib"] + +[features] +default = ["pg17"] +pg17 = ["pgrx/pg17"] + +[dependencies] +pgrx = { version = "0.16.1", default-features = false } +pg_debyte_core = { version = "0.2.0", path = "../../pg_debyte_core" } +pg_debyte_macros = { version = "0.2.0", path = "../../pg_debyte_macros" } +pg_debyte_pgrx = { version = "0.2.0", path = "../../pg_debyte_pgrx" } +serde = { version = "1.0", features = ["derive"] } +uuid = "1.8" diff --git a/examples/readme_known_schema/src/lib.rs b/examples/readme_known_schema/src/lib.rs new file mode 100644 index 0000000..910e370 --- /dev/null +++ b/examples/readme_known_schema/src/lib.rs @@ -0,0 +1,37 @@ +use pg_debyte_core::{BincodeCodec, StaticRegistry}; +use pg_debyte_macros::declare_know_schema; +use pgrx::prelude::*; +use serde::{Deserialize, Serialize}; +use uuid::Uuid as CoreUuid; + +pg_module_magic!(); + +#[derive(Debug, Deserialize, Serialize)] +struct MyRecord { + id: u32, + label: String, +} + +const MY_TYPE_ID: CoreUuid = CoreUuid::from_bytes([0x22; 16]); +const MY_SCHEMA_VERSION: u16 = 1; +const MY_CODEC_ID: u16 = 1; +const MY_CODEC: BincodeCodec = BincodeCodec::new(MY_CODEC_ID, 32 * 1024 * 1024); + +declare_know_schema!( + MY_DECODER, + ty = MyRecord, + type_id = MY_TYPE_ID, + schema_version = MY_SCHEMA_VERSION, + codec = MY_CODEC, + codec_ty = BincodeCodec, + actions = [], + fn_name = bytea_to_json_my_record +); + +static REGISTRY: StaticRegistry = StaticRegistry::new(&[&MY_DECODER], &[]); + +#[pg_guard] +pub unsafe extern "C-unwind" fn _PG_init() { + pg_debyte_pgrx::init_gucs(); + pg_debyte_pgrx::set_registry(®ISTRY); +} diff --git a/scripts/docker-ci-build-examples.sh b/scripts/docker-ci-build-examples.sh new file mode 100644 index 0000000..34e17dd --- /dev/null +++ b/scripts/docker-ci-build-examples.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +docker run --rm -t pg-debyte-ci bash -lc "\ + cargo build -p readme_known_schema --all-targets --features pg17 && \ + cargo build -p readme_by_id --all-targets --features pg17 && \ + cargo build -p readme_envelope --all-targets --features pg17 \ +" diff --git a/scripts/docker-ci-build-workspace.sh b/scripts/docker-ci-build-workspace.sh index ab4bff6..7498035 100755 --- a/scripts/docker-ci-build-workspace.sh +++ b/scripts/docker-ci-build-workspace.sh @@ -2,6 +2,6 @@ set -euo pipefail docker run --rm -t pg-debyte-ci bash -lc "\ - cargo build --workspace --all-targets --exclude pg_debyte_ext && \ + cargo build --workspace --all-targets --exclude pg_debyte_ext --exclude readme_known_schema --exclude readme_by_id --exclude readme_envelope && \ cargo build -p pg_debyte_ext --all-targets --features pg17 \ " diff --git a/scripts/host-examples.sh b/scripts/host-examples.sh new file mode 100755 index 0000000..10791e9 --- /dev/null +++ b/scripts/host-examples.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +cargo build -p readme_known_schema --all-targets --features pg17 +cargo build -p readme_by_id --all-targets --features pg17 +cargo build -p readme_envelope --all-targets --features pg17 From 1f55352734cb0deb13f9f0de0cb4969ba043e5c6 Mon Sep 17 00:00:00 2001 From: xaneets Date: Sat, 3 Jan 2026 12:05:27 +0300 Subject: [PATCH 8/9] fix pipeline --- .github/workflows/core-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core-tests.yml b/.github/workflows/core-tests.yml index c90a1ae..3042a9b 100644 --- a/.github/workflows/core-tests.yml +++ b/.github/workflows/core-tests.yml @@ -14,4 +14,4 @@ jobs: - name: Setup Rust uses: dtolnay/rust-toolchain@stable - name: Workspace tests (exclude pgrx/extension) - run: cargo test --workspace --exclude pg_debyte_ext --exclude pg_debyte_pgrx + run: cargo test --workspace --exclude pg_debyte_ext --exclude pg_debyte_pgrx --exclude readme_known_schema --exclude readme_by_id --exclude readme_envelope From 23597442e092d86842b3e77b06113bf2feeff3d4 Mon Sep 17 00:00:00 2001 From: xaneets Date: Sat, 3 Jan 2026 12:22:45 +0300 Subject: [PATCH 9/9] +x for script --- scripts/docker-ci-build-examples.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/docker-ci-build-examples.sh diff --git a/scripts/docker-ci-build-examples.sh b/scripts/docker-ci-build-examples.sh old mode 100644 new mode 100755