Skip to content

Commit b56e5e6

Browse files
committed
fix: fix sstable serde to avoid losing bloom filter
See PR #24025
1 parent fe22d62 commit b56e5e6

File tree

2 files changed

+46
-8
lines changed

2 files changed

+46
-8
lines changed

src/storage/src/hummock/sstable/mod.rs

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ impl From<SerdeSstable> for Sstable {
128128
}
129129

130130
/// [`Sstable`] is a handle for accessing SST.
131-
#[derive(Clone, Serialize, Deserialize)]
131+
#[derive(Clone, Deserialize)]
132132
#[serde(from = "SerdeSstable")]
133133
pub struct Sstable {
134134
pub id: HummockSstableObjectId,
@@ -137,6 +137,20 @@ pub struct Sstable {
137137
pub filter_reader: XorFilterReader,
138138
}
139139

140+
impl Serialize for Sstable {
141+
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
142+
where
143+
S: serde::Serializer,
144+
{
145+
let mut serde_sstable = SerdeSstable {
146+
id: self.id,
147+
meta: self.meta.clone(),
148+
};
149+
serde_sstable.meta.bloom_filter = self.filter_reader.encode_to_bytes();
150+
serde_sstable.serialize(serializer)
151+
}
152+
}
153+
140154
impl Debug for Sstable {
141155
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
142156
f.debug_struct("Sstable")

src/storage/src/hummock/sstable/xor_filter.rs

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,16 @@ impl Xor8FilterBuilder {
4747
};
4848
Self { key_hash_entries }
4949
}
50+
51+
fn build_from_xor8(xor_filter: &Xor8) -> Vec<u8> {
52+
let mut buf = Vec::with_capacity(8 + 4 + xor_filter.fingerprints.len() + 1);
53+
buf.put_u64_le(xor_filter.seed);
54+
buf.put_u32_le(xor_filter.block_length as u32);
55+
buf.put_slice(xor_filter.fingerprints.as_ref());
56+
// Add footer to tell which kind of filter. 254 indicates a xor8 filter.
57+
buf.put_u8(FOOTER_XOR8);
58+
buf
59+
}
5060
}
5161

5262
impl Xor16FilterBuilder {
@@ -124,13 +134,8 @@ impl FilterBuilder for Xor8FilterBuilder {
124134
});
125135

126136
let xor_filter = Xor8::from(&self.key_hash_entries);
127-
let mut buf = Vec::with_capacity(8 + 4 + xor_filter.fingerprints.len() + 1);
128-
buf.put_u64_le(xor_filter.seed);
129-
buf.put_u32_le(xor_filter.block_length as u32);
130-
buf.put_slice(xor_filter.fingerprints.as_ref());
131-
// Add footer to tell which kind of filter. 254 indicates a xor8 filter.
132-
buf.put_u8(FOOTER_XOR8);
133-
buf
137+
self.key_hash_entries.clear();
138+
Self::build_from_xor8(&xor_filter)
134139
}
135140

136141
fn approximate_len(&self) -> usize {
@@ -414,6 +419,25 @@ impl XorFilterReader {
414419
pub fn is_block_based_filter(&self) -> bool {
415420
matches!(self.filter, XorFilter::BlockXor16(_))
416421
}
422+
423+
pub fn encode_to_bytes(&self) -> Vec<u8> {
424+
match &self.filter {
425+
XorFilter::Xor8(filter) => Xor8FilterBuilder::build_from_xor8(filter),
426+
XorFilter::Xor16(filter) => Xor16FilterBuilder::build_from_xor16(filter),
427+
XorFilter::BlockXor16(reader) => {
428+
let mut data = Vec::with_capacity(4 + reader.filters.len() * 1024);
429+
for (_, filter) in &reader.filters {
430+
let block = Xor16FilterBuilder::build_from_xor16(filter);
431+
data.put_u32_le(block.len() as u32);
432+
data.extend(block);
433+
}
434+
// Add footer to tell which kind of filter. 253 indicates a blocked xor16 filter.
435+
data.put_u32_le(reader.filters.len() as u32);
436+
data.put_u8(FOOTER_BLOCKED_XOR16);
437+
data
438+
}
439+
}
440+
}
417441
}
418442

419443
impl Clone for XorFilterReader {

0 commit comments

Comments
 (0)