More journal work

Author: nicholasbishop

CHANGELOG.md

@@ -9,6 +9,7 @@
 * Added `Ext4::uuid` to get the filesystem UUID.
 * Made the `Corrupt` type opaque. It is no longer possible to `match` on
   specific types of corruption.
+* Added support for reading filesystems that weren't cleanly unmounted.
 
 ## 0.7.0
 

src/checksum.rs

@@ -54,6 +54,11 @@ impl Checksum {
         self.digest.update(data);
     }
 
+    /// Extend the digest with a big-endian `u32`.
+    pub(crate) fn update_u32_be(&mut self, data: u32) {
+        self.update(&data.to_be_bytes());
+    }
+
     /// Extend the digest with a little-endian `u16`.
     pub(crate) fn update_u16_le(&mut self, data: u16) {
         self.update(&data.to_le_bytes());

src/error.rs

@@ -235,6 +235,24 @@ pub(crate) enum CorruptKind {
     /// Journal superblock checksum is invalid.
     JournalSuperblockChecksum,
 
+    /// Journal block size does not match the filesystem block size.
+    JournalBlockSize,
+
+    /// Journal does not have the expected number of blocks.
+    JournalTruncated,
+
+    /// Journal first commit doesn't match the sequence number in the superblock.
+    JournalSequence,
+
+    /// Journal commit block checksum is invalid.
+    JournalCommitBlockChecksum,
+
+    /// Journal descriptor block checksum is invalid.
+    JournalDescriptorBlockChecksum,
+
+    /// Journal has a descriptor block that contains no tag with the last-tag flag set.
+    JournalDescriptorBlockMissingLastTag,
+
     /// An inode's checksum is invalid.
     InodeChecksum(
         /// Inode number.
@@ -351,6 +369,27 @@ impl Display for CorruptKind {
             Self::JournalSuperblockChecksum => {
                 write!(f, "journal superblock checksum is invalid")
             }
+            Self::JournalBlockSize => {
+                write!(
+                    f,
+                    "journal block size does not match filesystem block size"
+                )
+            }
+            Self::JournalTruncated => write!(f, "journal is truncated"),
+            Self::JournalSequence => write!(
+                f,
+                "journal's first commit doesn't match the expected sequence"
+            ),
+            Self::JournalCommitBlockChecksum => {
+                write!(f, "journal commit block checksum is invalid")
+            }
+            Self::JournalDescriptorBlockChecksum => {
+                write!(f, "journal descriptor block checksum is invalid")
+            }
+            Self::JournalDescriptorBlockMissingLastTag => write!(
+                f,
+                "a journal descriptor block has no tag with the last-tag flag set"
+            ),
             Self::InodeChecksum(inode) => {
                 write!(f, "invalid checksum for inode {inode}")
             }

src/journal.rs

@@ -6,34 +6,189 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-#[expect(unused)] // TODO
 mod block_header;
-#[expect(unused)] // TODO
+mod descriptor_block;
 mod superblock;
 
-use crate::{Ext4, Ext4Error};
+use crate::checksum::Checksum;
+use crate::inode::Inode;
+use crate::iters::file_blocks::FileBlocks;
+use crate::util::{read_u32be, usize_from_u32};
+use crate::{CorruptKind, Ext4, Ext4Error};
+use alloc::collections::BTreeMap;
+use alloc::vec;
+use block_header::{JournalBlockHeader, JournalBlockType};
+use descriptor_block::{
+    is_descriptor_block_checksum_valid, JournalDescriptorBlockTag,
+};
+use superblock::JournalSuperblock;
 
 #[derive(Debug)]
 pub(crate) struct Journal {
-    // TODO: add journal data.
+    block_map: BTreeMap<u64, u64>,
 }
 
 impl Journal {
-    /// Create an empty journal.
     pub(crate) fn empty() -> Self {
-        Self {}
+        Self {
+            block_map: BTreeMap::new(),
+        }
     }
 
-    /// Load a journal from the filesystem.
+    /// Load the journal.
+    ///
+    /// If the filesystem has no journal, an empty journal is returned.
+    ///
+    /// Note: ext4 is all little-endian, except for the journal, which
+    /// is all big-endian.
     pub(crate) fn load(fs: &Ext4) -> Result<Self, Ext4Error> {
-        let Some(_journal_inode) = fs.0.superblock.journal_inode else {
+        let Some(journal_inode) = fs.0.superblock.journal_inode else {
             // Return an empty journal if this filesystem does not have
             // a journal.
             return Ok(Self::empty());
         };
 
-        // TODO: actually load the journal.
+        let journal_inode = Inode::read(fs, journal_inode)?;
+        let superblock = JournalSuperblock::load(fs, &journal_inode)?;
 
-        Ok(Self {})
+        // Ensure the journal block size matches the rest of the
+        // filesystem.
+        let block_size = fs.0.superblock.block_size;
+        if superblock.block_size != block_size {
+            return Err(CorruptKind::JournalBlockSize.into());
+        }
+
+        // Get an iterator over the journal's block indices.
+        let journal_block_iter = FileBlocks::new(fs.clone(), &journal_inode)?;
+
+        // Skip forward to the start block.
+        let mut journal_block_iter =
+            journal_block_iter.skip(usize_from_u32(superblock.start_block));
+
+        // TODO: the loop below currently returns an error if something
+        // bad is encountered (e.g. a wrong checksum). We should
+        // actually still apply valid commits, and just stop reading the
+        // journal when bad data is encountered.
+
+        let mut block = vec![0; block_size.to_usize()];
+        let mut block_map = BTreeMap::new();
+        let mut uncommitted_block_map = BTreeMap::new();
+        let mut sequence = superblock.sequence;
+        while let Some(block_index) = journal_block_iter.next() {
+            let block_index = block_index?;
+
+            fs.read_from_block(block_index, 0, &mut block)?;
+
+            let Some(header) = JournalBlockHeader::read_bytes(&block)? else {
+                // Journal block magic is not present, so we've reached
+                // the end of the journal.
+                break;
+            };
+
+            if header.sequence != sequence {
+                return Err(CorruptKind::JournalSequence.into());
+            }
+
+            if header.block_type == JournalBlockType::DESCRIPTOR {
+                if !is_descriptor_block_checksum_valid(&superblock, &block) {
+                    return Err(
+                        CorruptKind::JournalDescriptorBlockChecksum.into()
+                    );
+                }
+
+                let tags =
+                    JournalDescriptorBlockTag::read_bytes_to_vec(&block[12..])
+                        .unwrap();
+
+                for tag in &tags {
+                    let block_index = journal_block_iter
+                        .next()
+                        .ok_or(CorruptKind::JournalTruncated)??;
+
+                    // TODO: is it a good idea to do this here, vs when
+                    // the data is actually needed?
+                    // TODO: either way, we definitely shouldn't fail if
+                    // not committed yet, right?
+                    let mut checksum = Checksum::new();
+                    checksum.update(superblock.uuid.as_bytes());
+                    checksum.update_u32_be(sequence);
+                    fs.read_from_block(block_index, 0, &mut block)?;
+                    checksum.update(&block);
+                    if checksum.finalize() != tag.checksum {
+                        // TODO
+                        panic!();
+                    }
+
+                    uncommitted_block_map.insert(tag.block_number, block_index);
+                }
+            } else if header.block_type == JournalBlockType::COMMIT {
+                if !is_commit_block_checksum_valid(&superblock, &block) {
+                    return Err(CorruptKind::JournalCommitBlockChecksum.into());
+                }
+
+                // Move the entries from `uncommitted_block_map` to `block_map`.
+                block_map.extend(uncommitted_block_map.iter());
+                uncommitted_block_map.clear();
+
+                // TODO: unwrap
+                sequence = sequence.checked_add(1).unwrap();
+            } else {
+                todo!()
+            }
+        }
+
+        Ok(Self { block_map })
+    }
+
+    /// Map from an absolute block index to a block in the journal.
+    ///
+    /// If the journal does not contain a replacement for the input
+    /// block, the input block is returned.
+    pub(crate) fn map_block_index(&self, block_index: u64) -> u64 {
+        *self.block_map.get(&block_index).unwrap_or(&block_index)
+    }
+}
+
+fn is_commit_block_checksum_valid(
+    superblock: &JournalSuperblock,
+    block: &[u8],
+) -> bool {
+    // The kernel documentation says that fields 0xc and 0xd contain the
+    // checksum type and size, but this is not correct. If the
+    // superblock features include `CHECKSUM_V3`, the type/size fields
+    // are both zero.
+
+    const CHECKSUM_OFFSET: usize = 16;
+    const CHECKSUM_SIZE: usize = 4;
+
+    let expected_checksum = read_u32be(block, CHECKSUM_OFFSET);
+
+    let mut checksum = Checksum::new();
+    checksum.update(superblock.uuid.as_bytes());
+    checksum.update(&block[..CHECKSUM_OFFSET]);
+    checksum.update(&[0; CHECKSUM_SIZE]);
+    checksum.update(&block[CHECKSUM_OFFSET + CHECKSUM_SIZE..]);
+
+    checksum.finalize() == expected_checksum
+}
+
+#[cfg(all(test, feature = "std"))]
+mod tests {
+    use crate::test_util::load_compressed_filesystem;
+    use alloc::rc::Rc;
+
+    #[test]
+    fn test_journal() {
+        let mut fs =
+            load_compressed_filesystem("test_disk_4k_block_journal.bin.zst");
+
+        let test_dir = "/dir500";
+
+        // With the journal in place, this directory exists.
+        assert!(fs.exists(test_dir).unwrap());
+
+        // Clear the journal, and verify that the directory no longer exists.
+        Rc::get_mut(&mut fs.0).unwrap().journal.block_map.clear();
+        assert!(!fs.exists(test_dir).unwrap());
     }
 }

src/journal/descriptor_block.rs

@@ -0,0 +1,103 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use crate::checksum::Checksum;
+use crate::error::{CorruptKind, Ext4Error};
+use crate::journal::superblock::JournalSuperblock;
+use crate::util::{read_u32be, u64_from_hilo};
+use alloc::vec::Vec;
+use bitflags::bitflags;
+
+pub(super) fn is_descriptor_block_checksum_valid(
+    superblock: &JournalSuperblock,
+    block: &[u8],
+) -> bool {
+    // OK to unwrap: minimum block length is 1024.
+    let checksum_offset = block.len().checked_sub(4).unwrap();
+    let expected_checksum = read_u32be(block, checksum_offset);
+    let mut checksum = Checksum::new();
+    checksum.update(superblock.uuid.as_bytes());
+    checksum.update(&block[..checksum_offset]);
+    checksum.update_u32_be(0);
+
+    checksum.finalize() == expected_checksum
+}
+
+// TODO: the kernel docs for this are a mess
+#[derive(Debug)]
+pub(super) struct JournalDescriptorBlockTag {
+    pub(super) block_number: u64,
+    pub(super) flags: JournalDescriptorBlockTagFlags,
+    pub(super) checksum: u32,
+    #[expect(dead_code)] // TODO
+    uuid: [u8; 16],
+}
+
+impl JournalDescriptorBlockTag {
+    pub(super) fn read_bytes(bytes: &[u8]) -> (Self, usize) {
+        // TODO: for now assuming the `incompat_features` assert above.
+
+        let t_blocknr = read_u32be(bytes, 0);
+        let t_flags = read_u32be(bytes, 4);
+        let t_blocknr_high = read_u32be(bytes, 8);
+        let t_checksum = read_u32be(bytes, 12);
+
+        let flags = JournalDescriptorBlockTagFlags::from_bits_retain(t_flags);
+        let mut size: usize = 16;
+
+        let mut uuid = [0; 16];
+        if !flags.contains(JournalDescriptorBlockTagFlags::UUID_OMITTED) {
+            // OK to unwrap: length is 16.
+            uuid = bytes[16..32].try_into().unwrap();
+            // TODO: unwrap
+            size = size.checked_add(16).unwrap();
+        }
+
+        (
+            Self {
+                block_number: u64_from_hilo(t_blocknr_high, t_blocknr),
+                flags,
+                checksum: t_checksum,
+                uuid,
+            },
+            size,
+        )
+    }
+
+    // TODO: this could be an iterator instead of allocating.
+    pub(super) fn read_bytes_to_vec(
+        mut bytes: &[u8],
+    ) -> Result<Vec<Self>, Ext4Error> {
+        let mut v = Vec::new();
+
+        while !bytes.is_empty() {
+            let (tag, size) = Self::read_bytes(bytes);
+            let is_end =
+                tag.flags.contains(JournalDescriptorBlockTagFlags::LAST_TAG);
+            v.push(tag);
+
+            if is_end {
+                return Ok(v);
+            }
+
+            bytes = &bytes[size..];
+        }
+
+        Err(CorruptKind::JournalDescriptorBlockMissingLastTag.into())
+    }
+}
+
+bitflags! {
+    #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
+    pub struct JournalDescriptorBlockTagFlags: u32 {
+        const ESCAPED = 0x1;
+        const UUID_OMITTED = 0x2;
+        const DELETED = 0x4;
+        const LAST_TAG = 0x8;
+    }
+}