Skip to content
This repository has been archived by the owner on Nov 8, 2023. It is now read-only.

Commit

Permalink
Merge "Refactor off V2 Cow Ops" into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@zbw182
zbw182 authored and Gerrit Code Review committed Oct 5, 2023
2 parents 608e33f + c9770b2 commit adad3db
Show file tree
Hide file tree
Showing 9 changed files with 205 additions and 135 deletions.
55 changes: 26 additions & 29 deletions fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
namespace android {
namespace snapshot {

struct CowOperationV3;
typedef CowOperationV3 CowOperation;

static constexpr uint64_t kCowMagicNumber = 0x436f77634f572121ULL;
static constexpr uint32_t kCowVersionMajor = 2;
static constexpr uint32_t kCowVersionMinor = 0;
Expand Down Expand Up @@ -109,9 +112,8 @@ struct CowFooterOperation {
uint64_t num_ops;
} __attribute__((packed));

// Cow operations are currently fixed-size entries, but this may change if
// needed.
struct CowOperation {
// V2 version of COW. On disk format for older devices
struct CowOperationV2 {
// The operation code (see the constants and structures below).
uint8_t type;

Expand Down Expand Up @@ -146,42 +148,32 @@ struct CowOperation {
} __attribute__((packed));

// The on disk format of cow (currently == CowOperation)
struct CowOperationV2 {
struct CowOperationV3 {
// The operation code (see the constants and structures below).
uint8_t type;

// If this operation reads from the data section of the COW, this contains
// the compression type of that data (see constants below).
uint8_t compression;

// If this operation reads from the data section of the COW, this contains
// the length.
uint16_t data_length;

// The block of data in the new image that this operation modifies.
uint64_t new_block;
uint32_t new_block;

// The value of |source| depends on the operation code.
//
// For copy operations, this is a block location in the source image.
//
// For replace operations, this is a byte offset within the COW's data
// sections (eg, not landing within the header or metadata). It is an
// absolute position within the image.
//
// For zero operations (replace with all zeroes), this is unused and must
// be zero.
//
// For Label operations, this is the value of the applied label.
//
// For Cluster operations, this is the length of the following data region
// CopyOp: a 32-bit block location in the source image.
// ReplaceOp: an absolute byte offset within the COW's data section.
// XorOp: an absolute byte offset in the source image.
// ZeroOp: unused
// LabelOp: a 64-bit opaque identifier.
//
// For Xor operations, this is the byte location in the source image.
uint64_t source;
// For ops other than Label:
// Bits 47-62 are reserved and must be zero.
// A block is compressed if it’s data is < block_sz
uint64_t source_info;
} __attribute__((packed));

static_assert(sizeof(CowOperationV2) == sizeof(CowOperation));
static_assert(sizeof(CowOperation) == sizeof(CowFooterOperation));
static_assert(sizeof(CowOperationV2) == sizeof(CowFooterOperation));

static constexpr uint8_t kCowCopyOp = 1;
static constexpr uint8_t kCowReplaceOp = 2;
Expand All @@ -208,11 +200,14 @@ static constexpr uint8_t kCowReadAheadNotStarted = 0;
static constexpr uint8_t kCowReadAheadInProgress = 1;
static constexpr uint8_t kCowReadAheadDone = 2;

static constexpr uint64_t kCowOpSourceInfoDataMask = (1ULL << 48) - 1;
static constexpr uint64_t kCowOpSourceInfoCompressBit = (1ULL << 63);

static inline uint64_t GetCowOpSourceInfoData(const CowOperation* op) {
return op->source;
return op->source_info & kCowOpSourceInfoDataMask;
}
static inline bool GetCowOpSourceInfoCompression(const CowOperation* op) {
return op->compression != kCowCompressNone;
return !!(op->source_info & kCowOpSourceInfoCompressBit);
}

struct CowFooter {
Expand All @@ -236,10 +231,12 @@ struct BufferState {
// 2MB Scratch space used for read-ahead
static constexpr uint64_t BUFFER_REGION_DEFAULT_SIZE = (1ULL << 21);

std::ostream& operator<<(std::ostream& os, CowOperationV2 const& arg);

std::ostream& operator<<(std::ostream& os, CowOperation const& arg);

int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_size);
int64_t GetNextDataOffset(const CowOperation& op, uint32_t cluster_size);
int64_t GetNextOpOffset(const CowOperationV2& op, uint32_t cluster_size);
int64_t GetNextDataOffset(const CowOperationV2& op, uint32_t cluster_size);

// Ops that are internal to the Cow Format and not OTA data
bool IsMetadataOp(const CowOperation& op);
Expand Down
4 changes: 3 additions & 1 deletion fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,9 +165,10 @@ class CowReader final : public ICowReader {
void UpdateMergeOpsCompleted(int num_merge_ops) { header_.num_merge_ops += num_merge_ops; }

private:
bool ParseV2(android::base::borrowed_fd fd, std::optional<uint64_t> label);
bool PrepMergeOps();
uint64_t FindNumCopyops();
uint8_t GetCompressionType(const CowOperation* op);
uint8_t GetCompressionType();

android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
Expand All @@ -184,6 +185,7 @@ class CowReader final : public ICowReader {
std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc_;
ReaderFlags reader_flag_;
bool is_merge_{};
uint8_t compression_type_ = kCowCompressNone;
};

} // namespace snapshot
Expand Down
114 changes: 77 additions & 37 deletions fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,57 +14,97 @@
// limitations under the License.
//

#include <inttypes.h>
#include <libsnapshot/cow_format.h>
#include <sys/types.h>
#include <unistd.h>

#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <libsnapshot/cow_format.h>
#include "writer_v2.h"

namespace android {
namespace snapshot {

using android::base::unique_fd;

std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
os << "CowOperation(type:";
if (op.type == kCowCopyOp)
os << "kCowCopyOp, ";
else if (op.type == kCowReplaceOp)
os << "kCowReplaceOp, ";
else if (op.type == kCowZeroOp)
os << "kZeroOp, ";
else if (op.type == kCowFooterOp)
os << "kCowFooterOp, ";
else if (op.type == kCowLabelOp)
os << "kCowLabelOp, ";
else if (op.type == kCowClusterOp)
os << "kCowClusterOp ";
else if (op.type == kCowXorOp)
os << "kCowXorOp ";
else if (op.type == kCowSequenceOp)
os << "kCowSequenceOp ";
else if (op.type == kCowFooterOp)
os << "kCowFooterOp ";
else
os << (int)op.type << "?,";
os << "compression:";
if (op.compression == kCowCompressNone)
os << "kCowCompressNone, ";
else if (op.compression == kCowCompressGz)
os << "kCowCompressGz, ";
else if (op.compression == kCowCompressBrotli)
os << "kCowCompressBrotli, ";
else
os << (int)op.compression << "?, ";
os << "data_length:" << op.data_length << ",\t";
os << "new_block:" << op.new_block << ",\t";
std::ostream& EmitCowTypeString(std::ostream& os, uint8_t cow_type) {
switch (cow_type) {
case kCowCopyOp:
return os << "kCowCopyOp";
case kCowReplaceOp:
return os << "kCowReplaceOp";
case kCowZeroOp:
return os << "kZeroOp";
case kCowFooterOp:
return os << "kCowFooterOp";
case kCowLabelOp:
return os << "kCowLabelOp";
case kCowClusterOp:
return os << "kCowClusterOp";
case kCowXorOp:
return os << "kCowXorOp";
case kCowSequenceOp:
return os << "kCowSequenceOp";
default:
return os << (int)cow_type << "unknown";
}
}

std::ostream& operator<<(std::ostream& os, CowOperationV2 const& op) {
os << "CowOperationV2(";
EmitCowTypeString(os, op.type) << ", ";
switch (op.compression) {
case kCowCompressNone:
os << "uncompressed, ";
break;
case kCowCompressGz:
os << "gz, ";
break;
case kCowCompressBrotli:
os << "brotli, ";
break;
case kCowCompressLz4:
os << "lz4, ";
break;
case kCowCompressZstd:
os << "zstd, ";
break;
}
os << "data_length:" << op.data_length << ", ";
os << "new_block:" << op.new_block << ", ";
os << "source:" << op.source;
os << ")";
return os;
}

int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_ops) {
std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
os << "CowOperation(";
EmitCowTypeString(os, op.type);
if (op.type == kCowReplaceOp || op.type == kCowXorOp || op.type == kCowSequenceOp) {
if (op.source_info & kCowOpSourceInfoCompressBit) {
os << ", compressed";
} else {
os << ", uncompressed";
}
os << ", data_length:" << op.data_length;
}
if (op.type != kCowClusterOp && op.type != kCowSequenceOp && op.type != kCowLabelOp) {
os << ", new_block:" << op.new_block;
}
if (op.type == kCowXorOp || op.type == kCowReplaceOp || op.type == kCowCopyOp) {
os << ", source:" << (op.source_info & kCowOpSourceInfoDataMask);
} else if (op.type == kCowClusterOp) {
os << ", cluster_data:" << (op.source_info & kCowOpSourceInfoDataMask);
} else {
os << ", label:0x" << android::base::StringPrintf("%" PRIx64, op.source_info);
}
os << ")";
return os;
}

int64_t GetNextOpOffset(const CowOperationV2& op, uint32_t cluster_ops) {
if (op.type == kCowClusterOp) {
return op.source;
} else if ((op.type == kCowReplaceOp || op.type == kCowXorOp) && cluster_ops == 0) {
Expand All @@ -74,11 +114,11 @@ int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_ops) {
}
}

int64_t GetNextDataOffset(const CowOperation& op, uint32_t cluster_ops) {
int64_t GetNextDataOffset(const CowOperationV2& op, uint32_t cluster_ops) {
if (op.type == kCowClusterOp) {
return cluster_ops * sizeof(CowOperation);
return cluster_ops * sizeof(CowOperationV2);
} else if (cluster_ops == 0) {
return sizeof(CowOperation);
return sizeof(CowOperationV2);
} else {
return 0;
}
Expand Down
47 changes: 42 additions & 5 deletions fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ std::unique_ptr<CowReader> CowReader::CloneCowReader() {
cow->data_loc_ = data_loc_;
cow->block_pos_index_ = block_pos_index_;
cow->is_merge_ = is_merge_;
cow->compression_type_ = compression_type_;
return cow;
}

Expand Down Expand Up @@ -101,8 +102,44 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> lab
footer_ = parser.footer();
fd_size_ = parser.fd_size();
last_label_ = parser.last_label();
ops_ = parser.ops();
data_loc_ = parser.data_loc();
ops_ = std::make_shared<std::vector<CowOperation>>(parser.ops()->size());

// Translate the operation buffer from on disk to in memory
for (size_t i = 0; i < parser.ops()->size(); i++) {
const auto& v2_op = parser.ops()->at(i);

auto& new_op = ops_->at(i);
new_op.type = v2_op.type;
new_op.data_length = v2_op.data_length;

if (v2_op.new_block > std::numeric_limits<uint32_t>::max()) {
LOG(ERROR) << "Out-of-range new block in COW op: " << v2_op;
return false;
}
new_op.new_block = v2_op.new_block;

uint64_t source_info = v2_op.source;
if (new_op.type != kCowLabelOp) {
source_info &= kCowOpSourceInfoDataMask;
if (source_info != v2_op.source) {
LOG(ERROR) << "Out-of-range source value in COW op: " << v2_op;
return false;
}
}
if (v2_op.compression != kCowCompressNone) {
if (compression_type_ == kCowCompressNone) {
compression_type_ = v2_op.compression;
} else if (compression_type_ != v2_op.compression) {
LOG(ERROR) << "COW has mixed compression types which is not supported;"
<< " previously saw " << compression_type_ << ", got "
<< v2_op.compression << ", op: " << v2_op;
return false;
}
source_info |= kCowOpSourceInfoCompressBit;
}
new_op.source_info = source_info;
}

// If we're resuming a write, we're not ready to merge
if (label.has_value()) return true;
Expand Down Expand Up @@ -597,14 +634,14 @@ class CowDataStream final : public IByteStream {
size_t remaining_;
};

uint8_t CowReader::GetCompressionType(const CowOperation* op) {
return op->compression;
uint8_t CowReader::GetCompressionType() {
return compression_type_;
}

ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_size,
size_t ignore_bytes) {
std::unique_ptr<IDecompressor> decompressor;
switch (GetCompressionType(op)) {
switch (GetCompressionType()) {
case kCowCompressNone:
break;
case kCowCompressGz:
Expand All @@ -624,7 +661,7 @@ ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_
}
break;
default:
LOG(ERROR) << "Unknown compression type: " << GetCompressionType(op);
LOG(ERROR) << "Unknown compression type: " << GetCompressionType();
return -1;
}

Expand Down
Loading

0 comments on commit adad3db

Please sign in to comment.