Skip to content

Commit

Permalink
[blob stats todo]Tag io activity of sst/blob file write + stats + dbs…
Browse files Browse the repository at this point in the history 
…tress
  • Loading branch information
@hx235
hx235 committed Sep 6, 2023
1 parent 392d695 commit 170a665
Show file tree
Hide file tree
Showing 115 changed files with 1,589 additions and 919 deletions.
34 changes: 20 additions & 14 deletions db/blob/blob_file_builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,28 +34,30 @@ BlobFileBuilder::BlobFileBuilder(
VersionSet* versions, FileSystem* fs,
const ImmutableOptions* immutable_options,
const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
std::string db_id, std::string db_session_id, int job_id,
uint32_t column_family_id, const std::string& column_family_name,
Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint,
const WriteOptions* write_options, std::string db_id,
std::string db_session_id, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
BlobFileCompletionCallback* blob_callback,
BlobFileCreationReason creation_reason,
std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions)
: BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs,
immutable_options, mutable_cf_options, file_options,
db_id, db_session_id, job_id, column_family_id,
column_family_name, io_priority, write_hint, io_tracer,
blob_callback, creation_reason, blob_file_paths,
blob_file_additions) {}
write_options, db_id, db_session_id, job_id,
column_family_id, column_family_name, io_priority,
write_hint, io_tracer, blob_callback, creation_reason,
blob_file_paths, blob_file_additions) {}

BlobFileBuilder::BlobFileBuilder(
std::function<uint64_t()> file_number_generator, FileSystem* fs,
const ImmutableOptions* immutable_options,
const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
std::string db_id, std::string db_session_id, int job_id,
uint32_t column_family_id, const std::string& column_family_name,
Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint,
const WriteOptions* write_options, std::string db_id,
std::string db_session_id, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
BlobFileCompletionCallback* blob_callback,
BlobFileCreationReason creation_reason,
Expand All @@ -69,6 +71,7 @@ BlobFileBuilder::BlobFileBuilder(
blob_compression_type_(mutable_cf_options->blob_compression_type),
prepopulate_blob_cache_(mutable_cf_options->prepopulate_blob_cache),
file_options_(file_options),
write_options_(write_options),
db_id_(std::move(db_id)),
db_session_id_(std::move(db_session_id)),
job_id_(job_id),
Expand Down Expand Up @@ -214,7 +217,8 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(file), blob_file_paths_->back(), *file_options_,
immutable_options_->clock, io_tracer_, statistics,
immutable_options_->listeners,
// TODO: pass the correct Histogram for BLOB file writer
Histograms::HISTOGRAM_ENUM_MAX, immutable_options_->listeners,
immutable_options_->file_checksum_gen_factory.get(),
tmp_set.Contains(FileType::kBlobFile), false));

Expand All @@ -231,7 +235,7 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
expiration_range);

{
Status s = blob_log_writer->WriteHeader(header);
Status s = blob_log_writer->WriteHeader(*write_options_, header);

TEST_SYNC_POINT_CALLBACK(
"BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", &s);
Expand Down Expand Up @@ -296,7 +300,8 @@ Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob,

uint64_t key_offset = 0;

Status s = writer_->AddRecord(key, blob, &key_offset, blob_offset);
Status s =
writer_->AddRecord(*write_options_, key, blob, &key_offset, blob_offset);

TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AddRecord", &s);

Expand All @@ -321,7 +326,8 @@ Status BlobFileBuilder::CloseBlobFile() {
std::string checksum_method;
std::string checksum_value;

Status s = writer_->AppendFooter(footer, &checksum_method, &checksum_value);
Status s = writer_->AppendFooter(*write_options_, footer, &checksum_method,
&checksum_value);

TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AppendFooter", &s);

Expand Down
8 changes: 6 additions & 2 deletions db/blob/blob_file_builder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "rocksdb/advanced_options.h"
#include "rocksdb/compression_type.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/rocksdb_namespace.h"
#include "rocksdb/types.h"

Expand All @@ -36,7 +37,8 @@ class BlobFileBuilder {
BlobFileBuilder(VersionSet* versions, FileSystem* fs,
const ImmutableOptions* immutable_options,
const MutableCFOptions* mutable_cf_options,
const FileOptions* file_options, std::string db_id,
const FileOptions* file_options,
const WriteOptions* write_options, std::string db_id,
std::string db_session_id, int job_id,
uint32_t column_family_id,
const std::string& column_family_name,
Expand All @@ -51,7 +53,8 @@ class BlobFileBuilder {
BlobFileBuilder(std::function<uint64_t()> file_number_generator,
FileSystem* fs, const ImmutableOptions* immutable_options,
const MutableCFOptions* mutable_cf_options,
const FileOptions* file_options, std::string db_id,
const FileOptions* file_options,
const WriteOptions* write_options, std::string db_id,
std::string db_session_id, int job_id,
uint32_t column_family_id,
const std::string& column_family_name,
Expand Down Expand Up @@ -92,6 +95,7 @@ class BlobFileBuilder {
CompressionType blob_compression_type_;
PrepopulateBlobCache prepopulate_blob_cache_;
const FileOptions* file_options_;
const WriteOptions* write_options_;
const std::string db_id_;
const std::string db_session_id_;
int job_id_;
Expand Down
30 changes: 16 additions & 14 deletions db/blob/blob_file_builder_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ class BlobFileBuilderTest : public testing::Test {
FileSystem* fs_;
SystemClock* clock_;
FileOptions file_options_;
WriteOptions write_options_;
};

TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
Expand Down Expand Up @@ -144,8 +145,8 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, io_priority, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

Expand Down Expand Up @@ -229,8 +230,8 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, io_priority, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

Expand Down Expand Up @@ -317,8 +318,8 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, io_priority, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

Expand Down Expand Up @@ -372,8 +373,8 @@ TEST_F(BlobFileBuilderTest, Compression) {

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, io_priority, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

Expand Down Expand Up @@ -456,8 +457,8 @@ TEST_F(BlobFileBuilderTest, CompressionError) {

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, io_priority, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

Expand Down Expand Up @@ -536,8 +537,8 @@ TEST_F(BlobFileBuilderTest, Checksum) {

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, io_priority, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

Expand Down Expand Up @@ -594,6 +595,7 @@ class BlobFileBuilderIOErrorTest
std::unique_ptr<Env> mock_env_;
FileSystem* fs_;
FileOptions file_options_;
WriteOptions write_options_;
std::string sync_point_;
};

Expand Down Expand Up @@ -634,8 +636,8 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, io_priority, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

Expand Down
9 changes: 5 additions & 4 deletions db/blob/blob_file_cache_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ void WriteBlobFile(uint32_t column_family_id,
BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
expiration_range);

ASSERT_OK(blob_log_writer.WriteHeader(header));
ASSERT_OK(blob_log_writer.WriteHeader(WriteOptions(), header));

constexpr char key[] = "key";
constexpr char blob[] = "blob";
Expand All @@ -67,7 +67,8 @@ void WriteBlobFile(uint32_t column_family_id,
uint64_t key_offset = 0;
uint64_t blob_offset = 0;

ASSERT_OK(blob_log_writer.AddRecord(key, blob, &key_offset, &blob_offset));
ASSERT_OK(blob_log_writer.AddRecord(WriteOptions(), key, blob, &key_offset,
&blob_offset));

BlobLogFooter footer;
footer.blob_count = 1;
Expand All @@ -76,8 +77,8 @@ void WriteBlobFile(uint32_t column_family_id,
std::string checksum_method;
std::string checksum_value;

ASSERT_OK(
blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value));
ASSERT_OK(blob_log_writer.AppendFooter(WriteOptions(), footer,
&checksum_method, &checksum_value));
}

} // anonymous namespace
Expand Down
11 changes: 6 additions & 5 deletions db/blob/blob_file_reader_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,
BlobLogHeader header(column_family_id, compression, has_ttl,
expiration_range_header);

ASSERT_OK(blob_log_writer.WriteHeader(header));
ASSERT_OK(blob_log_writer.WriteHeader(WriteOptions(), header));

std::vector<std::string> compressed_blobs(num);
std::vector<Slice> blobs_to_write(num);
Expand Down Expand Up @@ -91,7 +91,8 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,

for (size_t i = 0; i < num; ++i) {
uint64_t key_offset = 0;
ASSERT_OK(blob_log_writer.AddRecord(keys[i], blobs_to_write[i], &key_offset,
ASSERT_OK(blob_log_writer.AddRecord(WriteOptions(), keys[i],
blobs_to_write[i], &key_offset,
&blob_offsets[i]));
}

Expand All @@ -101,8 +102,8 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,

std::string checksum_method;
std::string checksum_value;
ASSERT_OK(
blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value));
ASSERT_OK(blob_log_writer.AppendFooter(WriteOptions(), footer,
&checksum_method, &checksum_value));
}

// Creates a test blob file with a single blob in it. Note: this method
Expand Down Expand Up @@ -473,7 +474,7 @@ TEST_F(BlobFileReaderTest, Malformed) {
BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
expiration_range);

ASSERT_OK(blob_log_writer.WriteHeader(header));
ASSERT_OK(blob_log_writer.WriteHeader(WriteOptions(), header));
}

constexpr HistogramImpl* blob_file_read_hist = nullptr;
Expand Down
Loading

0 comments on commit 170a665

Please sign in to comment.