Skip to content

Commit

Permalink
[clang][deps] Serialize JSON without creating intermediate objects
Browse files Browse the repository at this point in the history 
The dependency scanner uses the `llvm::json` library for outputting the dependency information. Until now, it created an in-memory representation of the dependency graph using the `llvm::json::Object` hierarchy. This not only creates unnecessary copies of the data, but also forces lexicographical ordering of attributes in the output, both of which I'd like to avoid. This patch adopts the `llvm::json::OStream` API instead and reorders the attribute printing logic such that the existing lexicographical ordering is preserved (for now).
  • Loading branch information
@jansvoboda11
jansvoboda11 committed Oct 9, 2024
1 parent 418920b commit c4c5b9c
Showing 1 changed file with 87 additions and 72 deletions.
159 changes: 87 additions & 72 deletions clang/tools/clang-scan-deps/ClangScanDeps.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,38 +330,46 @@ handleMakeDependencyToolResult(const std::string &Input,
return false;
}

static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
std::vector<llvm::StringRef> Strings;
for (auto &&I : Set)
Strings.push_back(I.getKey());
template <typename Container>
static auto toJSONStrings(llvm::json::OStream &JOS, Container &&Strings) {
return [&JOS, Strings = std::forward<Container>(Strings)]() {
for (StringRef Str : Strings)
JOS.value(Str);
};
}

static auto toJSONSorted(llvm::json::OStream &JOS,
const llvm::StringSet<> &Set) {
SmallVector<StringRef> Strings(Set.keys());
llvm::sort(Strings);
return llvm::json::Array(Strings);
return toJSONStrings(JOS, std::move(Strings));
}

// Technically, we don't need to sort the dependency list to get determinism.
// Leaving these be will simply preserve the import order.
static llvm::json::Array toJSONSorted(std::vector<ModuleID> V) {
static auto toJSONSorted(llvm::json::OStream &JOS, std::vector<ModuleID> V) {
llvm::sort(V);

llvm::json::Array Ret;
for (const ModuleID &MID : V)
Ret.push_back(llvm::json::Object(
{{"module-name", MID.ModuleName}, {"context-hash", MID.ContextHash}}));
return Ret;
return [&JOS, V = std::move(V)]() {
for (const ModuleID &MID : V)
JOS.object([&]() {
JOS.attribute("context-hash", StringRef(MID.ContextHash));
JOS.attribute("module-name", StringRef(MID.ModuleName));
});
};
}

static llvm::json::Array
toJSONSorted(llvm::SmallVector<Module::LinkLibrary, 2> &LinkLibs) {
llvm::sort(LinkLibs, [](const Module::LinkLibrary &lhs,
const Module::LinkLibrary &rhs) {
return lhs.Library < rhs.Library;
static auto toJSONSorted(llvm::json::OStream &JOS,
SmallVector<Module::LinkLibrary, 2> LinkLibs) {
llvm::sort(LinkLibs, [](const auto &LHS, const auto &RHS) {
return LHS.Library < RHS.Library;
});

llvm::json::Array Ret;
for (const Module::LinkLibrary &LL : LinkLibs)
Ret.push_back(llvm::json::Object(
{{"link-name", LL.Library}, {"isFramework", LL.IsFramework}}));
return Ret;
return [&JOS, LinkLibs = std::move(LinkLibs)]() {
for (const auto &LL : LinkLibs)
JOS.object([&]() {
JOS.attribute("isFramework", LL.IsFramework);
JOS.attribute("link-name", StringRef(LL.Library));
});
};
}

// Thread safe.
Expand Down Expand Up @@ -450,58 +458,65 @@ class FullDeps {
ModuleIDs.push_back(M.first);
llvm::sort(ModuleIDs);

using namespace llvm::json;

Array OutModules;
for (auto &&ModID : ModuleIDs) {
auto &MD = Modules[ModID];
Object O{{"name", MD.ID.ModuleName},
{"context-hash", MD.ID.ContextHash},
{"file-deps", toJSONSorted(MD.FileDeps)},
{"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
{"clang-modulemap-file", MD.ClangModuleMapFile},
{"command-line", MD.getBuildArguments()},
{"link-libraries", toJSONSorted(MD.LinkLibraries)}};
OutModules.push_back(std::move(O));
}

Array TUs;
for (auto &&I : Inputs) {
Array Commands;
if (I.DriverCommandLine.empty()) {
for (const auto &Cmd : I.Commands) {
Object O{
{"input-file", I.FileName},
{"clang-context-hash", I.ContextHash},
{"file-deps", I.FileDeps},
{"clang-module-deps", toJSONSorted(I.ModuleDeps)},
{"executable", Cmd.Executable},
{"command-line", Cmd.Arguments},
};
Commands.push_back(std::move(O));
llvm::json::OStream JOS(OS, /*IndentSize=*/2);

JOS.object([&]() {
JOS.attributeArray("modules", [&]() {
for (auto &&ModID : ModuleIDs) {
auto &MD = Modules[ModID];
JOS.object([&]() {
JOS.attributeArray("clang-module-deps",
toJSONSorted(JOS, MD.ClangModuleDeps));
JOS.attribute("clang-modulemap-file",
StringRef(MD.ClangModuleMapFile));
JOS.attributeArray("command-line",
toJSONStrings(JOS, MD.getBuildArguments()));
JOS.attribute("context-hash", StringRef(MD.ID.ContextHash));
JOS.attributeArray("file-deps", toJSONSorted(JOS, MD.FileDeps));
JOS.attributeArray("link-libraries",
toJSONSorted(JOS, MD.LinkLibraries));
JOS.attribute("name", StringRef(MD.ID.ModuleName));
});
}
} else {
Object O{
{"input-file", I.FileName},
{"clang-context-hash", I.ContextHash},
{"file-deps", I.FileDeps},
{"clang-module-deps", toJSONSorted(I.ModuleDeps)},
{"executable", "clang"},
{"command-line", I.DriverCommandLine},
};
Commands.push_back(std::move(O));
}
TUs.push_back(Object{
{"commands", std::move(Commands)},
});
}

Object Output{
{"modules", std::move(OutModules)},
{"translation-units", std::move(TUs)},
};

OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
JOS.attributeArray("translation-units", [&]() {
for (auto &&I : Inputs) {
JOS.object([&]() {
JOS.attributeArray("commands", [&]() {
if (I.DriverCommandLine.empty()) {
for (const auto &Cmd : I.Commands) {
JOS.object([&]() {
JOS.attribute("clang-context-hash",
StringRef(I.ContextHash));
JOS.attributeArray("clang-module-deps",
toJSONSorted(JOS, I.ModuleDeps));
JOS.attributeArray("command-line",
toJSONStrings(JOS, Cmd.Arguments));
JOS.attribute("executable", StringRef(Cmd.Executable));
JOS.attributeArray("file-deps",
toJSONStrings(JOS, I.FileDeps));
JOS.attribute("input-file", StringRef(I.FileName));
});
}
} else {
JOS.object([&]() {
JOS.attribute("clang-context-hash", StringRef(I.ContextHash));
JOS.attributeArray("clang-module-deps",
toJSONSorted(JOS, I.ModuleDeps));
JOS.attributeArray("command-line",
toJSONStrings(JOS, I.DriverCommandLine));
JOS.attribute("executable", "clang");
JOS.attributeArray("file-deps",
toJSONStrings(JOS, I.FileDeps));
JOS.attribute("input-file", StringRef(I.FileName));
});
}
});
});
}
});
});
}

private:
Expand Down

0 comments on commit c4c5b9c

Please sign in to comment.