diff --git a/include/klee/Support/OptionCategories.h b/include/klee/Support/OptionCategories.h index 773800edfe..e559e6effe 100644 --- a/include/klee/Support/OptionCategories.h +++ b/include/klee/Support/OptionCategories.h @@ -26,6 +26,7 @@ extern llvm::cl::OptionCategory SeedingCat; extern llvm::cl::OptionCategory SolvingCat; extern llvm::cl::OptionCategory TerminationCat; extern llvm::cl::OptionCategory TestGenCat; +extern llvm::cl::OptionCategory AgentCat; } // namespace klee #endif /* KLEE_OPTIONCATEGORIES_H */ diff --git a/lib/Core/CMakeLists.txt b/lib/Core/CMakeLists.txt index 9f654afc0f..f92ac627bc 100644 --- a/lib/Core/CMakeLists.txt +++ b/lib/Core/CMakeLists.txt @@ -23,6 +23,7 @@ add_library(kleeCore ImpliedValue.cpp Memory.cpp MemoryManager.cpp + MetricCollectorAndSerializer.cpp ObjectManager.cpp PForest.cpp MockBuilder.cpp @@ -30,7 +31,9 @@ add_library(kleeCore Searcher.cpp SeedInfo.cpp SeedMap.cpp + ServerConnection.cpp SpecialFunctionHandler.cpp + StatisticQueue.cpp StatsTracker.cpp TargetCalculator.cpp TargetedExecutionReporter.cpp @@ -52,7 +55,12 @@ target_link_libraries(kleeCore PRIVATE llvm_config(kleeCore "${USE_LLVM_SHARED}" core executionengine mcjit native support) target_link_libraries(kleeCore PRIVATE ${SQLite3_LIBRARIES}) + +find_package(CURL REQUIRED) + target_include_directories(kleeCore SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS}) target_include_directories(kleeCore PRIVATE ${KLEE_INCLUDE_DIRS}) target_compile_options(kleeCore PRIVATE ${KLEE_COMPONENT_CXX_FLAGS}) target_compile_definitions(kleeCore PRIVATE ${KLEE_COMPONENT_CXX_DEFINES}) +target_include_directories(kleeCore SYSTEM PRIVATE ${CURL_INCLUDE_DIRS}) +target_link_libraries(kleeCore PRIVATE CURL::libcurl) diff --git a/lib/Core/Executor.cpp b/lib/Core/Executor.cpp index 9c37230634..b2a152d333 100644 --- a/lib/Core/Executor.cpp +++ b/lib/Core/Executor.cpp @@ -11,6 +11,7 @@ #include "AddressSpace.h" #include "CXXTypeSystem/CXXTypeManager.h" +#include "CallPathManager.h" #include "ConstructStorage.h" #include "CoreStats.h" #include "DistanceCalculator.h" @@ -20,11 +21,14 @@ #include "ImpliedValue.h" #include "Memory.h" #include "MemoryManager.h" +#include "MetricCollectorAndSerializer.h" #include "PForest.h" #include "PTree.h" #include "Searcher.h" #include "SeedInfo.h" +#include "ServerConnection.h" #include "SpecialFunctionHandler.h" +#include "StatisticQueue.h" #include "StatsTracker.h" #include "TargetCalculator.h" #include "TargetManager.h" @@ -72,6 +76,7 @@ #include "klee/Support/RoundingModeUtil.h" #include "klee/System/MemoryUsage.h" #include "klee/System/Time.h" +#include #include "CodeEvent.h" #include "CodeLocation.h" @@ -100,6 +105,7 @@ #include #include +#include #include #include #include @@ -136,6 +142,9 @@ cl::OptionCategory TestGenCat("Test generation options", cl::OptionCategory LazyInitCat("Lazy initialization option", "These options configure lazy initialization."); +cl::OptionCategory AgentCat("Agent options", + "These options specify agent settings."); + cl::opt UseAdvancedTypeSystem( "use-advanced-type-system", cl::desc("Use advanced information about type system from " @@ -338,6 +347,25 @@ cl::opt AllExternalWarnings( "as opposed to once per function (default=false)"), cl::cat(ExtCallsCat)); +/*** Agent options ***/ + +cl::opt UrlUID("url-uid", + cl::init("http://localhost:8080/new-session"), + cl::desc("The option allows to specify the url " + "from which can get the UID"), + cl::cat(AgentCat)); + +cl::opt MetricsUID( + "metrics-uid", cl::init("http://localhost:8080/metrics"), + cl::desc("The option allows to specify the url where can save metrics"), + cl::cat(AgentCat)); + +cl::opt + DeltaTime("delta-time", cl::init(100), + cl::desc("Using the option, you specify the time in milliseconds " + "at which statistics will be sent (default = 100)"), + cl::cat(AgentCat)); + /*** Seeding options ***/ cl::opt AlwaysOutputSeeds( @@ -489,9 +517,9 @@ Executor::Executor(LLVMContext &ctx, const InterpreterOptions &opts, InterpreterHandler *ih) : Interpreter(opts), interpreterHandler(ih), searcher(nullptr), externalDispatcher(new ExternalDispatcher(ctx)), statsTracker(0), - pathWriter(0), symPathWriter(0), - specialFunctionHandler(0), timers{time::Span(TimerInterval)}, - guidanceKind(opts.Guidance), codeGraphInfo(new CodeGraphInfo()), + pathWriter(0), symPathWriter(0), specialFunctionHandler(0), + timers{time::Span(TimerInterval)}, guidanceKind(opts.Guidance), + codeGraphInfo(new CodeGraphInfo()), distanceCalculator(new DistanceCalculator(*codeGraphInfo)), targetCalculator(new TargetCalculator(*codeGraphInfo)), targetManager(new TargetManager(guidanceKind, *distanceCalculator, @@ -4697,8 +4725,45 @@ void Executor::run(ExecutionState *initialState) { objectManager->initialUpdate(); + MetricCollectorAndSerializer mc; + std::thread consumer; + std::thread producer; + StatisticQueue StatQ; + + ServerConnection scForUID(UrlUID); + scForUID.getUIDFromServer(); + + ServerConnection sc(MetricsUID); + sc.setUID(scForUID.getUID()); + + std::atomic shouldStop = false; + if (sc.getUID() != "ServerNotValid") { + consumer = std::thread([&StatQ, &mc, &sc, &shouldStop]() { + while (!shouldStop) { + if (!StatQ.empty()) { + auto data = StatQ.pop(); + sc.PostRequest(mc.GetJson(std::move(data), sc.getUID())); + } + } + }); + } + auto startTime = std::chrono::high_resolution_clock::now(); + // main interpreter loop while (!haltExecution && !searcher->empty()) { + + if (sc.getUID() != "ServerNotValid") { + auto currentTime = std::chrono::high_resolution_clock::now(); + auto elapsedTime = std::chrono::duration_cast( + currentTime - startTime) + .count(); + if (elapsedTime >= 100) { + auto localStatisticMap = objectManager->StatisticMap; + StatQ.push(mc.getCurrentMetric(localStatisticMap)); + startTime = currentTime; + } + } + auto action = searcher->selectAction(); executeAction(action); objectManager->updateSubscribers(); @@ -4707,6 +4772,14 @@ void Executor::run(ExecutionState *initialState) { objectManager->updateSubscribers(); } } + auto a = mc.getCurrentMetric(objectManager->StatisticMap); + StatQ.push(a); + + shouldStop = true; + if (sc.getUID() != "ServerNotValid") { + consumer.join(); + producer.join(); + } if (guidanceKind == GuidanceKind::ErrorGuidance) { reportProgressTowardsTargets(); @@ -6829,8 +6902,7 @@ void Executor::executeMakeSymbolic(ExecutionState &state, (!AllowSeedTruncation && obj->numBytes > moSize))) { std::stringstream msg; msg << "replace size mismatch: " << mo->name << "[" << moSize - << "]" - << " vs " << obj->name << "[" << obj->numBytes << "]" + << "]" << " vs " << obj->name << "[" << obj->numBytes << "]" << " in test\n"; terminateStateOnUserError(state, msg.str()); @@ -7252,8 +7324,7 @@ void Executor::logState(const ExecutionState &state, int id, object.first->getSizeExpr()->print(*f); *f << "\n"; } - *f << state.symbolics.size() << " symbolics total. " - << "Symbolics:\n"; + *f << state.symbolics.size() << " symbolics total. " << "Symbolics:\n"; size_t sc = 0; for (const auto &symbolic : state.symbolics) { *f << "Symbolic number " << sc++ << "\n"; @@ -7503,8 +7574,8 @@ bool Executor::getSymbolicSolution(const ExecutionState &state, KTest &res) { std::back_inserter(symbolics), isReproducible); } - // we cannot be sure that an irreproducible state proves the presence of an - // error + // we cannot be sure that an irreproducible state proves the presence of + // an error if (uninitObjects.size() > 0 || state.symbolics.size() != symbolics.size()) { state.error = ReachWithError::None; } else if (FunctionCallReproduce != "" && @@ -7586,8 +7657,8 @@ bool Executor::getSymbolicSolution(const ExecutionState &state, KTest &res) { ref symcretizedAddress = sizeSymcrete->addressSymcrete.symcretized; - /* Receive address array linked with this size array to request address - * concretization. */ + /* Receive address array linked with this size array to request + * address concretization. */ ref condcretized = concretizations.at(symcrete->symcretized); uint64_t newSize = cast(condcretized)->getZExtValue(); diff --git a/lib/Core/Executor.h b/lib/Core/Executor.h index e990524a45..9a2bd5ac91 100644 --- a/lib/Core/Executor.h +++ b/lib/Core/Executor.h @@ -92,6 +92,7 @@ class MemoryManager; class MemoryObject; class ObjectState; class PForest; +class Delta; class Searcher; class SeedInfo; class SpecialFunctionHandler; diff --git a/lib/Core/MetricCollectorAndSerializer.cpp b/lib/Core/MetricCollectorAndSerializer.cpp new file mode 100644 index 0000000000..481e60a19d --- /dev/null +++ b/lib/Core/MetricCollectorAndSerializer.cpp @@ -0,0 +1,99 @@ +//===--MetricCollectorAndSerializer.cpp---------------------------*-C++ -*-===// +// +// The KLEE Symbolic Virtual Machine +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "MetricCollectorAndSerializer.h" +#include "CoreStats.h" + +using namespace klee; + +const double NANOSECONDS_PER_SECOND = 1000000000.0; + +std::vector MetricCollectorAndSerializer::GetJson( + const std::unordered_map> &DelMap, + std::string UID) { + + std::vector jsonArray = SerializeDelMap(DelMap, UID); + + previousMap = jsonArray; + + return jsonArray; +} + +std::vector MetricCollectorAndSerializer::SerializeDelMap( + const std::unordered_map> &DelMap, + const std::string UID) { + + std::vector jsonArray; + + for (const auto &funPair : DelMap) { + auto funName = funPair.first->getName(); + const auto &metricsMap = funPair.second; + + for (const auto &metricPair : metricsMap) { + const std::string &metricName = metricPair.first; + uint64_t prev; + auto count = metricPair.second; + for (const auto &jsonObject : previousMap) { + if (jsonObject["params"]["funName"] == funName && + jsonObject["name"] == metricName) { + prev = jsonObject["params"]["value"]; + break; + } + } + + if (count - prev != 0) { + if (metricName == "SolverTime") { + double solverCount = + static_cast(count) / NANOSECONDS_PER_SECOND; + solverCount = static_cast(round(solverCount * 100)) / 100; + jsonArray.push_back({{"guid", UID}, + {"name", metricName}, + {"params", + {{"funName", funName}, + {"type", "double"}, + {"value", solverCount}, + {"transitive", false}}}}); + } else { + jsonArray.push_back({{"guid", UID}, + {"name", metricName}, + {"params", + {{"funName", funName}, + {"type", "int"}, + {"value", count}, + {"transitive", false}}}}); + } + } + } + } + + return jsonArray; +} + +std::unordered_map> +MetricCollectorAndSerializer::getCurrentMetric( + std::unordered_map StatMap) { + + std::unordered_map> + DelMap; + + std::lock_guard lock(StatisticMapMutex); + for (const auto &pair : StatMap) { + CallPathNode *cpn = pair.first; + DelMap[cpn->function]["Instructions"] += + pair.second->getValue(stats::instructions); + DelMap[cpn->function]["SolverTime"] += + pair.second->getValue(stats::solverTime); + DelMap[cpn->function]["Forks"] += pair.second->getValue(stats::forks); + } + + return DelMap; +} diff --git a/lib/Core/MetricCollectorAndSerializer.h b/lib/Core/MetricCollectorAndSerializer.h new file mode 100644 index 0000000000..325ad42992 --- /dev/null +++ b/lib/Core/MetricCollectorAndSerializer.h @@ -0,0 +1,44 @@ +//===--MetricCollectorAndSerializer.h-----------------------------*-C++ -*-===// +// +// The KLEE Symbolic Virtual Machine +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef KLEE_GETDELTA_H +#define KLEE_GETDELTA_H + +#include "CallPathManager.h" +#include "nlohmann/json.hpp" +#include "llvm/IR/Function.h" + +#include + +namespace klee { + +class MetricCollectorAndSerializer { +private: + std::vector previousMap; + std::mutex StatisticMapMutex; + std::unordered_map> Delta; + + std::vector SerializeDelMap( + const std::unordered_map> &DelMap, + const std::string UID); + +public: + std::vector GetJson( + const std::unordered_map> &DelMap, + const std::string UID); + + std::unordered_map> + getCurrentMetric(std::unordered_map); +}; +} // namespace klee + +#endif /* KLEE_DELTA_H */ diff --git a/lib/Core/ObjectManager.cpp b/lib/Core/ObjectManager.cpp index 40a66eee23..d425ef1b42 100644 --- a/lib/Core/ObjectManager.cpp +++ b/lib/Core/ObjectManager.cpp @@ -1,9 +1,9 @@ #include "ObjectManager.h" +#include "CallPathManager.h" #include "CoreStats.h" #include "PForest.h" #include "TargetManager.h" - #include "klee/Module/KModule.h" using namespace llvm; @@ -26,6 +26,8 @@ void ObjectManager::setCurrentState(ExecutionState *_current) { assert(current == nullptr); current = _current; statesUpdated = true; + InfoStackFrame &sf = current->stack.infoStack().back(); + StatisticMap[sf.callPathNode] = &sf.callPathNode->statistics; } ExecutionState *ObjectManager::branchState(ExecutionState *state, diff --git a/lib/Core/ObjectManager.h b/lib/Core/ObjectManager.h index 4d3acd78f6..4e32cffddd 100644 --- a/lib/Core/ObjectManager.h +++ b/lib/Core/ObjectManager.h @@ -1,11 +1,13 @@ #ifndef KLEE_OBJECTMANAGER_H #define KLEE_OBJECTMANAGER_H +#include "CallPathManager.h" #include "ExecutionState.h" #include "PForest.h" #include "klee/ADT/Ref.h" #include "klee/Core/BranchTypes.h" #include "klee/Module/KModule.h" +#include #include @@ -49,6 +51,7 @@ class ObjectManager { ObjectManager(); ~ObjectManager(); + std::unordered_map StatisticMap; void addSubscriber(Subscriber *); void addProcessForest(PForest *); diff --git a/lib/Core/ServerConnection.cpp b/lib/Core/ServerConnection.cpp new file mode 100644 index 0000000000..da9184c8dc --- /dev/null +++ b/lib/Core/ServerConnection.cpp @@ -0,0 +1,77 @@ +//===-- ServerConnection.cpp-------------------------------------*- C++ -*-===// +// +// The KLEE Symbolic Virtual Machine +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ServerConnection.h" +#include + +using namespace klee; + +ServerConnection::ServerConnection(const std::string &serverUrl) + : url(serverUrl), curl(curl_easy_init()) { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + if (!curl) { + llvm::errs() << "Failed to initialize curl" << "\n"; + } + curl_global_init(CURL_GLOBAL_ALL); +} + +ServerConnection::~ServerConnection() { + curl_easy_cleanup(curl); + curl_global_cleanup(); +} + +void ServerConnection::PostRequest(const std::vector &metrics) { + CURLcode res; + + if (curl) { + nlohmann::json jsonData = metrics; + std::string jsonString = jsonData.dump(); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, jsonString.c_str()); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L); + curl_easy_setopt( + curl, CURLOPT_WRITEFUNCTION, + +[](void *, size_t size, size_t nmemb, void *) -> size_t { + return size * nmemb; + }); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, nullptr); + struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) + llvm::errs() << "curl_easy_perform() failed: " << curl_easy_strerror(res) + << "\n"; + } +}; + +size_t write_data(void *ptr, size_t size, size_t nmemb, void *userdata) { + std::string *buffer = static_cast(userdata); + buffer->append(static_cast(ptr), size * nmemb); + return size * nmemb; +} + +void ServerConnection::getUIDFromServer() { + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &UID); + + CURLcode res = curl_easy_perform(curl); + if (res != CURLE_OK) { + UID = "ServerNotValid"; + llvm::errs() << "curl_easy_perform() failed: " << curl_easy_strerror(res) + << "\n"; + } +} + +std::string ServerConnection::getUID() { return UID; } + +void ServerConnection::setUID(const std::string &newUID) { UID = newUID; } diff --git a/lib/Core/ServerConnection.h b/lib/Core/ServerConnection.h new file mode 100644 index 0000000000..4381e20bc0 --- /dev/null +++ b/lib/Core/ServerConnection.h @@ -0,0 +1,35 @@ +//===-- ServerConnection.h---------------------------------------*- C++ -*-===// +// +// The KLEE Symbolic Virtual Machine +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef KLEE_SERVERCONNECTION_H +#define KLEE_SERVERCONNECTION_H + +#include "nlohmann/json.hpp" +#include + +namespace klee { + +class ServerConnection { + +public: + ServerConnection(const std::string &serverUrl); + ~ServerConnection(); + void PostRequest(const std::vector &metrics); + void getUIDFromServer(); + std::string getUID(); + void setUID(const std::string &newUID); + +private: + std::string url; + CURL *curl; + std::string UID; +}; +} // namespace klee + +#endif diff --git a/lib/Core/StatisticQueue.cpp b/lib/Core/StatisticQueue.cpp new file mode 100644 index 0000000000..5901056b0b --- /dev/null +++ b/lib/Core/StatisticQueue.cpp @@ -0,0 +1,37 @@ +//===-- StatisticQueue.cpp---------------------------------------*- C++ -*-===// +// +// The KLEE Symbolic Virtual Machine +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "StatisticQueue.h" + +using namespace klee; + +void StatisticQueue::push( + const std::unordered_map> &value) { + std::unique_lock lock(mutex); + StatQueue.push(value); + cond_var.notify_one(); +} + +std::unordered_map> +StatisticQueue::pop() { + std::unique_lock lock(mutex); + cond_var.wait(lock, [this] { return !StatQueue.empty(); }); + std::unordered_map> + value = std::move(StatQueue.front()); + StatQueue.pop(); + return value; +} + +bool StatisticQueue::empty() { + + std::unique_lock lock(mutex); + return StatQueue.empty(); +} diff --git a/lib/Core/StatisticQueue.h b/lib/Core/StatisticQueue.h new file mode 100644 index 0000000000..b30f1c1dcd --- /dev/null +++ b/lib/Core/StatisticQueue.h @@ -0,0 +1,41 @@ +//===-- StatisticQueue.h-----------------------------------------*- C++ -*-===// +// +// The KLEE Symbolic Virtual Machine +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/Function.h" +#include +#include + +#ifndef KLEE_STATISTICQUEUE_H +#define KLEE_STATISTICQUEUE_H + +namespace klee { + +class StatisticQueue { + +private: + std::mutex mutex; + std::condition_variable cond_var; + std::queue>> + StatQueue; + +public: + void + push(const std::unordered_map> &value); + + std::unordered_map> + pop(); + bool empty(); +}; + +} // namespace klee + +#endif diff --git a/tools/klee/main.cpp b/tools/klee/main.cpp index 9cd06abbf7..618366b12c 100644 --- a/tools/klee/main.cpp +++ b/tools/klee/main.cpp @@ -50,8 +50,11 @@ #include "llvm/Support/TargetSelect.h" #include +#include #include +#include #include +#include #include #include #include @@ -646,7 +649,9 @@ static std::vector rules() { #define TTYPE(N, I, S) \ if ((I) > (unsigned int)StateTerminationType::SOLVERERR && \ (I) < (unsigned int)StateTerminationType::PROGERR) { \ - ret.push_back(RuleJson{#N, {"Program error"}, {}, {}}); \ + ret.push_back(RuleJson { \ + #N, {"Program error"}, {}, {} \ + }); \ } #define TTMARK(N, I) @@ -1557,8 +1562,8 @@ static int run_klee_on_function(int pArgc, char **pArgv, char **pEnvp, KTest *out = *it; interpreter->setReplayKTest(out); llvm::errs() << "KLEE: replaying: " << *it << " (" << kTest_numBytes(out) - << " bytes)" - << " (" << ++i << "/" << kTestFiles.size() << ")\n"; + << " bytes)" << " (" << ++i << "/" << kTestFiles.size() + << ")\n"; // XXX should put envp in .ktest ? interpreter->runFunctionAsMain(mainFn, out->numArgs, out->args, pEnvp); if (interrupted) @@ -2451,6 +2456,16 @@ int main(int argc, char **argv, char **envp) { uint64_t instructions = *theStatisticManager->getStatisticByName("Instructions"); uint64_t forks = *theStatisticManager->getStatisticByName("Forks"); + uint64_t solverTime = + (*theStatisticManager->getStatisticByName("SolverTime")) / 1000000000; + + auto [solverH, solverM, solverS] = time::seconds(solverTime).toHMS(); + std::stringstream ss; + ss << std::setw(2) << std::setfill('0') << solverH << ':' << std::setw(2) + << std::setfill('0') << +solverM << ':' << std::setw(2) + << std::setfill('0') << +solverS; + + handler->getInfoStream() << "KLEE: done: solver time = " << ss.str() << "\n"; handler->getInfoStream() << "KLEE: done: explored paths = " << 1 + forks << "\n"; @@ -2477,7 +2492,12 @@ int main(int argc, char **argv, char **envp) { << handler->getNumPathsExplored() - handler->getNumPathsCompleted() << '\n' << "KLEE: done: generated tests = " << handler->getNumTestCases() - << '\n'; + << '\n' + << "KLEE: done: forks = " << forks << '\n' + << "KLEE: done: coverage = " + << (static_cast(handler->getNumPathsCompleted()) / + static_cast(handler->getNumTestCases())) * + 100 << "%"; bool useColors = llvm::errs().is_displayed(); if (useColors)