diff --git a/CHANGELOG.md b/CHANGELOG.md index d063442d..f86f5370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # ChangeLog +## [10.2.2] - 2023-09-20 +### Added +- Allow user to specify any 'key' from 'ORC_Offline' special command set +- NTFSInfo: add columns to volstats.csv for output file name +- WolfLauncher: remove working directory on exit when it was created and empty + +### Changed +- Yara: update to 4.3.2 + +### Fixed +- Yara: possible execution loop issue depending on the rule +- FastFind: in the XML results file the 'Type' values for a registry match was always set to 'Type' +- FastFind: XML output style +- NTFSInfo/FATInfo: unexpected FirstBytes column zero padding + ## [10.2.1] - 2023-06-20 ### Changed - Configuration: accept wildcard as exclusion path diff --git a/external/vcpkg b/external/vcpkg index 149484f2..a4a0659e 160000 --- a/external/vcpkg +++ b/external/vcpkg @@ -1 +1 @@ -Subproject commit 149484f2de6130354bf5006b41397441b6ea96ca +Subproject commit a4a0659ed84fa94c214edb735436a7e95032b37e diff --git a/src/OrcCommand/Command/FastFind/FastFind_Run.cpp b/src/OrcCommand/Command/FastFind/FastFind_Run.cpp index 5d45c9f1..58d8988b 100644 --- a/src/OrcCommand/Command/FastFind/FastFind_Run.cpp +++ b/src/OrcCommand/Command/FastFind/FastFind_Run.cpp @@ -249,7 +249,10 @@ HRESULT Main::RunFileSystem() HRESULT hr = E_FAIL; if (pStructuredOutput) + { pStructuredOutput->BeginCollection(L"filesystem"); + pStructuredOutput->BeginElement(nullptr); + } hr = config.FileSystem.Files.Find( config.FileSystem.Locations, @@ -267,7 +270,11 @@ HRESULT Main::RunFileSystem() if (pFileSystemTableOutput) aMatch->Write(*pFileSystemTableOutput); if (pStructuredOutput) + { + pStructuredOutput->BeginCollection(L"filefind_match"); aMatch->Write(*pStructuredOutput, nullptr); + pStructuredOutput->EndCollection(L"filefind_match"); + } return; }, @@ -281,7 +288,10 @@ HRESULT Main::RunFileSystem() } if (pStructuredOutput) + { + pStructuredOutput->EndElement(nullptr); pStructuredOutput->EndCollection(L"filesystem"); + } m_console.PrintNewLine(); ::PrintStatistics(m_console.OutputTree(), config.FileSystem.Files.AllSearchTerms()); @@ -311,7 +321,6 @@ HRESULT Main::RunRegistry() }, false, ResurrectRecordsMode::kNo); - if (FAILED(hr)) { Log::Error(L"Failed to parse location while searching for registry hives"); @@ -320,6 +329,7 @@ HRESULT Main::RunRegistry() if (pStructuredOutput) { pStructuredOutput->BeginCollection(L"registry"); + pStructuredOutput->BeginElement(nullptr); } for (const auto& aFileMatch : config.Registry.Files.Matches()) @@ -328,6 +338,7 @@ HRESULT Main::RunRegistry() if (pStructuredOutput) { + pStructuredOutput->BeginCollection(L"hive"); pStructuredOutput->BeginElement(nullptr); pStructuredOutput->WriteNamed(L"volume_id", aFileMatch->VolumeReader->VolumeSerialNumber(), true); @@ -385,11 +396,15 @@ HRESULT Main::RunRegistry() } if (pStructuredOutput) + { pStructuredOutput->EndElement(nullptr); + pStructuredOutput->EndCollection(L"hive"); + } } if (pStructuredOutput) { + pStructuredOutput->EndElement(nullptr); pStructuredOutput->EndCollection(L"registry"); } @@ -403,10 +418,11 @@ Main::LogObjectMatch(const ObjectSpec::ObjectItem& spec, const ObjectDirectory:: if (pStructuredOutput) { - pStructuredOutput->BeginElement(szElement); + pStructuredOutput->BeginElement(L"object_match"); + pStructuredOutput->WriteNamed(L"description", spec.Description().c_str()); obj.Write(*pStructuredOutput); - pStructuredOutput->EndElement(szElement); + pStructuredOutput->EndElement(L"object_match"); } if (pObjectTableOutput) { @@ -423,10 +439,11 @@ Main::LogObjectMatch(const ObjectSpec::ObjectItem& spec, const FileDirectory::Fi if (pStructuredOutput) { - pStructuredOutput->BeginElement(szElement); + pStructuredOutput->BeginElement(L"object_match"); + pStructuredOutput->WriteNamed(L"description", spec.Description().c_str()); file.Write(*pStructuredOutput); - pStructuredOutput->EndElement(szElement); + pStructuredOutput->EndElement(L"object_match"); } if (pObjectTableOutput) @@ -442,7 +459,10 @@ HRESULT Main::RunObject() HRESULT hr = E_FAIL; if (pStructuredOutput) - pStructuredOutput->BeginCollection(L"object_directory"); + { + pStructuredOutput->BeginCollection(L"object"); + pStructuredOutput->BeginElement(nullptr); + } for (const auto& objdir : ObjectDirs) { @@ -600,7 +620,10 @@ HRESULT Main::RunObject() } if (pStructuredOutput) - pStructuredOutput->EndCollection(L"object_directory"); + { + pStructuredOutput->EndElement(nullptr); + pStructuredOutput->EndCollection(L"object"); + } return S_OK; } diff --git a/src/OrcCommand/Command/FatInfo/FatInfo_Run.cpp b/src/OrcCommand/Command/FatInfo/FatInfo_Run.cpp index 17904e29..c6ebf319 100644 --- a/src/OrcCommand/Command/FatInfo/FatInfo_Run.cpp +++ b/src/OrcCommand/Command/FatInfo/FatInfo_Run.cpp @@ -62,7 +62,7 @@ HRESULT Main::Run() return loc->GetParse() && (loc->IsFAT12() || loc->IsFAT16() || loc->IsFAT32()); }); - hr = m_FileInfoOutput.GetWriters(m_Config.output, L"FatInfo", locations); + hr = m_FileInfoOutput.GetWriters(m_Config.output, L"FatInfo", locations, OutputInfo::DataType::kFatInfo); if (FAILED(hr)) { Log::Error(L"Failed to create file information writers [{}]", SystemError(hr)); @@ -91,7 +91,8 @@ HRESULT Main::Run() fileEntry, m_CodeVerifier); - HRESULT hr = fi.WriteFileInformation(FatFileInfo::g_FatColumnNames, *dir.second, m_Config.Filters); + HRESULT hr = + fi.WriteFileInformation(FatFileInfo::g_FatColumnNames, *dir.second.Writer(), m_Config.Filters); if (FAILED(hr)) { Log::Error(L"Could not WriteFileInformation for '{}' [{}]", szFullName, SystemError(hr)); diff --git a/src/OrcCommand/Command/NTFSInfo/NTFSInfo.h b/src/OrcCommand/Command/NTFSInfo/NTFSInfo.h index 2d9dd401..824a43d0 100644 --- a/src/OrcCommand/Command/NTFSInfo/NTFSInfo.h +++ b/src/OrcCommand/Command/NTFSInfo/NTFSInfo.h @@ -48,6 +48,15 @@ enum KindOfTime : DWORD class ORCUTILS_API Main : public UtilitiesMain { public: + struct OutputPaths + { + std::optional fileInfo; + std::optional i30Info; + std::optional attrInfo; + std::optional ntfsTimeline; + std::optional secDescr; + }; + class Configuration : public UtilitiesMain::Configuration { public: @@ -123,6 +132,16 @@ class ORCUTILS_API Main : public UtilitiesMain HRESULT Prepare(); HRESULT GetWriters(std::vector>& locs); + + void GetOutputPathsByLocation( + const std::vector>& locations, + std::unordered_map& outputPathsByLocation) const; + + HRESULT WriteVolStats( + const OutputSpec& volStatsSpec, + const std::vector>& locations, + std::shared_ptr& newWriter); + HRESULT WriteTimeLineEntry( ITableOutput& pTimelineOutput, const std::shared_ptr& volreader, diff --git a/src/OrcCommand/Command/NTFSInfo/NTFSInfoSqlSchema.xml b/src/OrcCommand/Command/NTFSInfo/NTFSInfoSqlSchema.xml index cbd01a55..b811ad63 100644 --- a/src/OrcCommand/Command/NTFSInfo/NTFSInfoSqlSchema.xml +++ b/src/OrcCommand/Command/NTFSInfo/NTFSInfoSqlSchema.xml @@ -203,6 +203,11 @@ + + + + + diff --git a/src/OrcCommand/Command/NTFSInfo/NTFSInfo_Run.cpp b/src/OrcCommand/Command/NTFSInfo/NTFSInfo_Run.cpp index 5265fabc..f0152c03 100644 --- a/src/OrcCommand/Command/NTFSInfo/NTFSInfo_Run.cpp +++ b/src/OrcCommand/Command/NTFSInfo/NTFSInfo_Run.cpp @@ -596,31 +596,38 @@ HRESULT Main::GetWriters(std::vector>& locations) { HRESULT hr = E_FAIL; - if (FAILED(hr = m_FileInfoOutput.GetWriters(config.outFileInfo, L"NTFSInfo", locations))) + if (FAILED( + hr = m_FileInfoOutput.GetWriters( + config.outFileInfo, L"NTFSInfo", locations, OutputInfo::DataType::kFileInfo))) { Log::Error("Failed to create file information writers [{}]", SystemError(hr)); return hr; } - if (FAILED(hr = m_AttrOutput.GetWriters(config.outAttrInfo, L"AttrInfo", locations))) + if (FAILED( + hr = m_AttrOutput.GetWriters(config.outAttrInfo, L"AttrInfo", locations, OutputInfo::DataType::kAttrInfo))) { Log::Error("Failed to create attribute information writers [{}]", SystemError(hr)); return hr; } - if (FAILED(hr = m_I30Output.GetWriters(config.outI30Info, L"I30Info", locations))) + if (FAILED(hr = m_I30Output.GetWriters(config.outI30Info, L"I30Info", locations, OutputInfo::DataType::ki30Info))) { Log::Error("Failed to create I30 information writers [{}]", SystemError(hr)); return hr; } - if (FAILED(hr = m_TimeLineOutput.GetWriters(config.outTimeLine, L"NTFSTimeLine", locations))) + if (FAILED( + hr = m_TimeLineOutput.GetWriters( + config.outTimeLine, L"NTFSTimeLine", locations, OutputInfo::DataType::kNtfsTimeline))) { Log::Error("Failed to create timeline information writers [{}]", SystemError(hr)); return hr; } - if (FAILED(hr = m_SecDescrOutput.GetWriters(config.outSecDescrInfo, L"SecDescr", locations))) + if (FAILED( + hr = m_SecDescrOutput.GetWriters( + config.outSecDescrInfo, L"SecDescr", locations, OutputInfo::DataType::kSecDescrInfo))) { Log::Error("Failed to create security descriptors information writers [{}]", SystemError(hr)); return hr; @@ -628,6 +635,104 @@ HRESULT Main::GetWriters(std::vector>& locations) return S_OK; } +void Main::GetOutputPathsByLocation( + const std::vector>& locations, + std::unordered_map& outputPathsByLocation) const +{ + auto GetOutputPath = [&](const auto& location, const auto& outputs) -> std::optional { + for (const auto& [outputLocation, info] : outputs) + { + if (outputLocation.GetIdentifier() == location->GetIdentifier()) + { + return info.Path(); + } + } + + return {}; + }; + + for (const auto& location : locations) + { + auto it = outputPathsByLocation.find(location->GetLocation()); + if (it == std::cend(outputPathsByLocation)) + { + it = outputPathsByLocation.emplace(location->GetLocation(), OutputPaths {}).first; + } + + it->second.fileInfo = GetOutputPath(location, m_FileInfoOutput.Outputs()); + it->second.i30Info = GetOutputPath(location, m_I30Output.Outputs()); + it->second.attrInfo = GetOutputPath(location, m_AttrOutput.Outputs()); + it->second.ntfsTimeline = GetOutputPath(location, m_TimeLineOutput.Outputs()); + it->second.secDescr = GetOutputPath(location, m_SecDescrOutput.Outputs()); + } +} + +HRESULT Main::WriteVolStats( + const OutputSpec& volStatsSpec, + const std::vector>& locations, + std::shared_ptr& newWriter) +{ + if (volStatsSpec.Type == OutputSpec::Kind::Archive || volStatsSpec.Type == OutputSpec::Kind::Directory) + newWriter = Orc::TableOutput::GetWriter(L"volstats.csv", volStatsSpec); + else + return S_OK; + + if (newWriter == nullptr) + { + return E_FAIL; + } + + std::unordered_map outputPathsByLocation; + GetOutputPathsByLocation(locations, outputPathsByLocation); + + auto& volStatOutput = *newWriter; + for (auto& loc : locations) + { + std::optional outputPaths; + auto it = outputPathsByLocation.find(loc->GetLocation()); + if (it != std::cend(outputPathsByLocation)) + { + outputPaths = it->second; + } + + auto reader = loc->GetReader(); + + if (reader == nullptr) + { + return E_FAIL; + } + + SystemDetails::WriteComputerName(volStatOutput); + volStatOutput.WriteInteger(reader->VolumeSerialNumber()); + volStatOutput.WriteString(reader->GetLocation()); + volStatOutput.WriteString(FSVBR::GetFSName(reader->GetFSType()).c_str()); + volStatOutput.WriteBool(loc->GetParse()); + volStatOutput.WriteString(fmt::format(L"{}", fmt::join(loc->GetPaths(), L";"))); + volStatOutput.WriteString(loc->GetShadow() ? ToStringW(loc->GetShadow()->guid).c_str() : L""); + + if (!outputPaths) + { + volStatOutput.WriteNothing(); + volStatOutput.WriteNothing(); + volStatOutput.WriteNothing(); + volStatOutput.WriteNothing(); + volStatOutput.WriteNothing(); + } + else + { + volStatOutput.WriteString(outputPaths->fileInfo.value_or(L"")); + volStatOutput.WriteString(outputPaths->i30Info.value_or(L"")); + volStatOutput.WriteString(outputPaths->attrInfo.value_or(L"")); + volStatOutput.WriteString(outputPaths->ntfsTimeline.value_or(L"")); + volStatOutput.WriteString(outputPaths->secDescr.value_or(L"")); + } + + volStatOutput.WriteEndOfLine(); + } + + return S_OK; +} + HRESULT Main::RunThroughMFT() { auto output = m_console.OutputTree(); @@ -685,13 +790,40 @@ HRESULT Main::RunThroughMFT() return hr; } - m_FileInfoOutput.WriteVolStats(config.volumesStatsOutput, allLocations); - } + if (FAILED(hr = GetWriters(locations))) + { + Log::Error("Failed to create writers for NTFSInfo [{}]", SystemError(hr)); + return hr; + } + + std::shared_ptr volStatWriter; + hr = WriteVolStats(config.volumesStatsOutput, allLocations, volStatWriter); + if (FAILED(hr)) + { + Log::Error("Failed to write volstats [{}]", SystemError(hr)); + } + else + { + volStatWriter->Close(); - if (FAILED(hr = GetWriters(locations))) + auto pStreamWriter = std::dynamic_pointer_cast(volStatWriter); + if (config.volumesStatsOutput.Type == OutputSpec::Kind::Archive && pStreamWriter + && pStreamWriter->GetStream()) + { + // Need to attach the stream to one of "MultipleOutput" + m_FileInfoOutput.AddStream( + config.volumesStatsOutput, L"volstats.csv", pStreamWriter->GetStream(), false, true); + } + } + } + else { - Log::Error("Failed to create writers for NTFSInfo [{}]", SystemError(hr)); - return hr; + if (FAILED(hr = GetWriters(locations))) + { + Log::Error("Failed to create writers for NTFSInfo [{}]", SystemError(hr)); + return hr; + } + } auto fileinfoIterator = begin(m_FileInfoOutput.Outputs()); @@ -734,47 +866,47 @@ HRESULT Main::RunThroughMFT() MFTWalker::Callbacks callBacks; - if (fileinfoIterator->second != nullptr) + if (fileinfoIterator->second.Writer() != nullptr) { callBacks.FileNameAndDataCallback = [this, fileinfoIterator]( const std::shared_ptr& volreader, MFTRecord* pElt, const PFILE_NAME pFileName, const std::shared_ptr& pDataAttr) { - FileAndDataInformation(*fileinfoIterator->second, volreader, pElt, pFileName, pDataAttr); + FileAndDataInformation(*fileinfoIterator->second.Writer(), volreader, pElt, pFileName, pDataAttr); }; callBacks.DirectoryCallback = [this, fileinfoIterator]( const std::shared_ptr& volreader, MFTRecord* pElt, const PFILE_NAME pFileName, const std::shared_ptr& pAttr) { - DirectoryInformation(*fileinfoIterator->second, volreader, pElt, pFileName, pAttr); + DirectoryInformation(*fileinfoIterator->second.Writer(), volreader, pElt, pFileName, pAttr); }; } - if (timelineIterator->second != nullptr) + if (timelineIterator->second.Writer() != nullptr) { callBacks.ElementCallback = [this, timelineIterator](const std::shared_ptr& volreader, MFTRecord* pElt) { - ElementInformation(*timelineIterator->second, volreader, pElt); + ElementInformation(*timelineIterator->second.Writer(), volreader, pElt); }; callBacks.FileNameCallback = [this, timelineIterator]( const std::shared_ptr& volreader, MFTRecord* pElt, const PFILE_NAME pFileName) { - TimelineInformation(*timelineIterator->second, volreader, pElt, pFileName); + TimelineInformation(*timelineIterator->second.Writer(), volreader, pElt, pFileName); }; } - if (attrIterator->second != nullptr) + if (attrIterator->second.Writer() != nullptr) { callBacks.AttributeCallback = [this, attrIterator]( const std::shared_ptr& volreader, MFTRecord* pElt, const AttributeListEntry& AttrEntry) { - AttrInformation(*attrIterator->second, volreader, pElt, AttrEntry); + AttrInformation(*attrIterator->second.Writer(), volreader, pElt, AttrEntry); }; } - if (i30Iterator->second != nullptr) + if (i30Iterator->second.Writer() != nullptr) { callBacks.I30Callback = [this, i30Iterator]( const std::shared_ptr& volreader, @@ -782,16 +914,16 @@ HRESULT Main::RunThroughMFT() const PINDEX_ENTRY& pEntry, const PFILE_NAME pFileName, bool bCarvedEntry) { - I30Information(*i30Iterator->second, volreader, pElt, pEntry, pFileName, bCarvedEntry); + I30Information(*i30Iterator->second.Writer(), volreader, pElt, pEntry, pFileName, bCarvedEntry); }; } - if (secdescrIterator->second != nullptr) + if (secdescrIterator->second.Writer() != nullptr) { callBacks.SecDescCallback = [this, secdescrIterator]( const std::shared_ptr& volreader, const PSECURITY_DESCRIPTOR_ENTRY pEntry) { - SecurityDescriptorInformation(*secdescrIterator->second, volreader, pEntry); + SecurityDescriptorInformation(*secdescrIterator->second.Writer(), volreader, pEntry); }; } diff --git a/src/OrcCommand/Command/ObjInfo/ObjInfo_Run.cpp b/src/OrcCommand/Command/ObjInfo/ObjInfo_Run.cpp index 705a41f1..f1855b52 100644 --- a/src/OrcCommand/Command/ObjInfo/ObjInfo_Run.cpp +++ b/src/OrcCommand/Command/ObjInfo/ObjInfo_Run.cpp @@ -65,7 +65,7 @@ HRESULT Main::Run() BOOST_SCOPE_EXIT(&config, &m_outputs) { m_outputs.CloseAll(config.output); } BOOST_SCOPE_EXIT_END; - hr = m_outputs.GetWriters(config.output, L"ObjInfo"); + hr = m_outputs.GetWriters(config.output, L"ObjInfo", OutputInfo::DataType::kObjInfo); if (FAILED(hr)) { Log::Error("Failed to create objinfo output writers [{}]", SystemError(hr)); @@ -95,7 +95,7 @@ HRESULT Main::Run() case DirectoryType::ObjectDir: if (SUCCEEDED(hr = objectdir.ParseObjectDirectory(dir.first.m_Directory, objects))) { - if (dir.second == nullptr) + if (dir.second.Writer() == nullptr) { Log::Error(L"No output writer configured for '{}' directory, skipped", dir.first.m_Directory); return hr; @@ -103,7 +103,7 @@ HRESULT Main::Run() for (auto& obj : objects) { - obj.Write(*dir.second, L""s); + obj.Write(*dir.second.Writer(), L""s); } } else @@ -120,7 +120,7 @@ HRESULT Main::Run() { for (auto& file : files) { - file.Write(*dir.second, L""s); + file.Write(*dir.second.Writer(), L""s); } } else diff --git a/src/OrcCommand/Command/USNInfo/USNInfo_Run.cpp b/src/OrcCommand/Command/USNInfo/USNInfo_Run.cpp index 68223bf3..768c670f 100644 --- a/src/OrcCommand/Command/USNInfo/USNInfo_Run.cpp +++ b/src/OrcCommand/Command/USNInfo/USNInfo_Run.cpp @@ -181,11 +181,9 @@ HRESULT Main::Run() } } - Guard::Scope closeOnExit([&]() { - m_outputs.CloseAll(config.output); - }); + Guard::Scope closeOnExit([&]() { m_outputs.CloseAll(config.output); }); - hr = m_outputs.GetWriters(config.output, L"USNInfo", locations); + hr = m_outputs.GetWriters(config.output, L"USNInfo", locations, OutputInfo::DataType::kUsnInfo); if (FAILED(hr)) { Log::Error(L"Failed to get writers for locations [{}]", SystemError(hr)); @@ -235,7 +233,7 @@ HRESULT Main::Run() callbacks.RecordCallback = [this, &outputIt](const std::shared_ptr& volreader, WCHAR* szFullName, USN_RECORD* pElt) { - USNRecordInformation(*outputIt->second, volreader, szFullName, pElt); + USNRecordInformation(*outputIt->second.Writer(), volreader, szFullName, pElt); }; hr = walker.ReadJournal(callbacks); diff --git a/src/OrcCommand/Command/WolfLauncher/WolfLauncher.h b/src/OrcCommand/Command/WolfLauncher/WolfLauncher.h index 3b6b1999..da3e9e4d 100644 --- a/src/OrcCommand/Command/WolfLauncher/WolfLauncher.h +++ b/src/OrcCommand/Command/WolfLauncher/WolfLauncher.h @@ -159,6 +159,20 @@ class ORCUTILS_API Main : public UtilitiesMain m_standardOutput.EnableTeeRedirection(); } + ~Main() + { + std::sort( + std::begin(m_emptyDirectoriesToRemove), + std::end(m_emptyDirectoriesToRemove), + [](const auto& lhs, const auto& rhs) { return lhs.size() > rhs.size(); }); + + for (const auto& directory : m_emptyDirectoriesToRemove) + { + std::error_code ec; + std::filesystem::remove(directory, ec); + } + } + void PrintUsage(); void PrintFooter(); void PrintParameters(); @@ -217,6 +231,7 @@ class ORCUTILS_API Main : public UtilitiesMain std::shared_ptr m_pUploadAgent; std::unique_ptr m_pUploadMessageQueue; std::unique_ptr> m_pUploadNotification; + std::vector m_emptyDirectoriesToRemove; }; } // namespace Command::Wolf diff --git a/src/OrcCommand/Command/WolfLauncher/WolfLauncher_Config.cpp b/src/OrcCommand/Command/WolfLauncher/WolfLauncher_Config.cpp index 1ed9be20..8a40b362 100644 --- a/src/OrcCommand/Command/WolfLauncher/WolfLauncher_Config.cpp +++ b/src/OrcCommand/Command/WolfLauncher/WolfLauncher_Config.cpp @@ -38,6 +38,33 @@ namespace fs = std::filesystem; namespace { +constexpr std::wstring_view kOrcOffline(L"ORC_Offline"); + +// Very close to std::filesystem::create_directories but keep tracks or created directories +void CreateDirectories(std::filesystem::path path, std::vector& newDirectories, std::error_code& ec) +{ + if (std::filesystem::exists(path)) + { + return; + } + + auto parent = path.parent_path(); + if (!parent.empty()) + { + CreateDirectories(parent, newDirectories, ec); + if (ec) + { + Log::Debug(L"Failed to create directory: {} [{}]", parent, ec); + return; + } + } + + if (std::filesystem::create_directory(path, ec)) + { + newDirectories.push_back(path); + } +} + bool IgnoreOptions(LPCWSTR szArg) { const auto kConsole = "console"; @@ -837,7 +864,26 @@ HRESULT Main::CheckConfiguration() if (config.TempWorkingDir.Type == OutputSpec::Kind::None) { - if (FAILED(hr = config.TempWorkingDir.Configure(OutputSpec::Kind::Directory, L"%TEMP%\\WorkingTemp"))) + const std::wstring workingTemp(L"%TEMP%\\WorkingTemp"); + + // Prevent 'Configure' to create directories so they can be tracked for removal + { + std::error_code ec; + + auto temp = OutputSpec::Resolve(workingTemp); + ::CreateDirectories(temp, m_emptyDirectoriesToRemove, ec); + if (ec) + { + Log::Error( + L"Failed to pre-create working directory (initial value: {}, value: {}) [{}]", + workingTemp, + temp, + ec); + ec.clear(); + } + } + + if (FAILED(hr = config.TempWorkingDir.Configure(OutputSpec::Kind::Directory, workingTemp))) { Log::Error(L"Failed to use default temporary directory from %TEMP%"); return hr; @@ -960,8 +1006,10 @@ HRESULT Main::CheckConfiguration() } // Then, we set ORC_Offline as a "OnlyThis" keyword - config.OnlyTheseKeywords.clear(); - config.OnlyTheseKeywords.insert(L"ORC_Offline"); + if (config.OnlyTheseKeywords.empty()) + { + config.OnlyTheseKeywords.insert(std::wstring(kOrcOffline)); + } // Finally, we set the %OfflineLocation env var so the location is known to chikdren SetEnvironmentVariable(L"OfflineLocation", config.strOfflineLocation.value().c_str()); @@ -1069,6 +1117,21 @@ HRESULT Main::CheckConfiguration() } } + if (config.strOfflineLocation.has_value()) + { + for (const auto& exec : m_wolfexecs) + { + for (const auto& command : exec->GetCommands()) + { + if (!command->IsOptional() && !boost::iequals(exec->GetKeyword(), kOrcOffline)) + { + command->SetOptional(); + m_journal.Print(exec->GetKeyword(), command->Keyword(), L"Error: incompatible with 'offline' mode"); + } + } + } + } + // Forward some options to executed commands like 'nolimits' for (const auto& exec : m_wolfexecs) { diff --git a/src/OrcCommand/UtilitiesMain.h b/src/OrcCommand/UtilitiesMain.h index 61c8c482..12eb6c20 100644 --- a/src/OrcCommand/UtilitiesMain.h +++ b/src/OrcCommand/UtilitiesMain.h @@ -84,11 +84,55 @@ class UtilitiesMain UtilitiesLoggerConfiguration log; }; + class OutputInfo + { + public: + enum class DataType + { + kFileInfo = 1, + kAttrInfo, + ki30Info, + kNtfsTimeline, + kSecDescrInfo, + kFatInfo, + kObjInfo, + kUsnInfo + }; + + OutputInfo(std::shared_ptr writer) + : m_writer(std::move(writer)) + , m_path() + , m_type() + { + } + + OutputInfo(std::shared_ptr writer, const std::wstring& path, DataType type) + : m_writer(std::move(writer)) + , m_path(path) + , m_type(type) + { + } + + std::shared_ptr& Writer() { return m_writer; } + const std::shared_ptr& Writer() const { return m_writer; } + + void SetPath(const std::wstring& path) { m_path = path; } + const std::optional& Path() const { return m_path; } + + void SetType(DataType type) { m_type = type; } + std::optional Type() const { return m_type; } + + public: + std::shared_ptr m_writer; + std::optional m_path; + DataType m_type; + }; + template class MultipleOutput { public: - using OutputPair = std::pair>; + using OutputPair = std::pair; private: std::vector m_outputs; @@ -99,6 +143,7 @@ class UtilitiesMain public: std::vector& Outputs() { return m_outputs; }; + const std::vector& Outputs() const { return m_outputs; }; HRESULT WriteVolStats(const OutputSpec& volStatsSpec, const std::vector>& locations) { @@ -126,6 +171,7 @@ class UtilitiesMain volStatOutput.WriteBool(loc->GetParse()); volStatOutput.WriteString(fmt::format(L"{}", fmt::join(loc->GetPaths(), L";"))); volStatOutput.WriteString(loc->GetShadow() ? ToStringW(loc->GetShadow()->guid).c_str() : L""); + volStatOutput.WriteEndOfLine(); } else @@ -200,7 +246,7 @@ class UtilitiesMain return S_OK; } - HRESULT GetWriters(const OutputSpec& output, LPCWSTR szPrefix) + HRESULT GetWriters(const OutputSpec& output, LPCWSTR szPrefix, OutputInfo::DataType dataType) { HRESULT hr = E_FAIL; @@ -226,7 +272,7 @@ class UtilitiesMain } for (auto& out : m_outputs) { - out.second = pWriter; + out.second = {pWriter, output.Path, dataType}; } } break; @@ -243,7 +289,8 @@ class UtilitiesMain Log::Error("Failed to create output file information"); return E_FAIL; } - out.second = pW; + + out.second = {pW, szOutputFile, dataType}; } } break; @@ -252,7 +299,6 @@ class UtilitiesMain for (auto& out : m_outputs) { - std::shared_ptr<::Orc::TableOutput::IWriter> pW; WCHAR szOutputFile[ORC_MAX_PATH]; @@ -277,7 +323,8 @@ class UtilitiesMain m_messageBuf, ArchiveMessage::MakeAddStreamRequest(szOutputFile, pStreamWriter->GetStream(), true)); } - out.second = pW; + + out.second = {pW, szOutputFile, dataType}; } Concurrency::send(m_messageBuf, ArchiveMessage::MakeFlushQueueRequest()); } @@ -288,15 +335,20 @@ class UtilitiesMain }; template - HRESULT GetWriters(const OutputSpec& output, LPCWSTR szPrefix, const InputContainer& container) + HRESULT GetWriters( + const OutputSpec& output, + LPCWSTR szPrefix, + const InputContainer& container, + OutputInfo::DataType dataType) { - m_outputs.reserve(container.size()); + for (const auto& item : container) { m_outputs.emplace_back(item, nullptr); } - return GetWriters(output, szPrefix); + + return GetWriters(output, szPrefix, dataType); }; HRESULT ForEachOutput(const OutputSpec& output, std::function aCallback) @@ -373,8 +425,8 @@ class UtilitiesMain { case OutputSpec::Kind::Directory: case OutputSpec::Kind::Archive: - item.second->Close(); - item.second.reset(); + item.second.Writer()->Close(); + item.second.Writer().reset(); break; default: break; @@ -395,10 +447,10 @@ class UtilitiesMain case OutputSpec::Kind::TableFile | OutputSpec::Kind::Parquet: case OutputSpec::Kind::ORC: case OutputSpec::Kind::TableFile | OutputSpec::Kind::ORC: - if (!m_outputs.empty() && m_outputs.front().second != nullptr) + if (!m_outputs.empty() && m_outputs.front().second.Writer() != nullptr) { - m_outputs.front().second->Close(); - m_outputs.front().second.reset(); + m_outputs.front().second.Writer()->Close(); + m_outputs.front().second.Writer().reset(); } break; default: diff --git a/src/OrcLib/DataDetails.h b/src/OrcLib/DataDetails.h index c34caf6b..351ee7f4 100644 --- a/src/OrcLib/DataDetails.h +++ b/src/OrcLib/DataDetails.h @@ -88,7 +88,7 @@ class DataDetails std::swap(m_FirstBytes, buffer); return S_OK; } - bool FirstBytesAvailable() const { return m_FirstBytes.GetCount() == BYTES_IN_FIRSTBYTES; } + bool FirstBytesAvailable() const { return !m_FirstBytes.empty(); } CBinaryBuffer& FirstBytes() { return m_FirstBytes; } HRESULT SetMD5(CBinaryBuffer&& buffer) diff --git a/src/OrcLib/FileInfo.cpp b/src/OrcLib/FileInfo.cpp index 36d3be3a..38346e41 100644 --- a/src/OrcLib/FileInfo.cpp +++ b/src/OrcLib/FileInfo.cpp @@ -646,6 +646,7 @@ HRESULT FileInfo::OpenFirstBytes() if (FAILED(hr = stream->Read(FBBuffer.GetData(), BYTES_IN_FIRSTBYTES, &ullBytesRead))) return hr; + FBBuffer.SetCount(ullBytesRead); GetDetails()->SetFirstBytes(std::move(FBBuffer)); return S_OK; diff --git a/src/OrcLib/Flags.cpp b/src/OrcLib/Flags.cpp index eb6abe49..e5050369 100644 --- a/src/OrcLib/Flags.cpp +++ b/src/OrcLib/Flags.cpp @@ -48,4 +48,29 @@ std::string FlagsToString(DWORD flags, const FlagsDefinition flagValues[], CHAR return out; } +std::optional ExactFlagToString(DWORD dwFlags, const FlagsDefinition FlagValues[]) +{ + int idx = 0; + bool found = false; + + std::wstring buffer; + while (FlagValues[idx].dwFlag != 0xFFFFFFFF) + { + if (dwFlags == FlagValues[idx].dwFlag) + { + fmt::format_to(std::back_inserter(buffer), L"{}", FlagValues[idx].szShortDescr); + found = true; + break; + } + idx++; + } + + if (!found) + { + return {}; + } + + return buffer; +} + } // namespace Orc diff --git a/src/OrcLib/Flags.h b/src/OrcLib/Flags.h index 25db0b74..9d6b0e0c 100644 --- a/src/OrcLib/Flags.h +++ b/src/OrcLib/Flags.h @@ -23,4 +23,8 @@ struct FlagsDefinition std::string FlagsToString(DWORD flags, const FlagsDefinition flagValues[], CHAR separator = '|'); +// There are FlagsDefinition structure which are used without being flags. Multiple +// functions where doing this conversion without using any mask but raw value. +std::optional ExactFlagToString(DWORD dwFlags, const FlagsDefinition FlagValues[]); + }; // namespace Orc diff --git a/src/OrcLib/ObjectDirectory.cpp b/src/OrcLib/ObjectDirectory.cpp index a98d4491..086e5273 100644 --- a/src/OrcLib/ObjectDirectory.cpp +++ b/src/OrcLib/ObjectDirectory.cpp @@ -6,6 +6,9 @@ // Author(s): Jean Gautier (ANSSI) // #include "stdafx.h" + +#include + #include "ObjectDirectory.h" #include "NtDllExtension.h" @@ -17,6 +20,7 @@ #include "TableOutputWriter.h" #include "boost\scope_exit.hpp" +#include "Flags.h" using namespace Orc; @@ -144,14 +148,11 @@ HRESULT ObjectDirectory::ObjectInstance::Write(ITableOutput& output, const std:: return S_OK; } -HRESULT ObjectDirectory::ObjectInstance::Write( - - IStructuredOutput& pWriter, - LPCWSTR szElement) const +HRESULT ObjectDirectory::ObjectInstance::Write(IStructuredOutput& pWriter, LPCWSTR szElement) const { pWriter.BeginElement(szElement); - pWriter.WriteNamed(L"type", (DWORD)Type, ObjectDirectory::g_ObjectTypeDefinition); + pWriter.WriteNamed(L"type", ExactFlagToString(Type, ObjectDirectory::g_ObjectTypeDefinition).value_or(L"")); pWriter.WriteNamed(L"name", Name.c_str()); if (!Path.empty()) diff --git a/src/OrcLib/OutputSpec.cpp b/src/OrcLib/OutputSpec.cpp index 0555babc..ec1c265a 100644 --- a/src/OrcLib/OutputSpec.cpp +++ b/src/OrcLib/OutputSpec.cpp @@ -139,6 +139,37 @@ bool OutputSpec::IsArchive() const return HasFlag(Type, Kind::Archive); } +std::filesystem::path OutputSpec::Resolve(const std::wstring& path) +{ + fs::path output; + + if (IsPattern(path)) + { + wstring patternApplied; + if (auto hr = ApplyPattern(path, L""s, patternApplied); SUCCEEDED(hr)) + { + output = patternApplied; + } + else + { + output = path; + } + } + else + { + output = path; + } + + WCHAR szExpanded[ORC_MAX_PATH] = {0}; + ExpandEnvironmentStrings(output.c_str(), szExpanded, ORC_MAX_PATH); + if (wcscmp(output.c_str(), szExpanded) != 0) + { + output.assign(szExpanded); + } + + return output; +} + HRESULT OutputSpec::Configure( OutputSpec::Kind supported, const std::wstring& strInputString, @@ -148,26 +179,13 @@ HRESULT OutputSpec::Configure( Type = OutputSpec::Kind::None; - // Now on with regular file paths - fs::path outPath; - if (IsPattern(strInputString)) { Pattern = strInputString; - - wstring patternApplied; - if (auto hr = ApplyPattern(strInputString, L""s, patternApplied); SUCCEEDED(hr)) - outPath = patternApplied; - else - outPath = strInputString; } - else - outPath = strInputString; - WCHAR szExpanded[ORC_MAX_PATH] = {0}; - ExpandEnvironmentStrings(outPath.c_str(), szExpanded, ORC_MAX_PATH); - if (wcscmp(outPath.c_str(), szExpanded) != 0) - outPath.assign(szExpanded); + // Now on with regular file paths + fs::path outPath = Resolve(strInputString); if (parent.has_value()) { @@ -175,7 +193,9 @@ HRESULT OutputSpec::Configure( Path = outPath; } else + { Path = outPath; + } FileName = outPath.filename().wstring(); diff --git a/src/OrcLib/OutputSpec.h b/src/OrcLib/OutputSpec.h index 95fac297..25ce093e 100644 --- a/src/OrcLib/OutputSpec.h +++ b/src/OrcLib/OutputSpec.h @@ -114,6 +114,8 @@ class OutputSpec static OutputSpec::Encoding ToEncoding(Text::Encoding); + static std::filesystem::path Resolve(const std::wstring& path); + HRESULT Configure( OutputSpec::Kind supportedTypes, const std::wstring& inputString, diff --git a/src/OrcLib/PEInfo.cpp b/src/OrcLib/PEInfo.cpp index 156bdeef..72a8cfe6 100644 --- a/src/OrcLib/PEInfo.cpp +++ b/src/OrcLib/PEInfo.cpp @@ -1,4 +1,3 @@ -// // SPDX-License-Identifier: LGPL-2.1-or-later // // Copyright © 2011-2021 ANSSI. All Rights Reserved. @@ -159,12 +158,15 @@ HRESULT PEInfo::OpenPEInformation() return hr; buf.SetCount(static_cast(ullBytesRead)); - if (buf.GetCount() >= BYTES_IN_FIRSTBYTES) + if (!buf.empty()) { CBinaryBuffer fb; - if (!fb.SetCount(BYTES_IN_FIRSTBYTES)) + if (!fb.SetCount(std::min(static_cast(BYTES_IN_FIRSTBYTES), buf.GetCount()))) + { return E_OUTOFMEMORY; - CopyMemory(fb.GetData(), buf.GetData(), BYTES_IN_FIRSTBYTES); + } + + CopyMemory(fb.GetData(), buf.GetData(), fb.GetCount()); m_FileInfo.GetDetails()->SetFirstBytes(std::move(fb)); } diff --git a/src/OrcLib/YaraScanner.cpp b/src/OrcLib/YaraScanner.cpp index 041d71d6..8f4e0e5c 100644 --- a/src/OrcLib/YaraScanner.cpp +++ b/src/OrcLib/YaraScanner.cpp @@ -543,7 +543,9 @@ HRESULT YaraScanner::ScanBlocks(const std::shared_ptr& stream, Match } // On first fetch only read 1MB as it will be often enough for header matching - context->block.size = std::min(context->buffer.size(), static_cast(1048576)); + context->block.size = std::min( + static_cast(context->buffer.size()), + std::min(static_cast(1048576), context->streamSize)); context->block.base = 0; return &context->block; }; @@ -562,7 +564,14 @@ HRESULT YaraScanner::ScanBlocks(const std::shared_ptr& stream, Match return nullptr; } - context->block.size = context->buffer.size(); // TODO: std::min with stream size ? + if (context->block.base > context->streamSize) + { + Log::Warn(L"Unexpected file offset in Yara callback but rule evaluation should consistent"); + return nullptr; + } + + context->block.size = + std::min(context->streamSize - context->block.base, static_cast(context->buffer.size())); return &context->block; }; diff --git a/tools/ci/test.psm1 b/tools/ci/test.psm1 index 50de3b00..2ca0ba83 100644 --- a/tools/ci/test.psm1 +++ b/tools/ci/test.psm1 @@ -790,28 +790,41 @@ function Expand-OrcArchive { Write-Verbose "Decrypting archive: '$Archive'" $DecryptOutput = Join-Path $Destination ($Archive.BaseName + ".7zs") - Remove-Item $DecryptOutput -ErrorAction SilentlyContinue - - # FIX: Calling 'OpensslExe' break color output so it is done with 'Start-Process' - #OpensslExe cms -decrypt -in $Archive -out $DecryptOutput -inkey $PrivateKey -inform DER -binary - Start-Process ` - -WindowStyle Hidden ` - -Wait ` - -FilePath $OpensslExePath ` - -ArgumentList ` - "cms", - "-decrypt", - "-in","`"$Archive`"", - "-out","`"$DecryptOutput`"", - "-inkey","`"$PrivateKey`"", - "-inform","DER", - "-binary" + if (-Not (Test-Path $DecryptOutput) -Or $Force) + { + Remove-Item $DecryptOutput -ErrorAction SilentlyContinue + + # FIX: Calling 'OpensslExe' break color output so it is done with 'Start-Process' + #OpensslExe cms -decrypt -in $Archive -out $DecryptOutput -inkey $PrivateKey -inform DER -binary + Start-Process ` + -WindowStyle Hidden ` + -Wait ` + -FilePath $OpensslExePath ` + -ArgumentList ` + "cms", + "-decrypt", + "-in","`"$Archive`"", + "-out","`"$DecryptOutput`"", + "-inkey","`"$PrivateKey`"", + "-inform","DER", + "-binary" + } + else + { + Write-Verbose "Using existing file: '$DecryptOutput'" + } Write-Verbose "Unstreaming archive journal: '$DecryptOutput'" $UnstreamOutput = Join-Path $Destination $Archive.BaseName - Remove-Item $UnstreamOutput -ErrorAction SilentlyContinue - UnstreamExe $DecryptOutput $UnstreamOutput - Remove-Item $DecryptOutput + if (-Not (Test-Path $UnstreamOutput) -Or $Force) + { + Remove-Item $UnstreamOutput -ErrorAction SilentlyContinue + UnstreamExe $DecryptOutput $UnstreamOutput + } + else + { + Write-Verbose "Using existing file: '$UnstreamOutput'" + } $Archive = Get-Item $UnstreamOutput } @@ -838,11 +851,6 @@ function Expand-OrcArchive { Write-Error "Failed to unpack '$Path' into '$ExpandDirectory'" } - if ($UnstreamOutput) - { - Remove-Item $UnstreamOutput - } - $Archives = Get-ChildItem -Recurse -File -Path $ExpandDirectory -Filter *.7z foreach ($SubArchive in $Archives) { @@ -866,8 +874,6 @@ function Expand-OrcArchive { { Write-Error "Failed to unpack '$SubArchive' into '$SubDir'" } - - Remove-Item $SubArchive } return $ExpandDirectory @@ -1108,10 +1114,15 @@ function Get-OrcOutcome { continue } + if (-Not $Command.exit_code) + { + Write-Warning "$OutcomePath, $($Set.name)/$($Command.name): Missing 'exit_code'" + continue + } + # On current powershell version the value was an UInt32 that could not be easily converted to Int32 # when being 0xFFFFFFFF $Command.exit_code = [Convert]::ToInt32($Command.exit_code.ToString("x"), 16) - if ($Command.exit_code -eq 0) { continue