Merge branch 'main' into release/10.1.x

Author: fabienfl-orc

CHANGELOG.md

@@ -1,5 +1,30 @@
 # ChangeLog
 
+## [10.1.1] - 2022-06-20
+### Added
+- Toolembed: display a message for missing/broken resource because of bad configuration
+- Syslog: display configuration on startup
+
+### Changed
+- GetThis: replace from output filenames the possible leading '__' with '_'
+- Bits: do not create bits job on 'once' mode if server is unreachable
+- Log: set default backtrace log level to debug for commands
+
+### Fixed
+- Ntfs: fix parsing for very long paths
+- Ntfs: fix parsing for very fragmented mfts
+- Ntfs: fix possible crash with nested record resolution
+- FastFind: fix matching yara rule display when wildcard is used for rule selection
+- Configuration: fix missing log option processing for local configuration
+- Fix version string for recents windows releases
+- Fix possible crash with GetNetworkAdapters
+- Fix possible locking issue when uploading console's log
+- Bits: add more detail to failure logs
+- Bits: fix some missing error handling
+- Outcome: fix possible failure when calculating mothership's hash
+- Vcpkg: removed uneeded/broken dependency to libwinpthread
+
+
 ## [10.1.0] - 2022-03-25
 Summary of changes since the 10.0.24. For more details look at rc versions.
 From 10.1.0 the semantic versioning will be applied.

external/vcpkg

@@ -185,6 +185,17 @@ HRESULT Main::Launch(const std::wstring& command, const std::wstring& commandArg
 
     std::vector<WCHAR> szCommandLine(MAX_CMDLINE);
     std::wstring strCommandLine = cmdLineBuilder.str();
+
+    HANDLE hMothership = OpenProcess(PROCESS_QUERY_INFORMATION, TRUE, GetCurrentProcessId());
+    if (hMothership)
+    {
+        strCommandLine.append(fmt::format(L" /MothershipHandle={:#x}", reinterpret_cast<size_t>(hMothership)));
+    }
+    else
+    {
+        Log::Error("Failed to append mothership handle on the cli [{}]", LastWin32Error());
+    }
+
     wcsncpy_s(szCommandLine.data(), MAX_CMDLINE, strCommandLine.c_str(), strCommandLine.size());
 
     DWORD dwMajor = 0, dwMinor = 0;

src/Orc/Mothership_Run.cpp

@@ -539,7 +539,7 @@ CreateSampleFileName(const Main::SampleRef& sample, const PFILE_NAME pFileName,
         len = swprintf_s(
             name.data(),
             name.size(),
-            L"%llX_%llX_%llX_%x__%.*s_%.*s_%s.%s",
+            L"%llX_%llX_%llX_%x_%.*s_%.*s_%s.%s",
             sample.VolumeSerial,
             parentFRN,
             FRN,

src/OrcCommand/Command/GetThis/GetThis_Run.cpp

@@ -732,7 +732,7 @@ HRESULT WolfExecution::CompleteExecution()
     }
     catch (Concurrency::operation_timed_out e)
     {
-        Log::Error("Command agent completion timeout: {} msecs reached", m_CmdTimeOut.count());
+        Log::Critical("Command agent completion timeout: {} msecs reached", m_CmdTimeOut.count());
         return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
     }
     if (m_pTermination)
@@ -768,7 +768,7 @@ HRESULT WolfExecution::TerminateAllAndComplete()
     }
     catch (Concurrency::operation_timed_out e)
     {
-        Log::Error(L"Command agent completion timeout: {} msecs reached", m_CmdTimeOut.count());
+        Log::Critical(L"Command agent completion timeout: {} msecs reached", m_CmdTimeOut.count());
         return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
     }
 
@@ -813,7 +813,7 @@ HRESULT WolfExecution::CompleteArchive(UploadMessage::ITarget* pUploadMessageQue
     }
     catch (Concurrency::operation_timed_out e)
     {
-        Log::Error(
+        Log::Critical(
             "Command archive completion timeout: {} secs reached",
             std::chrono::duration_cast<std::chrono::seconds>(m_ArchiveTimeOut).count());
         return HRESULT_FROM_WIN32(ERROR_TIMEOUT);

src/OrcCommand/Command/WolfLauncher/WolfExecution_Execute.cpp

@@ -94,6 +94,7 @@ class ORCUTILS_API Main : public UtilitiesMain
         OutputSpec Outcome;
 
         std::wstring strCompressionLevel;
+        std::wstring strMothershipHandle;
 
         OutputSpec TempWorkingDir;
 
@@ -196,6 +197,8 @@ class ORCUTILS_API Main : public UtilitiesMain
     HRESULT ExecuteKeyword(WolfExecution& execution);
     HRESULT Run_Keywords();
 
+    void ReadLogConfiguration(const ConfigItem& configItem, bool hasConsoleConfigItem);
+
 private:
     ConsoleConfiguration m_consoleConfiguration;
 

src/OrcCommand/Command/WolfLauncher/WolfLauncher.h

@@ -28,6 +28,7 @@
 #include "Command/WolfLauncher/ConfigFile_WOLFLauncher.h"
 #include "Command/WolfLauncher/ConsoleConfiguration.h"
 #include "Configuration/Option.h"
+#include "Text/Hex.h"
 
 using namespace Orc;
 using namespace Orc::Command::Wolf;
@@ -173,6 +174,48 @@ std::shared_ptr<WolfExecution::Recipient> Main::GetRecipientFromItem(const Confi
     return retval;
 }
 
+void Main::ReadLogConfiguration(const ConfigItem& configItem, bool hasConsoleConfigItem)
+{
+    if (!configItem.empty())
+    {
+        //
+        // DEPRECATED: compatibility with 10.0.x log configuration
+        //
+        // Usually configuration is close to '<log ...>ORC_{SystemType}_{FullComputerName}_{TimeStamp}.log</log>'
+        // instead of '<log><file><output>...</output></file></log>'.
+        //
+        // Silently convert any old configuration into new structures.
+        //
+
+        OutputSpec output;
+        auto hr = output.Configure(configItem);
+        if (FAILED(hr))
+        {
+            Log::Warn(L"Failed to configure DFIR-Orc log file [{}]", SystemError(hr));
+        }
+        else if (!hasConsoleConfigItem)
+        {
+            m_consoleConfiguration.output.path = output.Path;
+            m_consoleConfiguration.output.encoding = ToEncoding(output.OutputEncoding);
+            m_consoleConfiguration.output.disposition = ToFileDisposition(output.disposition);
+
+            if (!m_utilitiesConfig.log.level)
+            {
+                m_utilitiesConfig.log.level = Log::Level::Info;
+                UtilitiesLoggerConfiguration::ApplyLogLevel(m_logging, m_utilitiesConfig.log);
+            }
+
+            Log::Warn(
+                L"This use of the configuration element '<log>' is DEPRECATED, you should use '<console> to capture "
+                L"console output or keep using 'log' for detailed logging.");
+        }
+    }
+    else if (configItem)
+    {
+        UtilitiesLoggerConfiguration::Parse(configItem, m_utilitiesConfig.log);
+    }
+}
+
 HRESULT Main::GetConfigurationFromConfig(const ConfigItem& configitem)
 {
     using namespace std::string_view_literals;
@@ -227,44 +270,7 @@ HRESULT Main::GetConfigurationFromConfig(const ConfigItem& configitem)
         }
     }
 
-    if (configitem[WOLFLAUNCHER_LOG] && !configitem[WOLFLAUNCHER_LOG].empty())
-    {
-        //
-        // DEPRECATED: compatibility with 10.0.x log configuration
-        //
-        // Usually configuration is close to '<log ...>ORC_{SystemType}_{FullComputerName}_{TimeStamp}.log</log>'
-        // instead of '<log><file><output>...</output></file></log>'.
-        //
-        // Silently convert any old configuration into new structures.
-        //
-
-        OutputSpec output;
-        auto hr = output.Configure(configitem[WOLFLAUNCHER_LOG]);
-        if (FAILED(hr))
-        {
-            Log::Warn(L"Failed to configure DFIR-Orc log file [{}]", SystemError(hr));
-        }
-        else if (!configitem[WOLFLAUNCHER_CONSOLE])
-        {
-            m_consoleConfiguration.output.path = output.Path;
-            m_consoleConfiguration.output.encoding = ToEncoding(output.OutputEncoding);
-            m_consoleConfiguration.output.disposition = ToFileDisposition(output.disposition);
-
-            if (!m_utilitiesConfig.log.level)
-            {
-                m_utilitiesConfig.log.level = Log::Level::Info;
-                UtilitiesLoggerConfiguration::ApplyLogLevel(m_logging, m_utilitiesConfig.log);
-            }
-
-            Log::Warn(
-                L"This use of the configuration element '<log>' is DEPRECATED, you should use '<console> to capture "
-                L"console output or keep using 'log' for detailed logging.");
-        }
-    }
-    else if (configitem[WOLFLAUNCHER_LOG])
-    {
-        UtilitiesLoggerConfiguration::Parse(configitem[WOLFLAUNCHER_LOG], m_utilitiesConfig.log);
-    }
+    ReadLogConfiguration(configitem[WOLFLAUNCHER_LOG], configitem[WOLFLAUNCHER_CONSOLE].Status == ConfigItem::PRESENT);
 
     if (configitem[WOLFLAUNCHER_CONSOLE])
     {
@@ -533,6 +539,16 @@ HRESULT Main::GetLocalConfigurationFromConfig(const ConfigItem& configitem)
         }
     }
 
+    if (configitem[ORC_LOGGING])
+    {
+        ReadLogConfiguration(configitem[ORC_LOGGING], configitem[ORC_CONSOLE].Status == ConfigItem::PRESENT);
+    }
+
+    if (configitem[ORC_CONSOLE])
+    {
+        ConsoleConfiguration::Parse(configitem[ORC_CONSOLE], m_consoleConfiguration);
+    }
+
     return S_OK;
 }
 
@@ -591,6 +607,8 @@ HRESULT Main::GetConfigurationFromArgcArgv(int argc, LPCWSTR argv[])
                         ;
                     else if (ParameterOption(argv[i] + 1, L"Compression", config.strCompressionLevel))
                         ;
+                    else if (ParameterOption(argv[i] + 1, L"MothershipHandle", config.strMothershipHandle))
+                        ;
                     else if (ParameterOption(argv[i] + 1, L"archive_timeout", config.msArchiveTimeOut))
                         ;
                     else if (ParameterOption(argv[i] + 1, L"command_timeout", config.msCommandTerminationTimeOut))
@@ -715,6 +733,23 @@ HRESULT Main::CheckConfiguration()
         m_utilitiesConfig.log.logFile = logPath;
     }
 
+    if (!config.strMothershipHandle.empty())
+    {
+        auto handle = Text::FromHexToLittleEndian<HANDLE>(std::wstring_view(config.strMothershipHandle));
+        if (handle)
+        {
+            m_hMothership = *handle;
+        }
+        else
+        {
+            Log::Error("Failed to parse mothership handle [{}]", handle.error());
+        }
+    }
+    else
+    {
+        Log::Warn("Missing mothership handle");
+    }
+
     UtilitiesLoggerConfiguration::Apply(m_logging, m_utilitiesConfig.log);
 
     if ((config.bRepeatCreateNew ? 1 : 0) + (config.bRepeatOnce ? 1 : 0) + (config.bRepeatOverwrite ? 1 : 0) > 1)

src/OrcCommand/Command/WolfLauncher/WolfLauncher_Config.cpp

@@ -149,16 +149,16 @@ HRESULT GetRemoteOutputFileInformations(
 
     if (exec.ShouldUpload())
     {
-        DWORD dwFileSize;
-        hr = uploadAgent.CheckFileUpload(exec.GetOutputFileName(), &dwFileSize);
+        std::optional<DWORD> fileSize;
+        hr = uploadAgent.CheckFileUpload(exec.GetOutputFileName(), fileSize);
         if (FAILED(hr))
         {
             return hr;
         }
 
         fileInformations.exist = true;
         fileInformations.path = uploadAgent.GetRemoteFullPath(exec.GetOutputFileName());
-        fileInformations.size = dwFileSize;
+        fileInformations.size = fileSize;
         return S_OK;
     }
 
@@ -178,22 +178,32 @@ Result<std::wstring> GetCurrentExecutableHash(CryptoHashStream::Algorithm algori
     return Hash(path, algorithm);
 }
 
+Result<std::wstring> GetProcessExecutableHash(HANDLE hProcess, CryptoHashStream::Algorithm algorithm)
+{
+    std::error_code ec;
+    const auto path = GetModuleFileNameExApi(hProcess, NULL, ec);
+    if (ec)
+    {
+        return ec;
+    }
+
+    return Hash(path, algorithm);
+}
+
 Result<std::wstring> GetProcessExecutableHash(DWORD dwProcessId, CryptoHashStream::Algorithm algorithm)
 {
     Guard::Handle hProcess = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, dwProcessId);
     if (!hProcess)
     {
-        const auto error = LastWin32Error();
-        Log::Debug(L"Failed OpenProcess: {} [{}]", dwProcessId, error);
-        return error;
+        const auto ec = LastWin32Error();
+        Log::Debug(L"Failed OpenProcess: {} [{}]", dwProcessId, ec);
+        return ec;
     }
 
-    std::error_code ec;
-    const auto path = GetModuleFileNameExApi(hProcess.value(), NULL, ec);
-    return Hash(path, algorithm);
+    return GetProcessExecutableHash(*hProcess, algorithm);
 }
 
-void UpdateOutcome(Command::Wolf::Outcome::Outcome& outcome)
+void UpdateOutcome(Command::Wolf::Outcome::Outcome& outcome, HANDLE hMothership)
 {
     auto&& lock = outcome.Lock();
 
@@ -235,10 +245,13 @@ void UpdateOutcome(Command::Wolf::Outcome::Outcome& outcome)
             mothership.SetCommandLineValue(commandLine.value());
         }
 
-        auto sha1 = GetProcessExecutableHash(mothershipPID.value(), CryptoHashStream::Algorithm::SHA1);
-        if (sha1)
+        if (hMothership)
         {
-            mothership.SetSha1(sha1.value());
+            auto sha1 = GetProcessExecutableHash(hMothership, CryptoHashStream::Algorithm::SHA1);
+            if (sha1)
+            {
+                mothership.SetSha1(sha1.value());
+            }
         }
     }
 
@@ -703,7 +716,7 @@ Orc::Result<void> Main::CreateAndUploadOutcome()
         return Success<void>();
     }
 
-    ::UpdateOutcome(m_outcome);
+    ::UpdateOutcome(m_outcome, m_hMothership);
     ::UpdateOutcome(m_outcome, config.m_Recipients);
     ::UpdateOutcome(m_outcome, m_standardOutput);
     ::UpdateOutcome(m_outcome, m_logging);
@@ -930,7 +943,7 @@ HRESULT Main::Run_Execute()
                     else if (!info.size || *info.size != 0)
                     {
                         commandSetNode.Add(
-                            "Skipping set because non-empty remote output file already exists: '{}' ({})",
+                            "Skipping set because non-empty remote output file already exists: '{}' (size: {})",
                             info.path,
                             info.size);
                         commandSetNode.AddEmptyLine();
@@ -984,6 +997,8 @@ HRESULT Main::Run_Execute()
             Log::Error("Failed to flush standard output [{}]", ec);
         }
 
+        m_standardOutput.FileTee().Close(ec);
+
         const std::filesystem::path localPath = *m_standardOutput.FileTee().Path();
         hr = UploadSingleFile(localPath.filename(), localPath);
         if (FAILED(hr))

src/OrcCommand/Command/WolfLauncher/WolfLauncher_Run.cpp

@@ -12,6 +12,8 @@
 
 #include "ConfigFile_OrcConfig.h"
 #include "Command/WolfLauncher/ConfigFile_WOLFLauncher.h"
+#include "Command/WolfLauncher/ConsoleConfiguration.h"
+#include "Log/UtilitiesLoggerConfiguration.h"
 
 HRESULT Orc::Config::Wolf::Local::root(ConfigItem& item)
 {
@@ -38,13 +40,15 @@ HRESULT Orc::Config::Wolf::Local::root(ConfigItem& item)
         return hr;
     if (FAILED(hr = item.AddChild(Orc::Config::Wolf::recipient, ORC_RECIPIENT)))
         return hr;
-    if (FAILED(hr = item.AddChild(Orc::Config::Common::logging, ORC_LOGGING)))
+    if (FAILED(hr = item.AddChild(Orc::Command::UtilitiesLoggerConfiguration::Register, ORC_LOGGING)))
         return hr;
     if (FAILED(hr = item.AddAttribute(L"priority", ORC_PRIORITY, ConfigItem::OPTION)))
         return hr;
     if (FAILED(hr = item.AddAttribute(L"powerstate", ORC_POWERSTATE, ConfigItem::OPTION)))
         return hr;
     if (FAILED(hr = item.AddAttribute(L"altitude", ORC_ALTITUDE, ConfigItem::OPTION)))
         return hr;
+    if (FAILED(hr = item.AddChild(Orc::Command::ConsoleConfiguration::Register, ORC_CONSOLE)))
+        return hr;
     return S_OK;
 }

src/OrcCommand/ConfigFile_OrcConfig.cpp

@@ -25,6 +25,7 @@ constexpr auto ORC_LOGGING = 8L;
 constexpr auto ORC_PRIORITY = 9L;
 constexpr auto ORC_POWERSTATE = 10L;
 constexpr auto ORC_ALTITUDE = 11L;
+constexpr auto ORC_CONSOLE = 12L;
 
 constexpr auto ORC_ORC = 0L;