Ab#75723

.github/workflows/keyfactor-starter-workflow.yml

@@ -11,10 +11,17 @@ on:
 
 jobs:
   call-starter-workflow:
-    uses: keyfactor/actions/.github/workflows/starter.yml@3.1.2
+    uses: keyfactor/actions/.github/workflows/starter.yml@v4
+    with:
+      command_token_url: ${{ vars.COMMAND_TOKEN_URL }} # Only required for doctool generated screenshots
+      command_hostname: ${{ vars.COMMAND_HOSTNAME }} # Only required for doctool generated screenshots
+      command_base_api_path: ${{ vars.COMMAND_API_PATH }} # Only required for doctool generated screenshots
     secrets:
-      token: ${{ secrets.V2BUILDTOKEN}}
-      APPROVE_README_PUSH: ${{ secrets.APPROVE_README_PUSH}}
-      gpg_key: ${{ secrets.KF_GPG_PRIVATE_KEY }}
-      gpg_pass: ${{ secrets.KF_GPG_PASSPHRASE }}
-      scan_token: ${{ secrets.SAST_TOKEN }}
+      token: ${{ secrets.V2BUILDTOKEN}} # REQUIRED
+      gpg_key: ${{ secrets.KF_GPG_PRIVATE_KEY }} # Only required for golang builds
+      gpg_pass: ${{ secrets.KF_GPG_PASSPHRASE }} # Only required for golang builds
+      scan_token: ${{ secrets.SAST_TOKEN }} # REQUIRED
+      entra_username: ${{ secrets.DOCTOOL_ENTRA_USERNAME }} # Only required for doctool generated screenshots
+      entra_password: ${{ secrets.DOCTOOL_ENTRA_PASSWD }} # Only required for doctool generated screenshots
+      command_client_id: ${{ secrets.COMMAND_CLIENT_ID }} # Only required for doctool generated screenshots
+      command_client_secret: ${{ secrets.COMMAND_CLIENT_SECRET }} # Only required for doctool generated screenshots

CHANGELOG.md

@@ -1,3 +1,10 @@
+v2.12.0
+- Added config.json setting and its override store level custom field - AllowShellCommands.  If "N" (default "Y"), SFTP will be used to create stores and move files on Linux-based certificate store servers.  No Linux shell commands will be used in the integration.
+
+v2.11.5
+- Bug Fix: Rare race condition loading config settings when multiple RemoteFile jobs are running simultaneously on the same orchestrator
+- Documentation update to better list out what Linux commands get executed under what situations in Requirements & Prerequisites section
+
 v2.11.4
 - Bug Fix: Handle condition where a certificate store definition that contains an invalid value for `FileTransferProtocol` 
 would return empty inventory. If no value is set or an invalid value is set, the default value of `Both` will be used 

RemoteFile.UnitTests/ApplicationSettingsTests.cs

@@ -8,16 +8,12 @@ public class ApplicationSettingsTests
     [Fact]
     public void FileTransferProtocol_WhenPopulatedWithValidValue_ReturnsValue()
     {
-        var path = Path.Combine(Directory.GetCurrentDirectory(), "fixtures", "config", "valid", "config.json");
-        ApplicationSettings.Initialize(path);
         Assert.Equal(ApplicationSettings.FileTransferProtocolEnum.SCP, ApplicationSettings.FileTransferProtocol);
     }
 
     [Fact]
     public void FileTransferProtocol_WhenAllThreePopulated_DefaultsToBoth()
     {
-        var path = Path.Combine(Directory.GetCurrentDirectory(), "fixtures", "config", "file_transfer_protocol_all_three", "config.json");
-        ApplicationSettings.Initialize(path);
         Assert.Equal(ApplicationSettings.FileTransferProtocolEnum.Both, ApplicationSettings.FileTransferProtocol);
     }
 }

RemoteFile/ApplicationSettings.cs

@@ -12,6 +12,7 @@
 using Newtonsoft.Json;
 using Microsoft.Extensions.Logging;
 using Keyfactor.Logging;
+using System.Reflection;
 
 
 namespace Keyfactor.Extensions.Orchestrator.RemoteFile 
@@ -41,6 +42,7 @@
         public static string DefaultSudoImpersonatedUser { get { return configuration.ContainsKey("DefaultSudoImpersonatedUser") ? configuration["DefaultSudoImpersonatedUser"] : DEFAULT_SUDO_IMPERSONATION_SETTING; } }
         public static bool CreateCSROnDevice { get { return configuration.ContainsKey("CreateCSROnDevice") ? configuration["CreateCSROnDevice"]?.ToUpper() == "Y" : false; } }
         public static string TempFilePathForODKG { get { return configuration.ContainsKey("TempFilePathForODKG") ? configuration["TempFilePathForODKG"] : string.Empty; } }
+        public static bool UseShellCommands { get { return configuration.ContainsKey("UseShellCommands") ? configuration["UseShellCommands"]?.ToUpper() == "Y" : true; } }
         public static int SSHPort
         {
             get
@@ -71,7 +73,7 @@
                     protocolNames += protocolName + ", ";
                 }
                 protocolNames = protocolNames.Substring(0, protocolNames.Length - 2);
                 string? protocolValue = configuration["FileTransferProtocol"];
 
                 if (!PropertyUtilities.TryEnumParse(protocolValue, out bool isFlagCombination, out FileTransferProtocolEnum protocol))
                     throw new RemoteFileException($"Invalid optional config.json FileTransferProtocol option of {protocolValue}.  If present, must be one of these values: {protocolNames}.");
@@ -88,13 +90,13 @@
             }
         }
 
-        public static void Initialize(string configLocation)
+        static ApplicationSettings()
         {
             ILogger logger = LogHandler.GetClassLogger<ApplicationSettings>();
             logger.MethodEntry(LogLevel.Debug);
 
             configuration = new Dictionary<string, string>();
-            configLocation = $"{Path.GetDirectoryName(configLocation)}{Path.DirectorySeparatorChar}config.json";
+            string configLocation = $"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}config.json";
             string configContents = string.Empty;
 
             if (!File.Exists(configLocation))

RemoteFile/Discovery.cs

@@ -56,10 +56,8 @@ public JobResult ProcessJob(DiscoveryJobConfiguration config, SubmitDiscoveryUpd
                 string userName = PAMUtilities.ResolvePAMField(_resolver, logger, "Server User Name", config.ServerUsername);
                 string userPassword = PAMUtilities.ResolvePAMField(_resolver, logger, "Server Password", config.ServerPassword);
 
-                ApplicationSettings.Initialize(this.GetType().Assembly.Location);
-
                 certificateStore = new RemoteCertificateStore(config.ClientMachine, userName, userPassword, directoriesToSearch[0].Substring(0, 1) == "/" ? RemoteCertificateStore.ServerTypeEnum.Linux : RemoteCertificateStore.ServerTypeEnum.Windows, ApplicationSettings.SSHPort);
-                certificateStore.Initialize(ApplicationSettings.DefaultSudoImpersonatedUser);
+                certificateStore.Initialize(ApplicationSettings.DefaultSudoImpersonatedUser, true);
 
                 if (directoriesToSearch.Length == 0)
                     throw new RemoteFileException("Blank or missing search directories for Discovery.");

RemoteFile/InventoryBase.cs

@@ -35,12 +35,10 @@ public JobResult ProcessJob(InventoryJobConfiguration config, SubmitInventoryUpd
 
             try
             {
-                ApplicationSettings.Initialize(this.GetType().Assembly.Location);
-
                 SetJobProperties(config, config.CertificateStoreDetails, logger);
 
                 certificateStore = new RemoteCertificateStore(config.CertificateStoreDetails.ClientMachine, UserName, UserPassword, config.CertificateStoreDetails.StorePath, StorePassword, FileTransferProtocol, SSHPort, IncludePortInSPN);
-                certificateStore.Initialize(SudoImpersonatedUser);
+                certificateStore.Initialize(SudoImpersonatedUser, UseShellCommands);
                 certificateStore.LoadCertificateStore(certificateStoreSerializer, true);
 
                 List<X509Certificate2Collection> collections = certificateStore.GetCertificateChains();

RemoteFile/ManagementBase.cs

@@ -33,12 +33,10 @@ public JobResult ProcessJob(ManagementJobConfiguration config)
 
             try
             {
-                ApplicationSettings.Initialize(this.GetType().Assembly.Location);
-
                 SetJobProperties(config, config.CertificateStoreDetails, logger);
 
                 certificateStore = new RemoteCertificateStore(config.CertificateStoreDetails.ClientMachine, UserName, UserPassword, config.CertificateStoreDetails.StorePath, StorePassword, FileTransferProtocol, SSHPort, IncludePortInSPN);
-                certificateStore.Initialize(SudoImpersonatedUser);
+                certificateStore.Initialize(SudoImpersonatedUser, UseShellCommands);
 
                 PathFile storePathFile = RemoteCertificateStore.SplitStorePathFile(config.CertificateStoreDetails.StorePath);
 

RemoteFile/ReenrollmentBase.cs

@@ -50,8 +50,6 @@ public JobResult ProcessJobToDo(ReenrollmentJobConfiguration config, SubmitReenr
 
             try
             {
-                ApplicationSettings.Initialize(this.GetType().Assembly.Location);
-
                 SetJobProperties(config, config.CertificateStoreDetails, logger);
 
                 string alias = "abcd";
@@ -68,7 +66,7 @@ public JobResult ProcessJobToDo(ReenrollmentJobConfiguration config, SubmitReenr
                 ApplicationSettings.FileTransferProtocolEnum fileTransferProtocol = ApplicationSettings.FileTransferProtocol;
 
                 certificateStore = new RemoteCertificateStore(config.CertificateStoreDetails.ClientMachine, UserName, UserPassword, config.CertificateStoreDetails.StorePath, StorePassword, fileTransferProtocol, SSHPort, IncludePortInSPN);
-                certificateStore.Initialize(SudoImpersonatedUser);
+                certificateStore.Initialize(SudoImpersonatedUser, UseShellCommands);
 
                 PathFile storePathFile = RemoteCertificateStore.SplitStorePathFile(config.CertificateStoreDetails.StorePath);
 

RemoteFile/RemoteCertificateStore.cs

@@ -453,14 +453,14 @@ internal string GenerateCSROnDevice(string subjectText, SupportedKeyTypeEnum key
             return csr;
         }
 
-        internal void Initialize(string sudoImpersonatedUser)
+        internal void Initialize(string sudoImpersonatedUser, bool useShellCommands)
         {
             logger.MethodEntry(LogLevel.Debug);
 
             bool treatAsLocal = Server.ToLower().EndsWith(LOCAL_MACHINE_SUFFIX);
 
             if (ServerType == ServerTypeEnum.Linux || RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-                RemoteHandler = treatAsLocal ? new LinuxLocalHandler() as IRemoteHandler : new SSHHandler(Server, ServerId, ServerPassword, ServerType == ServerTypeEnum.Linux, FileTransferProtocol, SSHPort, sudoImpersonatedUser) as IRemoteHandler;
+                RemoteHandler = treatAsLocal ? new LinuxLocalHandler() as IRemoteHandler : new SSHHandler(Server, ServerId, ServerPassword, ServerType == ServerTypeEnum.Linux, FileTransferProtocol, SSHPort, sudoImpersonatedUser, useShellCommands) as IRemoteHandler;
             else
                 RemoteHandler = new WinRMHandler(Server, ServerId, ServerPassword, treatAsLocal, IncludePortInSPN);
 

RemoteFile/RemoteFileJobTypeBase.cs

@@ -29,6 +29,7 @@ public abstract class RemoteFileJobTypeBase
         internal bool IncludePortInSPN { get; set; }
         internal ApplicationSettings.FileTransferProtocolEnum FileTransferProtocol { get; set; }
         internal bool CreateCSROnDevice { get; set; }
+        internal bool UseShellCommands { get; set; }
         internal string KeyType { get; set; }
         internal int KeySize { get; set; }
         internal string SubjectText { get; set; }
@@ -57,7 +58,7 @@ internal void SetJobProperties(JobConfiguration config, CertificateStore certifi
                 ApplicationSettings.DefaultSudoImpersonatedUser :
                 properties.SudoImpersonatedUser.Value;
 
-            SSHPort = properties.SSHPort == null || string.IsNullOrEmpty(properties.SSHPort.Value) || !int.TryParse(properties.SSHPort.Value, out int notUsed) ?
+            SSHPort = properties.SSHPort == null || string.IsNullOrEmpty(properties.SSHPort.Value) || !int.TryParse(properties.SSHPort.Value, out int _) ?
                 ApplicationSettings.SSHPort :
                 properties.SSHPort;
 
@@ -73,6 +74,10 @@ internal void SetJobProperties(JobConfiguration config, CertificateStore certifi
                 ApplicationSettings.CreateCSROnDevice :
                 Convert.ToBoolean(properties.CreateCSROnDevice.Value);
 
+            UseShellCommands = properties.UseShellCommands == null || string.IsNullOrEmpty(properties.UseShellCommands.Value) ?
+                ApplicationSettings.UseShellCommands :
+                properties.UseShellCommands;
+
             FileTransferProtocol = ApplicationSettings.FileTransferProtocol;
             if (properties.FileTransferProtocol != null && !string.IsNullOrEmpty(properties.FileTransferProtocol.Value))
             {

RemoteFile/RemoteHandlers/SSHHandler.cs

@@ -31,18 +31,20 @@ class SSHHandler : BaseRemoteHandler
         private string SudoImpersonatedUser { get; set; }
         private ApplicationSettings.FileTransferProtocolEnum FileTransferProtocol { get; set; }
         private bool IsStoreServerLinux { get; set; }
+        private bool UseShellCommands { get; set; }
         private string UserId { get; set; }
         private string Password { get; set; }
         private SshClient sshClient;
 
-        internal SSHHandler(string server, string serverLogin, string serverPassword, bool isStoreServerLinux, ApplicationSettings.FileTransferProtocolEnum fileTransferProtocol, int sshPort, string sudoImpersonatedUser)
+        internal SSHHandler(string server, string serverLogin, string serverPassword, bool isStoreServerLinux, ApplicationSettings.FileTransferProtocolEnum fileTransferProtocol, int sshPort, string sudoImpersonatedUser, bool useShellCommands)
         {
             _logger.MethodEntry(LogLevel.Debug);
 
             Server = server;
             SudoImpersonatedUser = sudoImpersonatedUser;
             FileTransferProtocol = fileTransferProtocol;
             IsStoreServerLinux = isStoreServerLinux;
+            UseShellCommands = useShellCommands;
             UserId = serverLogin;
             Password = serverPassword;
 
@@ -80,7 +82,8 @@ internal SSHHandler(string server, string serverLogin, string serverPassword, bo
                 sshClient.Connect();
 
                 //method call below necessary to check edge condition where password for user id has expired. SCP (and possibly SFTP) download hangs in that scenario
-                CheckConnection();
+                if (useShellCommands)
+                    CheckConnection();
             }
             catch (Exception ex)
             {
@@ -368,13 +371,20 @@ public override void CreateEmptyStoreFile(string path, string linuxFilePermissio
             if (IsStoreServerLinux)
             {
                 string pathOnly = string.Empty;
-                SplitStorePathFile(path, out pathOnly, out _);
+                string fileName = string.Empty;
+                SplitStorePathFile(path, out pathOnly, out fileName);
 
-                linuxFilePermissions = string.IsNullOrEmpty(linuxFilePermissions) ? GetFolderPermissions(pathOnly) : linuxFilePermissions;
-                linuxFileOwner = string.IsNullOrEmpty(linuxFileOwner) ? GetFolderOwner(pathOnly) : linuxFileOwner;
+                if (UseShellCommands)
+                {
+                    linuxFilePermissions = string.IsNullOrEmpty(linuxFilePermissions) ? GetFolderPermissions(pathOnly) : linuxFilePermissions;
+                    linuxFileOwner = string.IsNullOrEmpty(linuxFileOwner) ? GetFolderOwner(pathOnly) : linuxFileOwner;
+
+                    AreLinuxPermissionsValid(linuxFilePermissions);
 
-                AreLinuxPermissionsValid(linuxFilePermissions);
-                RunCommand($"install -m {linuxFilePermissions} -o {linuxFileOwner} {linuxFileGroup} /dev/null {path}", null, ApplicationSettings.UseSudo, null);
+                    RunCommand($"install -m {linuxFilePermissions} -o {linuxFileOwner} {linuxFileGroup} /dev/null {path}", null, ApplicationSettings.UseSudo, null);
+                }
+                else
+                    UploadCertificateFile(pathOnly, fileName, Array.Empty<byte>());
             }
             else
                 RunCommand($@"Out-File -FilePath ""{path}""", null, false, null);
@@ -386,28 +396,38 @@ public override bool DoesFileExist(string path)
         {
             _logger.MethodEntry(LogLevel.Debug);
             _logger.LogDebug($"DoesFileExist: {path}");
-
-            string rtn = RunCommand($"ls {path} >> /dev/null 2>&1 && echo True || echo False", null, ApplicationSettings.UseSudo, null);
-            return Convert.ToBoolean(rtn);
-
-            //using (SftpClient client = new SftpClient(Connection))
-            //{
-            //    try
-            //    {
-            //        client.Connect();
-            //        string existsPath = FormatFTPPath(path, !IsStoreServerLinux);
-            //        bool exists = client.Exists(existsPath);
-            //        _logger.LogDebug(existsPath);
-
-            //        _logger.MethodExit(LogLevel.Debug);
-
-            //        return exists;
-            //    }
-            //    finally
-            //    {
-            //        client.Disconnect();
-            //    }
-            //}
+
+            bool exists = false;
+
+            if (UseShellCommands)
+            {
+                exists = Convert.ToBoolean(RunCommand($"ls {path} >> /dev/null 2>&1 && echo True || echo False", null, ApplicationSettings.UseSudo, null));
+            }
+            else
+            {
+                using (SftpClient client = new SftpClient(Connection))
+                {
+                    try
+                    {
+                        client.Connect();
+                        string existsPath = FormatFTPPath(path, !IsStoreServerLinux);
+                        exists = client.Exists(existsPath);
+                        _logger.LogDebug(existsPath);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(RemoteFileException.FlattenExceptionMessages(ex, $"Error checking existence of file {path} using SFTP"));
+                        throw;
+                    }
+                    finally
+                    {
+                        _logger.MethodExit(LogLevel.Debug);
+                        client.Disconnect();
+                    }
+                }
+            }
+
+            return exists;
         }
 
         public override void RemoveCertificateFile(string path, string fileName)