Cleaned up Powershell trigger failure emails

Added failure emails for ADChangeTrigger and FimServicePendingImportTrigger Added missing error details from sync step error reports (#48)
lithnet · Aug 29, 2017 · c5130ea · c5130ea
1 parent aa77775
commit c5130ea
Show file tree

Hide file tree

Showing 15 changed files with 202 additions and 39 deletions.
diff --git a/src/.vs/Lithnet.Miiserver.AutoSync/v15/sqlite3/storage.ide b/src/.vs/Lithnet.Miiserver.AutoSync/v15/sqlite3/storage.ide
diff --git a/src/Lithnet.Miiserver.AutoSync/Lithnet.Miiserver.AutoSync.csproj b/src/Lithnet.Miiserver.AutoSync/Lithnet.Miiserver.AutoSync.csproj
@@ -57,8 +57,8 @@
       <HintPath>..\packages\Lithnet.Logging.1.0.5774.20685\lib\net40\Lithnet.Logging.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="Lithnet.Miiserver.Client, Version=1.0.6434.38666, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Lithnet.Miiserver.Client.1.0.6434.38666\lib\net45\Lithnet.Miiserver.Client.dll</HintPath>
+    <Reference Include="Lithnet.Miiserver.Client, Version=1.0.6450.37102, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Lithnet.Miiserver.Client.1.0.6450.37102\lib\net45\Lithnet.Miiserver.Client.dll</HintPath>
     </Reference>
     <Reference Include="Lithnet.ResourceManagement.Client, Version=1.0.6297.14599, Culture=neutral, PublicKeyToken=bd0636700c1e2538, processorArchitecture=MSIL">
       <HintPath>..\packages\Lithnet.ResourceManagement.Client.1.0.6297.14599\lib\net40\Lithnet.ResourceManagement.Client.dll</HintPath>
@@ -186,6 +186,9 @@
     <EmbeddedResource Include="Resources\RunSummaryFragment.html" />
     <EmbeddedResource Include="Resources\EmailTemplate.html" />
   </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Resources\TriggerErrorFragment.html" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <UsingTask TaskName="CosturaCleanup" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll" TaskFactory="CodeTaskFactory">
     <ParameterGroup>

diff --git a/src/Lithnet.Miiserver.AutoSync/MAInterface/MAExecutor.cs b/src/Lithnet.Miiserver.AutoSync/MAInterface/MAExecutor.cs
@@ -251,7 +251,7 @@ private void StartTriggers()
                     t.Message += this.NotifierTriggerMessage;
                     t.Error += this.NotifierTriggerError;
                     t.TriggerFired += this.NotifierTriggerFired;
-                    t.Start();
+                    t.Start(this.ManagementAgentName);
                 }
                 catch (Exception ex)
                 {

diff --git a/src/Lithnet.Miiserver.AutoSync/Mail/MessageBuilder.cs b/src/Lithnet.Miiserver.AutoSync/Mail/MessageBuilder.cs
@@ -35,6 +35,16 @@ private static string GetTemplate(string name)
             }
         }
 
+        public static string GetMessageBody(string maName, string triggerType, string triggerDetails, DateTime errorTime, bool hasTerminated, Exception ex)
+        {
+            StringBuilder builder = new StringBuilder();
+            builder.AppendFormat(MessageBuilder.GetTemplate("TriggerErrorFragment"), maName, triggerType, triggerDetails, errorTime, hasTerminated ? "Yes" : "No", ex);
+
+            InlineResult result = PreMailer.Net.PreMailer.MoveCssInline(MessageBuilder.GetTemplate("EmailTemplate").Replace("%BODY%", builder.ToString()));
+
+            return result.Html;
+        }
+
         public static string GetMessageBody(RunDetails r)
         {
             StringBuilder builder = new StringBuilder();
@@ -64,6 +74,13 @@ public static string GetMessageBody(RunDetails r)
                 builder.AppendLine(exportErrors);
             }
 
+            string syncErrors = MessageBuilder.BuildSyncErrorDetails(r.StepDetails);
+
+            if (syncErrors != null)
+            {
+                builder.AppendLine(syncErrors);
+            }
+
 
             InlineResult result = PreMailer.Net.PreMailer.MoveCssInline(MessageBuilder.GetTemplate("EmailTemplate").Replace("%BODY%", builder.ToString()));
 
@@ -198,6 +215,30 @@ private static string BuildExportErrorDetails(IReadOnlyList<StepDetails> details
             }
         }
 
+        private static string BuildSyncErrorDetails(IReadOnlyList<StepDetails> details)
+        {
+            StringBuilder b = new StringBuilder();
+
+            foreach (StepDetails d in details)
+            {
+                string result = BuildSyncErrorDetails(d);
+
+                if (result != null)
+                {
+                    b.AppendLine(result);
+                }
+            }
+
+            if (b.Length == 0)
+            {
+                return null;
+            }
+            else
+            {
+                return b.ToString();
+            }
+        }
+
         private static string BuildStagingErrorDetails(StepDetails d)
         {
             if (d.MADiscoveryErrors.Count == 0)
@@ -290,7 +331,7 @@ private static string BuildImportErrorDetails(StepDetails d)
 
             StringBuilder errorsBuilder = new StringBuilder();
 
-            errorsBuilder.AppendLine("<h3>Synchronization errors</h3>");
+            errorsBuilder.AppendLine("<h3>Inbound synchronization errors</h3>");
 
             foreach (ImportError error in errors)
             {
@@ -301,13 +342,8 @@ private static string BuildImportErrorDetails(StepDetails d)
                 errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Date occurred", error.DateOccurred);
                 errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Retry count", error.RetryCount);
 
-                if (error.ExtensionErrorInfo != null)
-                {
-                    errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Extension name", error.ExtensionErrorInfo.ExtensionName);
-                    errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Context", error.ExtensionErrorInfo.ExtensionContext);
-                    errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Call site", error.ExtensionErrorInfo.ExtensionCallSite);
-                    errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Stack trace", error.ExtensionErrorInfo.CallStack);
-                }
+                MessageBuilder.BuildErrorExtensionInfo(error.ExtensionErrorInfo, errorBuilder);
+                MessageBuilder.BuildRulesErrorInfo(error.RulesErrorInfo, errorBuilder);
 
                 errorsBuilder.AppendLine(string.Format(MessageBuilder.GetTemplate("ErrorTableFragment"), errorBuilder));
                 errorsBuilder.AppendLine("<br/>");
@@ -404,5 +440,81 @@ private static string BuildSyncStepDetails(StepDetails d)
 
             return builder.ToString();
         }
+
+        private static string BuildSyncErrorDetails(StepDetails d)
+        {
+            if (d.MVRetryErrors == null || d.MVRetryErrors.Count == 0)
+            {
+                return null;
+            }
+
+            IEnumerable<MVRetryError> errors;
+            int remainingErrors = 0;
+
+            if (Program.ActiveConfig.Settings.MailMaxErrors <= 0 || d.MVRetryErrors.Count <= Program.ActiveConfig.Settings.MailMaxErrors)
+            {
+                errors = d.MVRetryErrors;
+            }
+            else
+            {
+                errors = d.MVRetryErrors.Take(Program.ActiveConfig.Settings.MailMaxErrors);
+                remainingErrors = d.MVRetryErrors.Count - Program.ActiveConfig.Settings.MailMaxErrors;
+            }
+
+            StringBuilder errorsBuilder = new StringBuilder();
+
+            errorsBuilder.AppendLine("<h3>Outbound synchronization errors</h3>");
+
+            foreach (MVRetryError error in errors)
+            {
+                StringBuilder errorBuilder = new StringBuilder();
+
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Object", error.DisplayName ?? error.MVID);
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Step", error.AlgorithmStep?.Value);
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Error type", error.ErrorType);
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Date occurred", error.DateOccurred);
+
+                MessageBuilder.BuildErrorExtensionInfo(error.ExtensionErrorInfo, errorBuilder);
+                MessageBuilder.BuildRulesErrorInfo(error.RulesErrorInfo, errorBuilder);
+
+                errorsBuilder.AppendLine(string.Format(MessageBuilder.GetTemplate("ErrorTableFragment"), errorBuilder));
+                errorsBuilder.AppendLine("<br/>");
+            }
+
+            if (remainingErrors > 0)
+            {
+                errorsBuilder.Append($"There are {remainingErrors} more errors that are not shown in this report<br/>");
+            }
+
+            return errorsBuilder.ToString();
+        }
+
+        private static void BuildRulesErrorInfo(RulesErrorInfoContext c, StringBuilder errorBuilder)
+        {
+            if (c != null)
+            {
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "CS Object ID", c.CSObjectID);
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "DN", c.DN);
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "MA Name", c.MAName);
+
+                if (c.AttributeFlow != null)
+                {
+                    errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Context", c.AttributeFlow.ContextID);
+                    errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Destination attribute", c.AttributeFlow.DestinationAttribute);
+                    errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Flow rule", c.AttributeFlow.FlowRule);
+                }
+            }
+        }
+
+        private static void BuildErrorExtensionInfo(ExtensionErrorInfo error, StringBuilder errorBuilder)
+        {
+            if (error != null)
+            {
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Extension name", error.ExtensionName);
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Context", error.ExtensionContext);
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Call site", error.ExtensionCallSite);
+                errorBuilder.AppendFormat(MessageBuilder.SimpleRow, "Call stack", error.CallStack);
+            }
+        }
     }
 }
diff --git a/src/Lithnet.Miiserver.AutoSync/Resources/TriggerErrorFragment.html b/src/Lithnet.Miiserver.AutoSync/Resources/TriggerErrorFragment.html
@@ -0,0 +1,12 @@
+<table class="profileSummary">
+    <tr><th>Management agent</th><td>{0}</td></tr>
+    <tr><th>Trigger type</th><td>{1}</td></tr>
+    <tr><th>Detail</th><td>{2}</td></tr>
+    <tr><th>Error time</th><td>{3}</td></tr>
+    <tr><th>Has been terminated</th><td>{4}</td></tr>
+</table>
+<br/>
+<h3>Error Details</h3>
+<pre>
+{5}
+</pre>
diff --git a/src/Lithnet.Miiserver.AutoSync/Triggers/ActiveDirectoryChangeTrigger.cs b/src/Lithnet.Miiserver.AutoSync/Triggers/ActiveDirectoryChangeTrigger.cs
@@ -119,6 +119,13 @@ private void SetupListener()
             catch (Exception ex)
             {
                 this.LogError("Could not start the listener", ex);
+
+                if (MessageSender.CanSendMail())
+                {
+                    string messageContent = MessageBuilder.GetMessageBody(this.ManagementAgentName, this.Type, this.Description, DateTime.Now, true, ex);
+                    MessageSender.SendMessage($"{this.ManagementAgentName}: {this.Type} trigger error", messageContent);
+                }
+
                 throw;
             }
         }
@@ -213,16 +220,30 @@ private void Notify(IAsyncResult result)
                 else
                 {
                     this.LogError("The AD change listener encountered an unexpected error", ex);
+
+                    if (MessageSender.CanSendMail())
+                    {
+                        string messageContent = MessageBuilder.GetMessageBody(this.ManagementAgentName, this.Type, this.Description, DateTime.Now, false, ex);
+                        MessageSender.SendMessage($"{this.ManagementAgentName}: {this.Type} trigger error", messageContent);
+                    }
                 }
             }
             catch (Exception ex)
             {
                 this.LogError("The AD change listener encountered an unexpected error", ex);
+
+                if (MessageSender.CanSendMail())
+                {
+                    string messageContent = MessageBuilder.GetMessageBody(this.ManagementAgentName, this.Type, this.Description, DateTime.Now, false, ex);
+                    MessageSender.SendMessage($"{this.ManagementAgentName}: {this.Type} trigger error", messageContent);
+                }
             }
         }
 
-        public override void Start()
+        public override void Start(string managementAgentName)
         {
+            this.ManagementAgentName = managementAgentName;
+
             if (this.Disabled)
             {
                 this.Log("AD/LDS change listener disabled");

diff --git a/src/Lithnet.Miiserver.AutoSync/Triggers/FimServicePendingImportTrigger.cs b/src/Lithnet.Miiserver.AutoSync/Triggers/FimServicePendingImportTrigger.cs
@@ -64,11 +64,19 @@ private void CheckTimer_Elapsed(object sender, ElapsedEventArgs e)
             catch (Exception ex)
             {
                 this.LogError("Change detection failed", ex);
+
+                if (MessageSender.CanSendMail())
+                {
+                    string messageContent = MessageBuilder.GetMessageBody(this.ManagementAgentName, this.Type, this.Description, DateTime.Now, false, ex);
+                    MessageSender.SendMessage($"{this.ManagementAgentName}: {this.Type} trigger error", messageContent);
+                }
             }
         }
-        
-        public override void Start()
+
+        public override void Start(string managementAgentName)
         {
+            this.ManagementAgentName = managementAgentName;
+
             this.checkTimer = new Timer
             {
                 AutoReset = true,

diff --git a/src/Lithnet.Miiserver.AutoSync/Triggers/IMAExecutionTrigger.cs b/src/Lithnet.Miiserver.AutoSync/Triggers/IMAExecutionTrigger.cs
@@ -20,7 +20,7 @@ public interface IMAExecutionTrigger
 
         event TriggerMessageEventHandler Error;
 
-        void Start();
+        void Start(string managementAgentName);
 
         void Stop();
     }

diff --git a/src/Lithnet.Miiserver.AutoSync/Triggers/IntervalExecutionTrigger.cs b/src/Lithnet.Miiserver.AutoSync/Triggers/IntervalExecutionTrigger.cs
@@ -27,8 +27,10 @@ private void CheckTimer_Elapsed(object sender, ElapsedEventArgs e)
             this.Fire(this.RunProfileName);
         }
 
-        public override void Start()
+        public override void Start(string managementAgentName)
         {
+            this.ManagementAgentName = managementAgentName;
+
             if (this.RunProfileName == null)
             {
                 this.LogError("Ignoring interval trigger with no run profile name");

diff --git a/src/Lithnet.Miiserver.AutoSync/Triggers/MAExecutionTrigger.cs b/src/Lithnet.Miiserver.AutoSync/Triggers/MAExecutionTrigger.cs
@@ -24,7 +24,9 @@ public abstract class MAExecutionTrigger : IMAExecutionTrigger
 
         public event TriggerMessageEventHandler Error;
 
-        public abstract void Start();
+        protected string ManagementAgentName { get; set; }
+
+        public abstract void Start(string managementAgentName);
 
         public abstract void Stop();
 

diff --git a/src/Lithnet.Miiserver.AutoSync/Triggers/PowerShellExecutionTrigger.cs b/src/Lithnet.Miiserver.AutoSync/Triggers/PowerShellExecutionTrigger.cs
@@ -36,11 +36,13 @@ public class PowerShellExecutionTrigger : MAExecutionTrigger
         [DataMember(Name = "interval")]
         public TimeSpan Interval { get; set; }
 
-        public override void Start()
+        public override void Start(string managementAgentName)
         {
+            this.ManagementAgentName = managementAgentName;
+
             if (!System.IO.File.Exists(this.ScriptPath))
             {
-                this.LogError($"Could not start PowerShell trigger as the script '{this.ScriptPath}' could not be found");
+                this.LogError($"Could not start PowerShell trigger for MA {managementAgentName} as the script '{this.ScriptPath}' could not be found");
                 return;
             }
 
@@ -91,26 +93,19 @@ private void Run()
                     }
                     catch (Exception ex)
                     {
-                        string message;
-
-                        if (this.ExceptionBehaviour == ExecutionErrorBehaviour.Terminate)
-                        {
-                            message = $"The PowerShell execution trigger '{this.DisplayName}' encountered an error and has been terminated";
-                        }
-                        else
-                        {
-                            message = $"The PowerShell execution trigger '{this.DisplayName}' encountered an error";
-                        }
+                        bool shouldTerminate = this.ExceptionBehaviour == ExecutionErrorBehaviour.Terminate;
 
-                        this.LogError(message, ex);
+                        this.LogError($"The PowerShell execution trigger '{this.DisplayName}' encountered an error", ex);
 
                         if (MessageSender.CanSendMail())
                         {
-                            MessageSender.SendMessage(message, ex.ToString());
+                            string messageContent = MessageBuilder.GetMessageBody(this.ManagementAgentName, this.Type, this.Description, DateTime.Now, shouldTerminate, ex);
+                            MessageSender.SendMessage($"{this.ManagementAgentName}: {this.Type} trigger error", messageContent);
                         }
 
-                        if (this.ExceptionBehaviour == ExecutionErrorBehaviour.Terminate)
+                        if (shouldTerminate)
                         {
+                            this.Log("The PowerShell trigger has been terminated as specified by config");
                             break;
                         }
                         else
@@ -152,7 +147,8 @@ private void Run()
 
                 if (MessageSender.CanSendMail())
                 {
-                    MessageSender.SendMessage($"The PowerShell execution trigger '{this.DisplayName}' encountered an error and has been terminated", ex.ToString());
+                    string messageContent = MessageBuilder.GetMessageBody(this.ManagementAgentName, this.Type, this.Description, DateTime.Now, true, ex);
+                    MessageSender.SendMessage($"{this.ManagementAgentName}: {this.Type} trigger error", messageContent);
                 }
             }
         }

diff --git a/src/Lithnet.Miiserver.AutoSync/Triggers/ScheduledExecutionTrigger.cs b/src/Lithnet.Miiserver.AutoSync/Triggers/ScheduledExecutionTrigger.cs
@@ -54,17 +54,24 @@ private void SetRemainingMilliseconds()
         private void CheckTimer_Elapsed(object sender, ElapsedEventArgs e)
         {
             this.Fire(this.RunProfileName);
-            this.Start();
+            this.ResetTimer();
         }
 
-        public override void Start()
+        public override void Start(string managementAgentName)
         {
+            this.ManagementAgentName = managementAgentName;
+
             if (this.RunProfileName == null)
             {
                 this.LogError("Ignoring scheduled trigger with no run profile name");
                 return;
             }
 
+            this.ResetTimer();
+        }
+
+        private void ResetTimer()
+        {
             this.SetRemainingMilliseconds();
             this.checkTimer = new Timer
             {