Fix coverage reporting across multiple invocations of the testing engine (#450)

Author: akashlal

Source/Core/Runtime/Machines/MachineFactory.cs

@@ -17,7 +17,7 @@ internal static class MachineFactory
         /// <summary>
         /// Cache storing machine constructors.
         /// </summary>
-        private static Dictionary<Type, Func<Machine>> MachineConstructorCache =
+        private static readonly Dictionary<Type, Func<Machine>> MachineConstructorCache =
             new Dictionary<Type, Func<Machine>>();
 
         /// <summary>
@@ -39,18 +39,5 @@ public static Machine Create(Type type)
                 return constructor();
             }
         }
-
-        /// <summary>
-        /// Checks if the constructor of the specified machine type exists in the cache.
-        /// </summary>
-        /// <param name="type">Type of the machine.</param>
-        /// <returns>True if the constructor exists, else false.</returns>
-        internal static bool IsCached(Type type)
-        {
-            lock (MachineConstructorCache)
-            {
-                return MachineConstructorCache.ContainsKey(type);
-            }
-        }
     }
 }

Source/TestingServices/Runtime/SystematicTestingRuntime.cs

@@ -495,7 +495,6 @@ private Machine CreateMachine(MachineId mid, Type type, string machineName, Mach
                 opGroupId = creator.OperationGroupId;
             }
 
-            var isMachineTypeCached = MachineFactory.IsCached(type);
             Machine machine = MachineFactory.Create(type);
             this.MachineOperations.GetOrAdd(mid.Value, new AsyncOperation(mid));
             IMachineStateManager stateManager = new SerializedMachineStateManager(this, machine, opGroupId);
@@ -504,7 +503,7 @@ private Machine CreateMachine(MachineId mid, Type type, string machineName, Mach
             machine.Initialize(this, mid, stateManager, eventQueue);
             machine.InitializeStateInformation();
 
-            if (this.Configuration.ReportActivityCoverage && !isMachineTypeCached)
+            if (this.Configuration.ReportActivityCoverage)
             {
                 this.ReportActivityCoverageOfMachine(machine);
             }
@@ -1474,18 +1473,20 @@ private void ReportActivityCoverageOfMonitorEvent(AsyncMachine sender, Monitor m
         private void ReportActivityCoverageOfMachine(Machine machine)
         {
             var machineName = machine.GetType().FullName;
+            if (this.CoverageInfo.IsMachineDeclared(machineName))
+            {
+                return;
+            }
 
             // Fetch states.
             var states = machine.GetAllStates();
-
             foreach (var state in states)
             {
                 this.CoverageInfo.DeclareMachineState(machineName, state);
             }
 
             // Fetch registered events.
             var pairs = machine.GetAllStateEventPairs();
-
             foreach (var tup in pairs)
             {
                 this.CoverageInfo.DeclareStateEvent(machineName, tup.Item1, tup.Item2);

Source/TestingServices/Statistics/Coverage/CoverageInfo.cs

@@ -44,14 +44,14 @@ public CoverageInfo()
             this.Transitions = new HashSet<Transition>();
         }
 
+        /// <summary>
+        /// Checks if the machine type has already been registered for coverage.
+        /// </summary>
+        public bool IsMachineDeclared(string machineName) => this.MachinesToStates.ContainsKey(machineName);
+
         /// <summary>
         /// Adds a new transition.
         /// </summary>
-        /// <param name="machineOrigin">Origin machine</param>
-        /// <param name="stateOrigin">Origin state</param>
-        /// <param name="edgeLabel">Edge label</param>
-        /// <param name="machineTarget">Target machine</param>
-        /// <param name="stateTarget">Target state</param>
         public void AddTransition(string machineOrigin, string stateOrigin, string edgeLabel,
             string machineTarget, string stateTarget)
         {
@@ -64,19 +64,11 @@ public void AddTransition(string machineOrigin, string stateOrigin, string edgeL
         /// <summary>
         /// Declares a state.
         /// </summary>
-        /// <param name="machine">Machine name</param>
-        /// <param name="state">state name</param>
-        public void DeclareMachineState(string machine, string state)
-        {
-            this.AddState(machine, state);
-        }
+        public void DeclareMachineState(string machine, string state) => this.AddState(machine, state);
 
         /// <summary>
         /// Declares a registered state, event pair.
         /// </summary>
-        /// <param name="machine">Machine name</param>
-        /// <param name="state">state name</param>
-        /// <param name="eventName">Event name that the state is prepared to handle</param>
         public void DeclareStateEvent(string machine, string state, string eventName)
         {
             this.AddState(machine, state);
@@ -87,7 +79,6 @@ public void DeclareStateEvent(string machine, string state, string eventName)
         /// Merges the information from the specified
         /// coverage info. This is not thread-safe.
         /// </summary>
-        /// <param name="coverageInfo">CoverageInfo</param>
         public void Merge(CoverageInfo coverageInfo)
         {
             foreach (var machine in coverageInfo.MachinesToStates)
@@ -113,8 +104,6 @@ public void Merge(CoverageInfo coverageInfo)
         /// <summary>
         /// Adds a new state.
         /// </summary>
-        /// <param name="machineName">Machine name</param>
-        /// <param name="stateName">State name</param>
         private void AddState(string machineName, string stateName)
         {
             if (!this.MachinesToStates.ContainsKey(machineName))

Tests/TestingServices.Tests/Coverage/ActivityCoverageTest.cs

@@ -220,5 +220,71 @@ public void TestMachineSendEventActivityCoverage()
             expected = RemoveExcessiveEmptySpaceFromReport(expected);
             Assert.Equal(expected, result);
         }
+
+        internal class M4 : Machine
+        {
+            [Start]
+            [OnEventGotoState(typeof(E), typeof(Done))]
+            internal class Init : MachineState
+            {
+            }
+
+            internal class Done : MachineState
+            {
+            }
+        }
+
+        [Fact(Timeout = 5000)]
+        public void TestCoverageOnMultipleTests()
+        {
+            var configuration = Configuration.Create().WithVerbosityEnabled();
+            configuration.ReportActivityCoverage = true;
+
+            ITestingEngine testingEngine1 = this.Test(r =>
+            {
+                var m = r.CreateMachine(typeof(M4));
+                r.SendEvent(m, new E());
+            },
+            configuration);
+
+            // Assert that the coverage is as expected.
+            var coverage1 = testingEngine1.TestReport.CoverageInfo;
+            Assert.Contains(typeof(M4).FullName, coverage1.MachinesToStates.Keys);
+            Assert.Contains(typeof(M4.Init).Name, coverage1.MachinesToStates[typeof(M4).FullName]);
+            Assert.Contains(typeof(M4.Done).Name, coverage1.MachinesToStates[typeof(M4).FullName]);
+            Assert.Contains(coverage1.RegisteredEvents, tup => tup.Item3 == typeof(E).FullName);
+
+            ITestingEngine testingEngine2 = this.Test(r =>
+            {
+                var m = r.CreateMachine(typeof(M4));
+                r.SendEvent(m, new E());
+            },
+            configuration);
+
+            // Assert that the coverage is the same as before.
+            var coverage2 = testingEngine2.TestReport.CoverageInfo;
+            Assert.Contains(typeof(M4).FullName, coverage2.MachinesToStates.Keys);
+            Assert.Contains(typeof(M4.Init).Name, coverage2.MachinesToStates[typeof(M4).FullName]);
+            Assert.Contains(typeof(M4.Done).Name, coverage2.MachinesToStates[typeof(M4).FullName]);
+            Assert.Contains(coverage2.RegisteredEvents, tup => tup.Item3 == typeof(E).FullName);
+
+            string coverageReport1, coverageReport2;
+
+            var activityCoverageReporter = new ActivityCoverageReporter(coverage1);
+            using (var writer = new StringWriter())
+            {
+                activityCoverageReporter.WriteCoverageText(writer);
+                coverageReport1 = writer.ToString();
+            }
+
+            activityCoverageReporter = new ActivityCoverageReporter(coverage2);
+            using (var writer = new StringWriter())
+            {
+                activityCoverageReporter.WriteCoverageText(writer);
+                coverageReport2 = writer.ToString();
+            }
+
+            Assert.Equal(coverageReport1, coverageReport2);
+        }
     }
 }