fix(parser): add template cleanup

m4lm · web-flow · commit dd7b817c7e6e · 2024-04-08T09:51:09.000+02:00
Added template cleanup by moving dictionary to a separate service. Now we also check whether template is already in dictionary, prevent ArgumentException.
diff --git a/Packrat/Fennec.Tests/Parsers/IpFixParserTests.cs b/Packrat/Fennec.Tests/Parsers/IpFixParserTests.cs
@@ -4,6 +4,7 @@
 using Fennec.Services;
 using NSubstitute;
 using Serilog;
+using IpFixTemplateRecord = DotNetFlow.Ipfix.TemplateRecord;
 
 namespace Fennec.Tests.Parsers;
 
@@ -15,8 +16,10 @@ public void ParseIpfixMessage()
         // Arrange
         var substituteLogger = Substitute.For<ILogger>();
         var substituteMetricService = Substitute.For<IMetricService>();
+        var substituteTemplateCleanupService = Substitute.For<IIpFixCleanupService>();
+        substituteTemplateCleanupService.TemplateRecords.Returns(new Dictionary<(IPAddress, ushort), IpFixTemplateRecord>());
 
-        var parser = new IpFixParser(substituteLogger, substituteMetricService);
+        var parser = new IpFixParser(substituteLogger, substituteMetricService,substituteTemplateCleanupService);
         var ipfixTemplates = Convert.FromBase64String(
             "AAoFgGUlYdkB1iL/AAAAAAACA3wBAAAVAAgABAAMAAQAAQAIAAIACACYAAgAmQAIAAcAAgALAAIACgAEAA4ABAFfAAgABAABAIgAAQAGAAEABQABADUAAQA9AAGDegACAAAa3IN4AAIAABrcg3kAAQAAGtwA0gABAQEAEgAIAAQADAAEAAEACAACAAgAmAAIAJkACAAKAAQADgAEAAQAAQCIAAEABQABADUAAQA9AAEBXwAIg3oAAgAAGtyDeAACAAAa3IN5AAEAABrcANIAAgECABUAGwAQABwAEAABAAgAAgAIAJgACACZAAgABwACAAsAAgAKAAQADgAEAV8ACAAEAAEAiAABAAYAAQAFAAEANQABAD0AAYN6AAIAABrcg3gAAgAAGtyDeQABAAAa3ADSAAEBAwASABsAEAAcABAAAQAIAAIACACYAAgAmQAIAAoABAAOAAQABAABAIgAAQAFAAEANQABAD0AAQFfAAiDegACAAAa3IN4AAIAABrcg3kAAQAAGtwA0gACAQQAGQAIAAQADAAEAAEACAACAAgAmAAIAJkACAAHAAIACwACAAoABAAOAAQABAABAIgAAQAGAAEABQABADUAAQA9AAEBXwAIg3EABAAAGtyDcgAEAAAa3IN2AAIAABrcg3cAAgAAGtyDcAABAAAa3IN6AAIAABrcg3gAAgAAGtyDeQABAAAa3AEFABcACAAEAAwABAABAAgAAgAIAJgACACZAAgABwACAAsAAgAKAAQADgAEAAQAAQCIAAEABQABADUAAQA9AAEBXwAIg3EABAAAGtyDcgAEAAAa3INwAAEAABrcg3oAAgAAGtyDeAACAAAa3IN5AAEAABrcANIAAQEGABkACAAEAAwABAABAAgAAgAIAJgACACZAAgABwACAAsAAgAKAAQADgAEAAQAAQCIAAEABgABAAUAAQA1AAEAPQABAV8ACINzABAAABrcg3QAEAAAGtyDdgACAAAa3IN3AAIAABrcg3AAAQAAGtyDegACAAAa3IN4AAIAABrcg3kAAQAAGtwBBwAXAAgABAAMAAQAAQAIAAIACACYAAgAmQAIAAcAAgALAAIACgAEAA4ABAAEAAEABQABADUAAQA9AAEAiAABAV8ACINzABAAABrcg3QAEAAAGtyDcAABAAAa3IN6AAIAABrcg3gAAgAAGtyDeQABAAAa3ADSAAEAAgDMAQgAFQAIAAQADAAEAAEACAACAAgAmAAIAJkACAAHAAIACwACAAoABAAOAAQBXwAIAAQAAQCIAAEABgABAAUAAQA1AAEAPQABg3oAAgAAGtyDeAACAAAa3IN5AAEAABrcANIAAQEJABUAGwAQABwAEAABAAgAAgAIAJgACACZAAgABwACAAsAAgAKAAQADgAEAV8ACAAEAAEAiAABAAYAAQAFAAEANQABAD0AAYN6AAIAABrcg3gAAgAAGtyDeQABAAAa3ADSAAEAAgDMAQoAFQAIAAQADAAEAAEACAACAAgAmAAIAJkACAAHAAIACwACAAoABAAOAAQBXwAIAAQAAQCIAAEABgABAAUAAQA1AAEAPQABg3oAAgAAGtyDeAACAAAa3IN5AAEAABrcANIAAQELABUAGwAQABwAEAABAAgAAgAIAJgACACZAAgABwACAAsAAgAKAAQADgAEAV8ACAAEAAEAiAABAAYAAQAFAAEANQABAD0AAYN6AAIAABrcg3gAAgAAGtyDeQABAAAa3ADSAAEAAgBcAQwAEgAIAAQADAAEAAEACAACAAgAmAAIAJkACAAKAAQADgAEAAQAAQCIAAEABQABADUAAQA9AAEBXwAIg3oAAgAAGtyDeAACAAAa3IN5AAEAABrcANIAAg==");
         var ipfixData = Convert.FromBase64String(
diff --git a/Packrat/Fennec.Tests/Parsers/NetFlow9ParserTests.cs b/Packrat/Fennec.Tests/Parsers/NetFlow9ParserTests.cs
@@ -1,5 +1,6 @@
 ﻿using System.Net;
 using System.Net.Sockets;
+using NetFlow9TemplateRecord = DotNetFlow.Netflow9.TemplateRecord;
 using Fennec.Parsers;
 using Fennec.Services;
 using NSubstitute;
@@ -15,8 +16,10 @@ public void ParseNetFlow9Message()
         // Arrange
         var substituteLogger = Substitute.For<ILogger>();
         var substituteMetricService = Substitute.For<IMetricService>();
+        var substituteTemplateCleanupService = Substitute.For<INetFlow9CleanupService>();
+        substituteTemplateCleanupService.TemplateRecords.Returns(new Dictionary<(IPAddress, ushort), NetFlow9TemplateRecord>());
 
-        var parser = new NetFlow9Parser(substituteLogger, substituteMetricService);
+        var parser = new NetFlow9Parser(substituteLogger, substituteMetricService, substituteTemplateCleanupService);
         var netflowTemplates = Convert.FromBase64String(
             "AAkAAgBaBI9k0SU7AAAAaAAAAAIBAwBRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAzC8AxwKgAlgG71i0GAAAAAQBZiYAAWYmAAAApaQAAAAAAAABAAAAAAAAAAAEAAAA8AQMADQAbABAAHAAQAAgABAAMAAQABwACAAsAAgAEAAEAMAAEABYABAAVAAQACgAEAAEACAACAAg=");
         var netflowData = Convert.FromBase64String(
diff --git a/Packrat/Fennec/Options/TemplateCleanupOptions.cs b/Packrat/Fennec/Options/TemplateCleanupOptions.cs
@@ -0,0 +1,12 @@
+﻿using Microsoft.Extensions.Options;
+
+namespace Fennec.Options;
+
+/// <summary>
+/// Options for cleaning up IPFIX and NetFlow9 templates.
+/// </summary>
+public class TemplateCleanupOptions
+{
+    public TimeSpan IpFixCleanupInterval { get; set; } = TimeSpan.FromDays(2);
+    public TimeSpan NetFlow9CleanupInterval { get; set; } = TimeSpan.FromDays(2);
+}
diff --git a/Packrat/Fennec/Parsers/IpFixParser.cs b/Packrat/Fennec/Parsers/IpFixParser.cs
@@ -1,4 +1,5 @@
-﻿using System.Net;
+﻿using System.Collections.Concurrent;
+using System.Net;
 using System.Net.Sockets;
 using DotNetFlow.Ipfix;
 using Fennec.Database;
@@ -16,22 +17,21 @@ public class IpFixParser : IParser
 {
     private readonly ILogger _log;
     // TODO: expand _templateRecords to a service, can be used to display/monitor templates in frontend
-    private readonly IDictionary<(IPAddress, ushort), TemplateRecord> _templateRecords;
     private readonly IMetricService _metricService;
+    private readonly IIpFixCleanupService _templateCleanupService;
 
-    public IpFixParser(ILogger log,  IMetricService metricService)
+    public IpFixParser(ILogger log,  IMetricService metricService, IIpFixCleanupService templateCleanupService)
     {
         _log = log.ForContext<IpFixParser>();
-        _templateRecords = new Dictionary<(IPAddress, ushort), TemplateRecord>();
         _metricService = metricService;
+        _templateCleanupService = templateCleanupService;
     }
 
     public IEnumerable<TraceImportInfo> Parse(UdpReceiveResult result)
     {
-        // result.RemoteEndPoint.Address --> address of exporter
         var stream = new MemoryStream(result.Buffer);
 
-        using var ipfixReader = new IpfixReader(stream, 0, _templateRecords.Values);
+        using var ipfixReader = new IpfixReader(stream, 0, _templateCleanupService.TemplateRecords.Values);
         _ = ipfixReader.ReadPacketHeader();
 
         // Process each set in the IPFIX message. Has to be while(true) because header doesn't provide a count of sets.
@@ -45,7 +45,7 @@ public IEnumerable<TraceImportInfo> Parse(UdpReceiveResult result)
                 {
                     case DataSet dataSet:
                         var key = (result.RemoteEndPoint.Address, set.ID);
-                        if (!_templateRecords.TryGetValue(key, out var template))
+                        if (!_templateCleanupService.TemplateRecords.TryGetValue(key, out var template))
                         {
                             _log.Warning("Could not parse data set... " +
                                          "Reading this set requires a not yet transmitted " +
@@ -58,7 +58,9 @@ public IEnumerable<TraceImportInfo> Parse(UdpReceiveResult result)
                     case TemplateSet templateSet:
                         foreach (var templateRecord in templateSet.Records)
                         {
-                            _templateRecords.Add((result.RemoteEndPoint.Address, templateRecord.ID), templateRecord);
+                            if (_templateCleanupService.TemplateRecords.ContainsKey((result.RemoteEndPoint.Address, templateRecord.ID)))
+                                continue;
+                            _templateCleanupService.TemplateRecords.Add((result.RemoteEndPoint.Address, templateRecord.ID), templateRecord);
                             _log.Information("Received new template set with id #{TemplateSetId}", templateRecord.ID);
                         }
 
diff --git a/Packrat/Fennec/Parsers/NetFlow9Parser.cs b/Packrat/Fennec/Parsers/NetFlow9Parser.cs
@@ -13,33 +13,33 @@ namespace Fennec.Parsers;
 public class NetFlow9Parser : IParser
 {
     private readonly ILogger _log;
-    private readonly IDictionary<(IPAddress, ushort), TemplateRecord> _templateRecords; // matches (ExporterIp, TemplateId) to TemplateRecord
     private readonly IMetricService _metricService;
-    
-    public NetFlow9Parser(ILogger log, IMetricService metricService)
+    private readonly INetFlow9CleanupService _templateCleanupService;
+
+    public NetFlow9Parser(ILogger log, IMetricService metricService, INetFlow9CleanupService templateCleanupService)
     {
         _log = log.ForContext<NetFlow9Parser>();
-        _templateRecords = new Dictionary<(IPAddress, ushort), TemplateRecord>();
         _metricService = metricService;
+        _templateCleanupService = templateCleanupService;
     }
     public IEnumerable<TraceImportInfo> Parse(UdpReceiveResult result)
     {
         var stream = new MemoryStream(result.Buffer);
-        using var nr = new NetflowReader(stream, 0, _templateRecords.Values);
+        using var nr = new NetflowReader(stream, 0, _templateCleanupService.TemplateRecords.Values);
         var header = nr.ReadPacketHeader();
 
         for (var i = 0; i < header.Count; i++)
         {
             try
             {
-                var dict = _templateRecords.Values.ToDictionary(t => t.ID, t => t);
+                var dict = _templateCleanupService.TemplateRecords.Values.ToDictionary(t => t.ID, t => t);
                 var set = nr.ReadFlowSet(dict);
 
                 switch (set)
                 {
                     case DataFlowSet dataFlowSet:
                         var key = (result.RemoteEndPoint.Address, set.ID);
-                        if (!_templateRecords.TryGetValue(key, out var template))
+                        if (!_templateCleanupService.TemplateRecords.TryGetValue(key, out var template))
                         {
                             _log.Warning("Could not parse data set... " +
                                          "Reading this set requires a not yet transmitted " +
@@ -52,7 +52,9 @@ public IEnumerable<TraceImportInfo> Parse(UdpReceiveResult result)
                     case TemplateFlowSet templateFlowSet:
                         foreach (var templateRecord in templateFlowSet.Records)
                         {
-                            _templateRecords.Add((result.RemoteEndPoint.Address, templateRecord.ID), templateRecord);
+                            if (_templateCleanupService.TemplateRecords.ContainsKey((result.RemoteEndPoint.Address, templateRecord.ID)))
+                                continue;
+                            _templateCleanupService.TemplateRecords.Add((result.RemoteEndPoint.Address, templateRecord.ID), templateRecord);
                             _log.Information("Received new template set with id #{TemplateSetId}", templateRecord.ID);
                         }
 
diff --git a/Packrat/Fennec/Services/TemplateCleanupService.cs b/Packrat/Fennec/Services/TemplateCleanupService.cs
@@ -0,0 +1,76 @@
+﻿using System.Collections.Concurrent;
+using System.Net;
+using DotNetFlow.Ipfix;
+using Fennec.Options;
+using Microsoft.Extensions.Options;
+using IpFixTemplateRecord = DotNetFlow.Ipfix.TemplateRecord;
+using Netflow9TemplateRecord = DotNetFlow.Netflow9.TemplateRecord;
+
+namespace Fennec.Services;
+
+/// <summary>
+/// Interface for cleanup service for IPFIX templates
+/// </summary>
+public interface IIpFixCleanupService
+{
+    Dictionary<(IPAddress, ushort), IpFixTemplateRecord> TemplateRecords { get; set; }
+}
+
+/// <summary>
+/// Interface for cleanup service for NetFlow9 templates
+/// </summary>
+public interface INetFlow9CleanupService
+{
+    Dictionary<(IPAddress, ushort), Netflow9TemplateRecord> TemplateRecords { get; set; }
+}
+
+/// <summary>
+/// Cleanup service for IPFIX templates
+/// </summary>
+public class IpFixTemplateCleanupService : BackgroundService, IIpFixCleanupService
+{
+    public Dictionary<(IPAddress, ushort), IpFixTemplateRecord> TemplateRecords { get; set; }
+    private readonly TimeSpan _cleanupInterval;
+
+    public IpFixTemplateCleanupService(IOptions<TemplateCleanupOptions> options)
+    {
+        TemplateRecords = new Dictionary<(IPAddress, ushort), IpFixTemplateRecord>();
+        _cleanupInterval = options.Value.IpFixCleanupInterval;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            await Task.Delay(_cleanupInterval, stoppingToken);
+
+            TemplateRecords.Clear();
+        }
+    }
+}
+
+/// <summary>
+/// Cleanup service for NetFlow9 templates
+/// </summary>
+public class NetFlow9TemplateCleanupService : BackgroundService, INetFlow9CleanupService
+{
+    public Dictionary<(IPAddress, ushort), Netflow9TemplateRecord> TemplateRecords { get; set; }
+    private readonly TimeSpan _cleanupInterval;
+
+
+    public NetFlow9TemplateCleanupService(IOptions<TemplateCleanupOptions> options)
+    {
+        TemplateRecords = new Dictionary<(IPAddress, ushort), Netflow9TemplateRecord>();
+        _cleanupInterval = options.Value.NetFlow9CleanupInterval;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            await Task.Delay(_cleanupInterval, stoppingToken);
+
+            TemplateRecords.Clear();
+        }
+    }
+}
diff --git a/Packrat/Fennec/Startup.cs b/Packrat/Fennec/Startup.cs
@@ -74,6 +74,18 @@ public void ConfigureServices(IServiceCollection services, IWebHostEnvironment e
         // Metric service
         services.AddSingleton<IMetricService, MetricService>();
         
+        // Template cleanup services
+        services.Configure<TemplateCleanupOptions>(Configuration.GetSection("TemplateCleanupOptions"));
+        
+        services.AddSingleton<IIpFixCleanupService, IpFixTemplateCleanupService>();
+        services.AddSingleton<INetFlow9CleanupService, NetFlow9TemplateCleanupService>();
+        
+        // services.AddSingleton<IpFixTemplateCleanupService>();
+        // services.AddSingleton<NetFlow9TemplateCleanupService>();
+
+        // services.AddHostedService<IpFixTemplateCleanupService>();
+        // services.AddHostedService<NetFlow9TemplateCleanupService>();
+
         // Database services
         services.AddSingleton<ITraceRepository, TraceRepository>();
         services.AddSingleton<ITimeService, TimeService>();
diff --git a/Packrat/Fennec/appsettings.json b/Packrat/Fennec/appsettings.json
@@ -5,20 +5,17 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  
   "_Startup": "Settings that will only be used at startup.",
   "Startup": {
     "_EnableSwagger": "Enable or disable the Swagger UI.",
     "EnableSwagger": false,
     "_AllowCors": "Allow CORS for every request.",
     "AllowCors": false
-  },  
-  
+  },
   "ConnectionStrings": {
     "_MongoConnection": "The connection string to the MongoDB database.",
     "MongoConnection": "mongodb://mongo:27017"
   },
-  
   "_Multiplexers": "Define ports and for what protocols to listen on these ports.",
   "Multiplexers": [
     {
@@ -35,15 +32,15 @@
       "Enabled": true,
       "Name": "Ipfix Multiplexer",
       "ListeningPort": 2056,
-      "Parsers": [ "Ipfix" ]
+      "Parsers": [
+        "Ipfix"
+      ]
     }
   ],
-  
   "_Security": "Security configuration.",
   "Security": {
     "_Enabled": "Enable or disable the requirement for authentication to access protected endpoints.",
     "Enabled": true,
-    
     "_Access": "Configuration for users and how accessing endpoints should behave.",
     "Access": {
       "_Username": "The username of the default user. Only generated if no users exist in the database.",
@@ -52,49 +49,46 @@
       "Password": null
     }
   },
-  
+  "_TemplateCleanupOptions": "TemplateCleanup configuration.",
+  "TemplateCleanupOptions": {
+    "_IpFixCleanupInterval": "Interval at which the IpFixCleanupService should check for expired templates. Format is dd:hh:mm:ss.",
+    "IpFixCleanupInterval": "02:00:00:00",
+    "_NetFlow9CleanupInterval": "Interval at which the NetFlow9CleanupService should check for expired templates. Format is dd:hh:mm:ss.",
+    "NetFlow9CleanupInterval": "02:00:00:00"
+  },
   "_DnsCache": "DnsCache configuration.",
   "DnsCache": {
     "_Enabled": "Enable or disable the DnsCacheService.",
     "Enabled": true,
-    
     "_CleanupInterval": "Interval at which the DnsCacheCleanupService should check for expired DNS records. Format is dd:hh:mm:ss.",
     "CleanupInterval": "00:06:00:00",
-
     "_InvalidationDuration": "Duration after which a DNS record is considered invalid. Format is dd:hh:mm:ss.",
     "InvalidationDuration": "00:01:00:00"
   },
-  
   "_DuplicateFlagging": "DuplicateFlagging configuration",
   "DuplicateFlagging": {
     "_ClaimExpirationLifespan": "How long a flow exporter has the claim to be the origin for traces.",
     "ClaimExpirationLifespan": "00:10:00",
-    
     "_CleanupInterval": "Interval between cleaning up unused claims.",
     "CleanupInterval": "01:00:00"
   },
-  
   "_TagsRequest": "Configuration for the option to request tag from a VMware machine.",
   "TagsRequest": {
     "_Enabled": "Enable or disable the tag service.",
     "Enabled": false,
-    
     "_VmWareRequest": "Configuration for the requesting of the tags.",
     "VmWareRequest": {
       "_VmWareTargetAddress": "The address of the target.",
       "VmWareTargetAddress": "",
-      
-      "_MaxRequestPerSecond" : "Maximum request per second",
-      "MaxRequestPerSecond" : "10",
-      
+      "_MaxRequestPerSecond": "Maximum request per second",
+      "MaxRequestPerSecond": "10",
       "_VmWareCredentials": "Credentials for the VMWare API.",
       "VmWareCredentials": {
         "_Username": "The username of the vmware client for the API.",
         "Username": "",
         "_Password": "The password of the vmware client for the API.",
         "Password": ""
       },
-      
       "_VmWareApiPaths": "Paths for getting certain information from the Vmware API.",
       "VmWareApiPaths": {
         "_SessionPath": "Path for getting a session token.",
@@ -104,18 +98,16 @@
       }
     }
   },
-  
-  "_TagsCache" : "Configurations for the tag cache service.",
-  "TagsCache" : {
-    "_RefreshPeriod" : "The time interval to send request to the target ip to get the tags. Format is dd:hh:mm:ss",
-    "RefreshPeriod" : "12:00:00"
+  "_TagsCache": "Configurations for the tag cache service.",
+  "TagsCache": {
+    "_RefreshPeriod": "The time interval to send request to the target ip to get the tags. Format is dd:hh:mm:ss",
+    "RefreshPeriod": "12:00:00"
   },
-  
   "_FlowImportMetrics": "Configuration for the flowImport",
   "FlowImportMetrics": {
     "_TraceSummationPeriod": "Defines how often the flow should be saved into a combined flow",
-    "TraceSummationPeriod": "00:01:00",  
-    "_FlowSavePeriod" : "Defines for what period the flowImports should be saved",
+    "TraceSummationPeriod": "00:01:00",
+    "_FlowSavePeriod": "Defines for what period the flowImports should be saved",
     "FlowSavePeriod": "12:00:00"
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -13,33 +13,33 @@ namespace Fennec.Parsers;`
`13`	`13`	`public class NetFlow9Parser : IParser`
`14`	`14`	`{`
`15`	`15`	`private readonly ILogger _log;`
`16`		`- private readonly IDictionary<(IPAddress, ushort), TemplateRecord> _templateRecords; // matches (ExporterIp, TemplateId) to TemplateRecord`
`17`	`16`	`private readonly IMetricService _metricService;`
`18`		`-`
`19`		`- public NetFlow9Parser(ILogger log, IMetricService metricService)`
	`17`	`+ private readonly INetFlow9CleanupService _templateCleanupService;`
	`18`	`+`
	`19`	`+ public NetFlow9Parser(ILogger log, IMetricService metricService, INetFlow9CleanupService templateCleanupService)`
`20`	`20`	`{`
`21`	`21`	`_log = log.ForContext<NetFlow9Parser>();`
`22`		`- _templateRecords = new Dictionary<(IPAddress, ushort), TemplateRecord>();`
`23`	`22`	`_metricService = metricService;`
	`23`	`+ _templateCleanupService = templateCleanupService;`
`24`	`24`	`}`
`25`	`25`	`public IEnumerable<TraceImportInfo> Parse(UdpReceiveResult result)`
`26`	`26`	`{`
`27`	`27`	`var stream = new MemoryStream(result.Buffer);`
`28`		`- using var nr = new NetflowReader(stream, 0, _templateRecords.Values);`
	`28`	`+ using var nr = new NetflowReader(stream, 0, _templateCleanupService.TemplateRecords.Values);`
`29`	`29`	`var header = nr.ReadPacketHeader();`
`30`	`30`
`31`	`31`	`for (var i = 0; i < header.Count; i++)`
`32`	`32`	`{`
`33`	`33`	`try`
`34`	`34`	`{`
`35`		`- var dict = _templateRecords.Values.ToDictionary(t => t.ID, t => t);`
	`35`	`+ var dict = _templateCleanupService.TemplateRecords.Values.ToDictionary(t => t.ID, t => t);`
`36`	`36`	`var set = nr.ReadFlowSet(dict);`
`37`	`37`
`38`	`38`	`switch (set)`
`39`	`39`	`{`
`40`	`40`	`case DataFlowSet dataFlowSet:`
`41`	`41`	`var key = (result.RemoteEndPoint.Address, set.ID);`
`42`		`- if (!_templateRecords.TryGetValue(key, out var template))`
	`42`	`+ if (!_templateCleanupService.TemplateRecords.TryGetValue(key, out var template))`
`43`	`43`	`{`
`44`	`44`	`_log.Warning("Could not parse data set... " +`
`45`	`45`	`"Reading this set requires a not yet transmitted " +`
`@@ -52,7 +52,9 @@ public IEnumerable<TraceImportInfo> Parse(UdpReceiveResult result)`
`52`	`52`	`case TemplateFlowSet templateFlowSet:`
`53`	`53`	`foreach (var templateRecord in templateFlowSet.Records)`
`54`	`54`	`{`
`55`		`- _templateRecords.Add((result.RemoteEndPoint.Address, templateRecord.ID), templateRecord);`
	`55`	`+ if (_templateCleanupService.TemplateRecords.ContainsKey((result.RemoteEndPoint.Address, templateRecord.ID)))`
	`56`	`+ continue;`
	`57`	`+ _templateCleanupService.TemplateRecords.Add((result.RemoteEndPoint.Address, templateRecord.ID), templateRecord);`
`56`	`58`	`_log.Information("Received new template set with id #{TemplateSetId}", templateRecord.ID);`
`57`	`59`	`}`
`58`	`60`