From 8abf83dafc3467675f284cd8dfddd43c6317a4a2 Mon Sep 17 00:00:00 2001 From: Laetitia El Badry <95191364+necoder27@users.noreply.github.com> Date: Sat, 9 Mar 2024 18:06:13 +0100 Subject: [PATCH] feat(astrapia): add netflow info onto home page - Application Status Cards - Flow Import Metric Cards - move Flow Import Graph to Home Page - add axis labels to Flow Import Graph - use window.innerWidth/Height for Graph Dimensions - remove Network Analysis Page - extend Metric Services --------- Co-authored-by: Denes TAKACS --- Astrapia/components/FlowImportGraph.vue | 58 ++++-- .../FlowImportGraphTimeframeSelector.vue | 51 ++++++ .../components/FlowImporterGraphContainer.vue | 22 +++ Astrapia/components/FlowMetricCard.vue | 69 +++++++ .../components/FlowMetricCardsContainer.vue | 32 ++++ Astrapia/components/SelfTestCard.vue | 115 ++++++++++++ Astrapia/components/SelfTestContainer.vue | 38 ++++ Astrapia/components/Sidebar.vue | 27 --- Astrapia/pages/index.vue | 34 ++-- Astrapia/pages/network-analysis.vue | 14 -- Astrapia/pages/settings.vue | 14 -- Astrapia/services/metricService.ts | 92 ++++++++++ Astrapia/services/networkAnalysisService.ts | 15 -- .../Fennec/Controllers/MetricController.cs | 18 +- Packrat/Fennec/Metrics/ApplicationStatus.cs | 173 ++++++++++++++++++ Packrat/Fennec/Services/DnsResolverService.cs | 15 +- Packrat/Fennec/Services/TagsCacheService.cs | 21 ++- Packrat/Fennec/Services/TimeService.cs | 5 +- Packrat/Fennec/Startup.cs | 3 + 19 files changed, 707 insertions(+), 109 deletions(-) create mode 100644 Astrapia/components/FlowImportGraphTimeframeSelector.vue create mode 100644 Astrapia/components/FlowImporterGraphContainer.vue create mode 100644 Astrapia/components/FlowMetricCard.vue create mode 100644 Astrapia/components/FlowMetricCardsContainer.vue create mode 100644 Astrapia/components/SelfTestCard.vue create mode 100644 Astrapia/components/SelfTestContainer.vue delete mode 100644 Astrapia/pages/network-analysis.vue delete mode 100644 Astrapia/pages/settings.vue create mode 100644 Astrapia/services/metricService.ts delete mode 100644 Astrapia/services/networkAnalysisService.ts create mode 100644 Packrat/Fennec/Metrics/ApplicationStatus.cs diff --git a/Astrapia/components/FlowImportGraph.vue b/Astrapia/components/FlowImportGraph.vue index 6c39c59..0e6c225 100644 --- a/Astrapia/components/FlowImportGraph.vue +++ b/Astrapia/components/FlowImportGraph.vue @@ -5,22 +5,33 @@ + + diff --git a/Astrapia/components/FlowImporterGraphContainer.vue b/Astrapia/components/FlowImporterGraphContainer.vue new file mode 100644 index 0000000..501634b --- /dev/null +++ b/Astrapia/components/FlowImporterGraphContainer.vue @@ -0,0 +1,22 @@ + + + + + diff --git a/Astrapia/components/FlowMetricCard.vue b/Astrapia/components/FlowMetricCard.vue new file mode 100644 index 0000000..72c59b1 --- /dev/null +++ b/Astrapia/components/FlowMetricCard.vue @@ -0,0 +1,69 @@ + + + + + diff --git a/Astrapia/components/FlowMetricCardsContainer.vue b/Astrapia/components/FlowMetricCardsContainer.vue new file mode 100644 index 0000000..42e7929 --- /dev/null +++ b/Astrapia/components/FlowMetricCardsContainer.vue @@ -0,0 +1,32 @@ + + + + + diff --git a/Astrapia/components/SelfTestCard.vue b/Astrapia/components/SelfTestCard.vue new file mode 100644 index 0000000..00dd179 --- /dev/null +++ b/Astrapia/components/SelfTestCard.vue @@ -0,0 +1,115 @@ + + + + + diff --git a/Astrapia/components/SelfTestContainer.vue b/Astrapia/components/SelfTestContainer.vue new file mode 100644 index 0000000..ac15c7d --- /dev/null +++ b/Astrapia/components/SelfTestContainer.vue @@ -0,0 +1,38 @@ + + + + + diff --git a/Astrapia/components/Sidebar.vue b/Astrapia/components/Sidebar.vue index 6e7c93b..f53bc4d 100644 --- a/Astrapia/components/Sidebar.vue +++ b/Astrapia/components/Sidebar.vue @@ -13,16 +13,6 @@ - -
-
- -
-
-

Network Analysis

-
-
-
@@ -43,16 +33,6 @@
- -
-
- -
-
-

Settings

-
-
- @@ -99,13 +79,6 @@ a { width: 3.5vw; } -#settings-icon { - padding-top: 1.5vh; - margin-top: auto; - margin-bottom: 1.5vh; - cursor: pointer; -} - .faicon { cursor: pointer; color: white; diff --git a/Astrapia/pages/index.vue b/Astrapia/pages/index.vue index 573d82e..e939c58 100644 --- a/Astrapia/pages/index.vue +++ b/Astrapia/pages/index.vue @@ -9,17 +9,19 @@ -
- - - -
+ +
+ +
+ - - diff --git a/Astrapia/pages/settings.vue b/Astrapia/pages/settings.vue deleted file mode 100644 index 8ba8dd9..0000000 --- a/Astrapia/pages/settings.vue +++ /dev/null @@ -1,14 +0,0 @@ - - - - - diff --git a/Astrapia/services/metricService.ts b/Astrapia/services/metricService.ts new file mode 100644 index 0000000..a0b3dea --- /dev/null +++ b/Astrapia/services/metricService.ts @@ -0,0 +1,92 @@ +import ApiService from "~/services/restService"; + +export interface FlowImport { + dateTime: string; + endpoints: { [key: string]: number }; +} + +export interface GeneralFlowImporterData { + receivedPacketCount: number, + receivedByteCount: number, + transmittedPacketCount: number, + transmittedByteCount: number, + successfullyParsedPacket: number, + failedParsedPacket: number +} + +export interface GeneralFlowImporterDataDictionary { + [key: string]: GeneralFlowImporterData; +} + +export interface SelfTestData { + database: { + reachable: boolean; + latency: string; + totalSingleTraceCount: number; + totalDatabaseSize: string; + }; + counting: { + totalEntriesSinceUptime: number; + totalCounts: { + "Netflow5": number; + "Netflow9": number; + "IPFIX": number; + "sFlow": number; + }; + }; + "multiplexers": { + [port: string]: { + enabled: boolean; + name: string; + port: string; + acceptedProtocols: string; + }; + }; + config: { + "VMWare Tagging": { + enabled: boolean; + targetServer: string; + cachedTagsCount: number; + refreshPeriod: string; + timeUntilNextRefresh: string; + }; + "DNS Server": { + enabled: boolean; + cachedEntriesCount: number; + refreshPeriod: string; + timeUntilNextRefresh: string; + }; + "Duplicate Flagging": { + claimLifetime: string; + refreshPeriod: string; + }; + }; + statistics: { + startTime: string; + upTime: string; + }; + +} + +class MetricService { + public async getFlowImport(from?: string, to?: string) { + let url = '/api/metrics/flowsSeries'; + if(from && to) { + url += `?from=${from}&to=${to}`; + } + return await ApiService.get(url) + .then(x => x.data); + } + + public async getGeneralFlowImporterData() { + return await ApiService.get('/api/metrics/flowAggregated') + .then(x => x.data); + } + + public async getApplicationStatusData() { + return await ApiService.get('/api/metrics/ApplicationStatus') + .then(x => x.data); + } +} + +export default new MetricService(); diff --git a/Astrapia/services/networkAnalysisService.ts b/Astrapia/services/networkAnalysisService.ts deleted file mode 100644 index ce0297a..0000000 --- a/Astrapia/services/networkAnalysisService.ts +++ /dev/null @@ -1,15 +0,0 @@ -import ApiService from "~/services/restService"; - -export interface FlowImport { - dateTime: string; - endpoints: { [key: string]: number }; -} - -class NetworkAnalysisService { - public async getFlowImport() { - return await ApiService.get('/api/metrics/flowImporter') - .then(x => x.data); - } -} - -export default new NetworkAnalysisService(); diff --git a/Packrat/Fennec/Controllers/MetricController.cs b/Packrat/Fennec/Controllers/MetricController.cs index c6105c2..142c819 100644 --- a/Packrat/Fennec/Controllers/MetricController.cs +++ b/Packrat/Fennec/Controllers/MetricController.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using MongoDB.Driver; namespace Fennec.Controllers; @@ -18,12 +19,14 @@ public class MetricController : ControllerBase private readonly IMetricService _metricService; private readonly IFlowImporterMetric _metricFlowImporter; private readonly TimeSpan _flowMetricSavePeriod; + private readonly IApplicationStatus _applicationStatus; - public MetricController(IMetricService metricService, IFlowImporterMetric flowImporterMetric, IOptions flowOptions) + public MetricController(IMetricService metricService, IFlowImporterMetric flowImporterMetric, IOptions flowOptions, IApplicationStatus applicationStatus) { _metricService = metricService; _metricFlowImporter = flowImporterMetric; _flowMetricSavePeriod = flowOptions.Value.FlowSavePeriod; + _applicationStatus = applicationStatus; } /// @@ -38,9 +41,9 @@ public async Task GetFlowSeries(DateTime? from = null, DateTime? var data = _metricService.GetMetrics("FlowSeriesData") .FlowImporterDataSeries; - - data.Where(fid => fid != null && fid.DateTime != null && fid.DateTime >= from && fid.DateTime <= to).ToArray(); - return Ok(data); + + return Ok(data.Where(fid => fid?.DateTime != null && fid.DateTime >= from && fid.DateTime <= to).ToArray() + ); } /// @@ -53,4 +56,11 @@ public async Task GetFlowGeneral() var data = _metricService.GetMetrics("FlowGeneralData").EndPointsData; return Ok(data); } + + [HttpGet("ApplicationStatus")] + public async Task GetApplicationStatus() + { + var data = _applicationStatus.GetLatestStatus(); + return Ok(data); + } } \ No newline at end of file diff --git a/Packrat/Fennec/Metrics/ApplicationStatus.cs b/Packrat/Fennec/Metrics/ApplicationStatus.cs new file mode 100644 index 0000000..94a5fe3 --- /dev/null +++ b/Packrat/Fennec/Metrics/ApplicationStatus.cs @@ -0,0 +1,173 @@ +using System.Diagnostics; +using System.Globalization; +using Fennec.Database.Domain; +using Fennec.Options; +using Fennec.Parsers; +using Fennec.Services; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace Fennec.Metrics; + +public interface IApplicationStatus +{ + Dictionary GetLatestStatus(); +} + +public class ApplicationStatus : IApplicationStatus +{ + private readonly IConfiguration _configuration; + private readonly IServiceProvider _serviceProvider; + + public ApplicationStatus(IConfiguration configuration, IServiceProvider serviceProvider) + { + _configuration = configuration; + _serviceProvider = serviceProvider; + } + + public Dictionary GetLatestStatus() + { + var status = new Dictionary(); + + SetRuntime(status); + SetDatabaseStatus(status); + SetFlowCounts(status); + SetMultiplexer(status); + SetConfig(status); + + return status; + } + + private void SetRuntime(IDictionary status) + { + status.Add("RunTime", new Dictionary + { + {"Start Time", _serviceProvider.GetService()?.StartTime.ToString(CultureInfo.InvariantCulture)}, + {"Uptime", (DateTime.UtcNow - _serviceProvider.GetService()!.StartTime).ToString()}, + }); + } + + private void SetDatabaseStatus(IDictionary status) + { + bool reachable; + string? latency = null; + string? totalSingleTraceCount = null; + string? totalDatabaseSize = null; + + try + { + var stopwatch = Stopwatch.StartNew(); + _serviceProvider.GetService()?.ListDatabaseNames(); + stopwatch.Stop(); + + reachable = true; + latency = stopwatch.ElapsedMilliseconds + "ms"; + totalSingleTraceCount = _serviceProvider.GetService()!.GetCollection("singleTraces").CountDocuments(trace => true).ToString(); + totalDatabaseSize = + _serviceProvider.GetService()!.RunCommand(new BsonDocument { { "dbStats", 1 } })["dataSize"].ToInt64() / + (1024 * 1024) + "MB"; + } + catch (MongoConnectionException) + { + reachable = false; + } + + status.Add("Database", new Dictionary + { + {"Reachable", reachable}, + {"Latency", latency}, + {"Single Trace Count", totalSingleTraceCount}, + {"Total Database Size", totalDatabaseSize} + }); + } + + private void SetFlowCounts(IDictionary status) + { + var totalEntriesSinceUptime = _serviceProvider.GetService()!.GetCollection("singleTraces").CountDocuments(trace => trace.Timestamp >= _serviceProvider.GetService()!.StartTime).ToString(); + + var totalCounts = new Dictionary(); + + foreach (var protocol in Enum.GetValues(typeof(FlowProtocol))) + { + totalCounts.Add(protocol.ToString()!, (_serviceProvider.GetService()!.GetCollection("singleTraces").CountDocuments(trace => trace.FlowProtocol == (FlowProtocol)protocol)).ToString()); + } + + status.Add("Counting", new Dictionary() + { + {"Total entries since up time", totalEntriesSinceUptime}, + {"Total counts", totalCounts} + }); + } + + private void SetMultiplexer(IDictionary status) + { + var multiplexerOptions = _configuration.GetSection("Multiplexers").Get>(); + var multiplexers = new Dictionary(); + + if (multiplexerOptions != null) + foreach (var multiplexer in multiplexerOptions) + { + if (multiplexer.Name != null) + { + var multiplexersInfo = new Dictionary + { + { "Enabled", multiplexer.Enabled }, + { "Name", multiplexer.Name }, + { "Port", multiplexer.ListeningPort.ToString() }, + { "Accepted protocols", string.Join(", ", multiplexer.Parsers) } + }; + multiplexers.Add(multiplexer.ListeningPort.ToString(), multiplexersInfo); + } + } + + status.Add("Multiplexers", multiplexers); + } + + private void SetConfig(IDictionary status) + { + var vmware = new Dictionary(); + var dns = new Dictionary(); + var duplicateFlag = new Dictionary(); + + var tagsRequestOptions = _serviceProvider.GetService>()?.Value; + var tagsCacheOptions = _serviceProvider.GetService>()?.Value; + var duplicateOptions = _serviceProvider.GetService>()?.Value; + var dnsCacheOptions = _serviceProvider.GetService>()?.Value; + var dnsService = _serviceProvider.GetService(); + var tagsCacheService = _serviceProvider.GetService(); + var timeService = _serviceProvider.GetRequiredService(); + + vmware.Add("Enabled", tagsRequestOptions is { Enabled: true }); + if (tagsRequestOptions is { Enabled: true } && tagsCacheOptions is not null) + { + vmware.Add("Target Server", tagsRequestOptions!.VmWareRequest.VmWareTargetAddress); + vmware.Add("Cached Tags Count", tagsCacheService?.CountOfTags); + vmware.Add("Refresh Period", tagsCacheOptions!.RefreshPeriod.ToString()); + vmware.Add("Last Refresh", tagsCacheService!.LastRefresh.ToString()); + vmware.Add("Next Refresh", ((tagsCacheService.LastRefresh ?? timeService.StartTime) + tagsCacheOptions.RefreshPeriod).ToString()); + } + + dns.Add("Enabled", dnsCacheOptions is { Enabled: true }); + if (dnsCacheOptions is not null) + { + dns.Add("Cached Entries Count", dnsService.CachedEntriesCount.ToString()); + dns.Add("Cleanup Interval", dnsCacheOptions.CleanupInterval.ToString()); + dns.Add("Last Cleanup", dnsService.LastCleanup.ToString()); + dns.Add("Next Cleanup", (dnsService.LastCleanup + dnsCacheOptions.CleanupInterval).ToString()); + } + + if (duplicateOptions is not null) + { + duplicateFlag.Add("Claim Life Time", duplicateOptions.ClaimExpirationLifespan.ToString()); + duplicateFlag.Add("Refresh Period", duplicateOptions.CleanupInterval.ToString()); + } + + status.Add("Services", new Dictionary() + { + {"VMWare Tagging", vmware}, + {"DNS Server", dns}, + {"Duplicate Flagging", duplicateFlag} + }); + } +} \ No newline at end of file diff --git a/Packrat/Fennec/Services/DnsResolverService.cs b/Packrat/Fennec/Services/DnsResolverService.cs index 8efbcfd..316d587 100644 --- a/Packrat/Fennec/Services/DnsResolverService.cs +++ b/Packrat/Fennec/Services/DnsResolverService.cs @@ -16,11 +16,18 @@ public interface IDnsResolverService /// IP to get from _dnsCache or resolve /// Task GetDnsEntryFromCacheOrResolve(IPAddress ipAddress); - + /// /// Remove entries older than the from the cache. /// public void CleanupDnsCache(); + + /// + /// Returns the number of cached entries. + /// + public int CachedEntriesCount { get; } + + public DateTimeOffset LastCleanup { get; } } public class DnsResolverService : IDnsResolverService @@ -28,6 +35,9 @@ public class DnsResolverService : IDnsResolverService private readonly ILogger _log; private readonly DnsCacheOptions _options; private Dictionary _dnsCache = new(); + public DateTimeOffset LastCleanup { get; private set; } = DateTimeOffset.Now; + + public int CachedEntriesCount => _dnsCache.Count; public DnsResolverService(ILogger log, IOptions options) { @@ -86,6 +96,7 @@ public void CleanupDnsCache() _log.Information("Cleaned up DNS cache... Before {BeforeCount} entries, removed {Difference} entries, now at {AfterCount} entries", countBefore, countBefore - countAfter, countAfter); + LastCleanup = DateTimeOffset.Now; } } @@ -113,7 +124,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _log.Information("Next DNS cache cleanup in {CleanupInterval}", _cleanupInterval); await Task.Delay(_cleanupInterval, stoppingToken); _log.Information("Cleaning up DNS cache"); - _dnsResolverService.CleanupDnsCache(); + _dnsResolverService.CleanupDnsCache(); } } } \ No newline at end of file diff --git a/Packrat/Fennec/Services/TagsCacheService.cs b/Packrat/Fennec/Services/TagsCacheService.cs index db12f53..455d306 100644 --- a/Packrat/Fennec/Services/TagsCacheService.cs +++ b/Packrat/Fennec/Services/TagsCacheService.cs @@ -1,5 +1,6 @@ using System.Net; using Fennec.Options; +using Fennec.Services; using Microsoft.Extensions.Options; namespace Fennec.Services; @@ -17,7 +18,11 @@ public interface ITagsCacheService /// public List? GetTags(IPAddress ipAddress); - public Task RequestLatestTagAndIps(); + public Task RefreshTags(); + + public DateTimeOffset? LastRefresh { get; } + + public int CountOfTags { get; } } public class TagsCacheService : ITagsCacheService @@ -25,6 +30,8 @@ public class TagsCacheService : ITagsCacheService private readonly ILogger _log; private readonly ITagsRequestService _tagsRequestService; private Dictionary> _ipTagsDict; + public int CountOfTags => _ipTagsDict.Count; + public DateTimeOffset? LastRefresh { get; private set; } public TagsCacheService(ILogger log, ITagsRequestService tagsRequestService) { @@ -36,18 +43,22 @@ public TagsCacheService(ILogger log, ITagsRequestService tagsRequestService) public List? GetTags(IPAddress ipAddress) => _ipTagsDict.TryGetValue(ipAddress, out var tags) ? tags : null; - public async Task RequestLatestTagAndIps() + public async Task RefreshTags() { _ipTagsDict = await _tagsRequestService.GetLatestTagsAndIps(); + LastRefresh = DateTimeOffset.Now; _log.Information("Updated VMware tags cache... Cache now has {CacheSize} IP entries", _ipTagsDict.Count); } } public class MockTagsCacheService : ITagsCacheService { + public int CountOfTags => 2; + public List GetTags(IPAddress ipAddress) => new() { "mock-tag-1/switch", "mock-tag-2/infra" }; - public Task RequestLatestTagAndIps() => Task.CompletedTask; + public Task RefreshTags() => Task.CompletedTask; + public DateTimeOffset? LastRefresh => DateTimeOffset.Now; } public class TagsCacheRefresherService : BackgroundService @@ -55,7 +66,7 @@ public class TagsCacheRefresherService : BackgroundService private readonly ILogger _log; private readonly TagsCacheOptions _options; private readonly ITagsCacheService _tagsCacheService; - + public TagsCacheRefresherService(ITagsCacheService tagsCacheService, ILogger log, IOptions options) { @@ -69,7 +80,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) while (!stoppingToken.IsCancellationRequested) { _log.Information("Updating VMware tags cache"); - await _tagsCacheService.RequestLatestTagAndIps(); + await _tagsCacheService.RefreshTags(); await Task.Delay(_options.RefreshPeriod, stoppingToken); } } diff --git a/Packrat/Fennec/Services/TimeService.cs b/Packrat/Fennec/Services/TimeService.cs index 2067492..f88cc01 100644 --- a/Packrat/Fennec/Services/TimeService.cs +++ b/Packrat/Fennec/Services/TimeService.cs @@ -6,9 +6,12 @@ public interface ITimeService /// The current time. /// public DateTime Now { get; } + + public DateTime StartTime { get; } } public class TimeService : ITimeService { public DateTime Now => DateTime.Now; -} \ No newline at end of file + + public DateTime StartTime { get; } = DateTime.UtcNow;} \ No newline at end of file diff --git a/Packrat/Fennec/Startup.cs b/Packrat/Fennec/Startup.cs index ad61db2..6ff8442 100644 --- a/Packrat/Fennec/Startup.cs +++ b/Packrat/Fennec/Startup.cs @@ -67,6 +67,9 @@ public void ConfigureServices(IServiceCollection services, IWebHostEnvironment e services.AddSingleton(); services.AddHostedService(); + // Application Status + services.AddSingleton(); + // Metric service services.AddSingleton();