From 7a3ebd1abf161826cba323a49ce5f4b6f3d39009 Mon Sep 17 00:00:00 2001 From: Roger Zander Date: Tue, 29 Aug 2017 23:15:01 +0200 Subject: [PATCH] RZCache InMemory caching Use InMemory caching for short therm caching operations --- .../RZCache/Controllers/ValuesController.cs | 9 +- RZ.Cache/RZCache/Program.cs | 4 +- RZ.Cache/RZCache/RZRestProxy.cs | 343 ++++++++++++------ RZ.Cache/RZCache/RZWCF.csproj | 37 ++ RZ.Cache/RZCache/Startup.cs | 11 +- 5 files changed, 285 insertions(+), 119 deletions(-) create mode 100644 RZ.Cache/RZCache/RZWCF.csproj diff --git a/RZ.Cache/RZCache/Controllers/ValuesController.cs b/RZ.Cache/RZCache/Controllers/ValuesController.cs index ed9bf5a..14c6eaf 100644 --- a/RZ.Cache/RZCache/Controllers/ValuesController.cs +++ b/RZ.Cache/RZCache/Controllers/ValuesController.cs @@ -10,6 +10,7 @@ using System.Reflection; using System.Net.Http; using System.Net.Http.Headers; +using Microsoft.Extensions.Caching.Memory; namespace RZWCF.Controllers { @@ -53,17 +54,22 @@ public string get() } + [Route("/rest")] public class RZController : Controller { private readonly IConfiguration _config; + private IMemoryCache _cache; - public RZController(IConfiguration config) + public RZController(IConfiguration config, IMemoryCache memoryCache) { this._config = config; + _cache = memoryCache; + RuckZuck_WCF.RZRestProxy.sURL = config.GetSection("ParentServer").Value ?? _config.GetSection("RuckZuck:ParentServer").Value; RuckZuck_WCF.RZRestProxy.CatalogTTL = int.Parse(_config.GetSection("CatalogTTL").Value ?? _config.GetSection("RuckZuck:CatalogTTL").Value); RuckZuck_WCF.RZRestProxy.localURL = config.GetSection("localURL").Value ?? _config.GetSection("RuckZuck:localURL").Value; + RuckZuck_WCF.RZRestProxy._cache = _cache; } [Route("AuthenticateUser")] @@ -217,7 +223,6 @@ public Stream Dl(string contentid, string filename) } } - public class GetSoftware { public string ProductName { get; set; } diff --git a/RZ.Cache/RZCache/Program.cs b/RZ.Cache/RZCache/Program.cs index e0d4e86..8904238 100644 --- a/RZ.Cache/RZCache/Program.cs +++ b/RZ.Cache/RZCache/Program.cs @@ -28,8 +28,8 @@ public static void Main(string[] args) var ClientEp = new IPEndPoint(IPAddress.Any, 0); var ClientRequestData = Server.Receive(ref ClientEp); var ClientRequest = Encoding.ASCII.GetString(ClientRequestData); - - Console.WriteLine("Recived '{0}' from {1}, sending response", ClientRequest, ClientEp.Address.ToString()); + + Console.WriteLine("Discovery request from {0}...", ClientRequest); Server.Send(ResponseData, ResponseData.Length, ClientEp); } }); diff --git a/RZ.Cache/RZCache/RZRestProxy.cs b/RZ.Cache/RZCache/RZRestProxy.cs index ab35340..6085408 100644 --- a/RZ.Cache/RZCache/RZRestProxy.cs +++ b/RZ.Cache/RZCache/RZRestProxy.cs @@ -14,11 +14,13 @@ using Newtonsoft.Json; using static RuckZuck_WCF.RZRestProxy; using System.Threading; +using Microsoft.Extensions.Caching.Memory; namespace RuckZuck_WCF { public static class RZRestProxy { + internal static IMemoryCache _cache; public static string sURL = "https://ruckzuck.azurewebsites.net/wcf/RZService.svc"; //internal static string sURL = "http://localhost:7727/RZService.svc"; @@ -31,20 +33,38 @@ public static string GetAuthToken(string Username, string Password) { try { - var oClient = new HttpClient(); - oClient.DefaultRequestHeaders.Add("Username", Username); - oClient.DefaultRequestHeaders.Add("Password", Password); - oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - var response = oClient.GetStringAsync(sURL + "/rest/AuthenticateUser"); - response.Wait(5000); - if (response.Result != null) + if (!_cache.TryGetValue("PW" + (Username + Password).GetHashCode(StringComparison.InvariantCultureIgnoreCase), out Token)) { - Token = response.Result.Replace("\"", ""); - return Token; + using (var oClient = new HttpClient()) + { + oClient.DefaultRequestHeaders.Add("Username", Username); + oClient.DefaultRequestHeaders.Add("Password", Password); + oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + var response = oClient.GetStringAsync(sURL + "/rest/AuthenticateUser"); + response.Wait(5000); + if (response.IsCompleted) + { + Token = response.Result.Replace("\"", ""); + + // Set cache options. + var cacheEntryOptions = new MemoryCacheEntryOptions() + // Keep in cache for this time, reset time if accessed. + .SetSlidingExpiration(TimeSpan.FromSeconds(300)); + + _cache.Set("PW" + (Username + Password).GetHashCode(StringComparison.InvariantCultureIgnoreCase), Token, cacheEntryOptions); + + return Token; + } + } } + else + return Token; + } + catch(Exception ex) + { + Console.WriteLine("Auth. Error: " + ex.Message); } - catch { } return ""; @@ -53,9 +73,15 @@ public static string GetAuthToken(string Username, string Password) public static string SWResults(string Searchstring) { string sCatFile = @"wwwroot/rzcat.xml"; - + string sResult = ""; if (contentType.ToLower() == "application/json") + { + if (_cache.TryGetValue("SWResult-" + Searchstring, out sResult)) + { + return sResult; + } sCatFile = @"wwwroot/rzcat.json"; + } try { @@ -63,26 +89,51 @@ public static string SWResults(string Searchstring) { if (File.Exists(sCatFile)) { - if (CatalogTTL == 0 || DateTime.Now.ToUniversalTime() - File.GetCreationTime(sCatFile).ToUniversalTime() <= new TimeSpan(1, 0, 0)) + if (CatalogTTL == 0 || DateTime.Now.ToUniversalTime() - File.GetCreationTime(sCatFile).ToUniversalTime() <= new TimeSpan(CatalogTTL, 0, 1)) { - return File.ReadAllText(sCatFile); + sResult = File.ReadAllText(sCatFile); + var cacheEntryOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromSeconds(60)); + _cache.Set("SWResult-" + Searchstring, sResult, cacheEntryOptions); + return sResult; } } } + else + { - var oClient = new HttpClient(); - oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); - oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); - var response = oClient.GetStringAsync(sURL + "/rest/SWResults?search=" + Searchstring); - response.Wait(5000); - if (response.Result != null) + } + + using (var handler = new HttpClientHandler()) { - string sResult = response.Result; - File.WriteAllText(sCatFile, sResult); - return sResult; + //handler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => { return true; }; //To prevent Issue with FW + + using (var oClient = new HttpClient(handler)) + { + oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); + oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); + var response = oClient.GetStringAsync(sURL + "/rest/SWResults?search=" + Searchstring); + response.Wait(7000); //7s max. + if (response.IsCompleted) + { + sResult = response.Result; + // Set cache options. + var cacheEntryOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromSeconds(60)); + _cache.Set("SWResult-" + Searchstring, sResult, cacheEntryOptions); + + if (string.IsNullOrEmpty(Searchstring)) + File.WriteAllText(sCatFile, sResult); + + return sResult; + } + } } } - catch { } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } //return old File if (File.Exists(sCatFile)) @@ -95,57 +146,91 @@ public static string SWResults(string Searchstring) public static string SWGet(string Shortname) { - try + string sResult = ""; + if (!_cache.TryGetValue("SWGET1-" + Shortname.GetHashCode(StringComparison.InvariantCultureIgnoreCase), out sResult)) { - var oClient = new HttpClient(); - oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); - oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); - var response = oClient.GetStringAsync(sURL + "/rest/SWGetShort?name=" + WebUtility.UrlEncode(Shortname)); - response.Wait(5000); - if (response.Result != null) + try { - return response.Result; + var oClient = new HttpClient(); + oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); + oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); + var response = oClient.GetStringAsync(sURL + "/rest/SWGetShort?name=" + WebUtility.UrlEncode(Shortname)); + response.Wait(5000); + if (response.IsCompleted) + { + sResult = response.Result; + + // Set cache options. + var cacheEntryOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromSeconds(330)); + + _cache.Set("SWGET1-" + Shortname.GetHashCode(StringComparison.InvariantCultureIgnoreCase), sResult, cacheEntryOptions); + + return sResult; + } } + catch { } } - catch { } - return ""; + return sResult; } public static string SWGet(string PackageName, string PackageVersion) { - try + string sResult = ""; + if (!_cache.TryGetValue("SWGET2-" + (PackageName + PackageVersion).GetHashCode(StringComparison.InvariantCultureIgnoreCase), out sResult)) { - var oClient = new HttpClient(); - oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); - oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); - var response = oClient.GetStringAsync(sURL + "/rest/SWGet?name=" + WebUtility.UrlEncode(PackageName) + "&ver=" + PackageVersion); - response.Wait(5000); - if (response.Result != null) + try { - return response.Result; + var oClient = new HttpClient(); + oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); + oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); + var response = oClient.GetStringAsync(sURL + "/rest/SWGet?name=" + WebUtility.UrlEncode(PackageName) + "&ver=" + PackageVersion); + response.Wait(5000); + if (response.IsCompleted) + { + sResult = response.Result; + + // Set cache options. + var cacheEntryOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromSeconds(330)); + sResult = response.Result; + _cache.Set("SWGET2-" + (PackageName + PackageVersion).GetHashCode(StringComparison.InvariantCultureIgnoreCase), sResult, cacheEntryOptions); + return sResult; + } } + catch { } } - catch { } return ""; } public static string SWGet(string PackageName, string Manufacturer, string PackageVersion) { - try + string sResult = ""; + if (!_cache.TryGetValue("SWGET3-" + (PackageName + Manufacturer + PackageVersion).GetHashCode(StringComparison.InvariantCultureIgnoreCase), out sResult)) { - var oClient = new HttpClient(); - oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); - oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); - var response = oClient.GetStringAsync(sURL + "/rest/SWGetPkg?name=" + WebUtility.UrlEncode(PackageName) + "&manuf=" + WebUtility.UrlEncode(Manufacturer) + "&ver=" + PackageVersion); - response.Wait(5000); - if (response.Result != null) + try { - return response.Result; + var oClient = new HttpClient(); + oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); + oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); + var response = oClient.GetStringAsync(sURL + "/rest/SWGetPkg?name=" + WebUtility.UrlEncode(PackageName) + "&manuf=" + WebUtility.UrlEncode(Manufacturer) + "&ver=" + PackageVersion); + response.Wait(5000); + if (response.IsCompleted) + { + sResult = response.Result; + + // Set cache options. + var cacheEntryOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromSeconds(330)); + sResult = response.Result; + _cache.Set("SWGET3-" + (PackageName + Manufacturer + PackageVersion).GetHashCode(StringComparison.InvariantCultureIgnoreCase), sResult, cacheEntryOptions); + return sResult; + } } + catch { } } - catch { } return ""; } @@ -186,56 +271,68 @@ public static string GetSWDefinitions(string productName, string productVersion, if (File.Exists(sSWFile)) { - if (CatalogTTL == 0 || DateTime.Now.ToUniversalTime() - File.GetCreationTime(sSWFile).ToUniversalTime() <= new TimeSpan(1, 0, 0)) + if (CatalogTTL == 0 || DateTime.Now.ToUniversalTime() - File.GetCreationTime(sSWFile).ToUniversalTime() <= new TimeSpan(CatalogTTL, 0, 1)) { return File.ReadAllText(sSWFile); } } - var oClient = new HttpClient(); - oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); - oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); - var response = oClient.GetStringAsync(sURL + "/rest/GetSWDefinition?name=" + WebUtility.UrlEncode(productName) + "&ver=" + WebUtility.UrlEncode(productVersion) + "&man=" + WebUtility.UrlEncode(manufacturer)); - response.Wait(5000); - if (response.Result != null) + using (var handler = new HttpClientHandler()) { - string sResult = response.Result; - try + handler.AllowAutoRedirect = true; + handler.MaxAutomaticRedirections = 5; + handler.CheckCertificateRevocationList = false; + + using (var oClient = new HttpClient(handler)) { - if (contentType.ToLower() == "application/json") + oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); + oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); + var response = oClient.GetStringAsync(sURL + "/rest/GetSWDefinition?name=" + WebUtility.UrlEncode(productName) + "&ver=" + WebUtility.UrlEncode(productVersion) + "&man=" + WebUtility.UrlEncode(manufacturer)); + response.Wait(5000); + if (response.IsCompleted) { - var oAddSW = Newtonsoft.Json.JsonConvert.DeserializeObject>(sResult); - foreach (var oSW in oAddSW) + string sResult = response.Result; + + Task.Run(() => { - foreach (var oDL in oSW.Files) + try { - string sDir = @"wwwroot/files/" + oSW.ContentID; - if (!Directory.Exists(sDir)) - { - Directory.CreateDirectory(sDir); - } - - if (oDL.URL.StartsWith("http") || oDL.URL.StartsWith("ftp")) + if (contentType.ToLower() == "application/json") { - if (!File.Exists(sDir + "/" + oDL.FileName)) + var oAddSW = Newtonsoft.Json.JsonConvert.DeserializeObject>(sResult); + foreach (var oSW in oAddSW) { - var oRes = _DownloadFile(oDL.URL, sDir + "/" + oDL.FileName).Status; - - //Thread.Sleep(5000); + foreach (var oDL in oSW.Files) + { + string sDir = @"wwwroot/files/" + oSW.ContentID; + if (!Directory.Exists(sDir)) + { + Directory.CreateDirectory(sDir); + } + + if (oDL.URL.StartsWith("http") || oDL.URL.StartsWith("ftp")) + { + if (!File.Exists(sDir + "/" + oDL.FileName)) + { + var oRes = _DownloadFile(oDL.URL, sDir + "/" + oDL.FileName).Status; + } + + oDL.URL = localURL + "/rest/dl/" + oSW.ContentID + "/" + oDL.FileName; + + } + } } - - oDL.URL = localURL + "/rest/dl/" + oSW.ContentID + "/" + oDL.FileName; - + sResult = Newtonsoft.Json.JsonConvert.SerializeObject(oAddSW); + File.WriteAllText(sSWFile, sResult); } } - } - sResult = Newtonsoft.Json.JsonConvert.SerializeObject(oAddSW); + catch { } + }); + + return response.Result; } } - catch { } - File.WriteAllText(sSWFile, sResult); - return sResult; } } catch { } @@ -339,7 +436,7 @@ public static Stream GetIcon(Int32 iconid) //oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("image/jpeg")); var response = oClient.GetStreamAsync(sURL + "/rest/GetIcon?id=" + iconid.ToString()); - //response.Wait(5000); + response.Wait(5000); if (response.Result != null) { var oRet = response.Result; @@ -386,28 +483,46 @@ public static async void TrackDownloads(string contentID) public static string CheckForUpdate(string lSoftware) { + string sResult = ""; + if (_cache.TryGetValue("CHK" + lSoftware.GetHashCode(StringComparison.InvariantCultureIgnoreCase), out sResult)) + { + return sResult; + } try { - var oClient = new HttpClient(); - oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); - oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); - HttpContent oCont = new StringContent(lSoftware, Encoding.UTF8, contentType); - if (contentType == "application/xml") + using (var oClient = new HttpClient()) { - var response = oClient.PostAsync(sURL + "/rest/CheckForUpdateXml", oCont); - response.Wait(5000); + oClient.DefaultRequestHeaders.Add("AuthenticatedToken", Token); + oClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(contentType)); + HttpContent oCont = new StringContent(lSoftware, Encoding.UTF8, contentType); + if (contentType == "application/xml") + { + var response = oClient.PostAsync(sURL + "/rest/CheckForUpdateXml", oCont); + response.Wait(5000); + if (response.IsCompleted) + { + string responseBody = response.Result.Content.ReadAsStringAsync().Result; + sResult = responseBody; + // Set cache options. + var cacheEntryOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromSeconds(60)); - string responseBody = response.Result.Content.ReadAsStringAsync().Result; - return responseBody; - } + _cache.Set("CHK" + lSoftware.GetHashCode(StringComparison.InvariantCultureIgnoreCase), sResult, cacheEntryOptions); - if (contentType == "application/json") - { - var response = oClient.PostAsync(sURL + "/rest/CheckForUpdateJSON", oCont); - response.Wait(5000); + return sResult; + } + } - string responseBody = response.Result.Content.ReadAsStringAsync().Result; - return responseBody; + if (contentType == "application/json") + { + var response = oClient.PostAsync(sURL + "/rest/CheckForUpdateJSON", oCont); + response.Wait(5000); + if (response.IsCompleted) + { + string responseBody = response.Result.Content.ReadAsStringAsync().Result; + return responseBody; + } + } } @@ -459,27 +574,31 @@ private static async Task _DownloadFile(string URL, string FileName) { try { - HttpClientHandler handler = new HttpClientHandler(); - handler.AllowAutoRedirect = true; - handler.MaxAutomaticRedirections = 5; - - using (var httpClient = new HttpClient(handler)) + using (HttpClientHandler handler = new HttpClientHandler()) { - httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "chocolatey command line"); - using (var request = new HttpRequestMessage(HttpMethod.Get, new Uri(URL))) + handler.AllowAutoRedirect = true; + handler.MaxAutomaticRedirections = 5; + handler.CheckCertificateRevocationList = false; + handler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => { return true; }; //To prevent Issue with FW + + using (var httpClient = new HttpClient(handler)) { - using (Stream contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync(), - stream = new FileStream(FileName, FileMode.Create, FileAccess.Write, FileShare.None, 32768, true)) + httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "chocolatey command line"); + using (var request = new HttpRequestMessage(HttpMethod.Get, new Uri(URL))) { - await contentStream.CopyToAsync(stream); + using (Stream contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync(), + stream = new FileStream(FileName, FileMode.Create, FileAccess.Write, FileShare.None, 32768, true)) + { + await contentStream.CopyToAsync(stream); + } + Console.WriteLine("Donwloaded: " + URL); } - } + } } } catch (Exception ex) { - System.Diagnostics.Debug.WriteLine(ex.Message); Console.WriteLine(ex.Message); return false; } diff --git a/RZ.Cache/RZCache/RZWCF.csproj b/RZ.Cache/RZCache/RZWCF.csproj new file mode 100644 index 0000000..6ee3aa8 --- /dev/null +++ b/RZ.Cache/RZCache/RZWCF.csproj @@ -0,0 +1,37 @@ + + + + netcoreapp2.0 + ..\docker-compose.dcproj + 1.0.0.2 + 1.0.0.2 + Roger Zander + Zander Tools + RZWCF Proxy Service + RuckZuck.tool Proxy and Caching Service + Copyright (c) 2017 by Roger Zander + https://github.com/rzander/ruckzuck + https://github.com/rzander/ruckzuck/blob/master/LICENSE.md + software package manager windows cache proxy + 3d943e5a-c0e9-40f0-b91e-9ab682223099 + + + + + + + + + + + + + + Always + + + + + + + diff --git a/RZ.Cache/RZCache/Startup.cs b/RZ.Cache/RZCache/Startup.cs index 4fdda76..2a1b268 100644 --- a/RZ.Cache/RZCache/Startup.cs +++ b/RZ.Cache/RZCache/Startup.cs @@ -20,6 +20,7 @@ public Startup(IHostingEnvironment env) .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true) .AddEnvironmentVariables(); + Configuration = builder.Build(); } @@ -30,14 +31,18 @@ public void ConfigureServices(IServiceCollection services) { // Adds services required for using options. services.AddOptions(); - + services.AddSingleton(Configuration); + services.AddMemoryCache(); + // Add framework services. services.AddMvc(options => options.OutputFormatters.RemoveType()).AddJsonOptions(options => { options.SerializerSettings.ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore; - }); ; + }); + + } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. @@ -45,7 +50,7 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF { loggerFactory.AddConsole(Configuration.GetSection("Logging")); loggerFactory.AddDebug(); - + app.UseMvc(); } }