Merge pull request #8 from jameson2011/dev

Dev

Author: jameson2011

src/zKbProxy/Configuration.fs

@@ -6,6 +6,8 @@
         [<Literal>]
         let KillSourceUri = "https://redisq.zkillboard.com/listen.php?ttw=10"
         [<Literal>]
+        let ZkbApiBaseUri = "https://zkillboard.com/api/"
+        [<Literal>]
         let MongoServer = "127.0.0.1"
         [<Literal>]
         let DbName = "zkbproxy"
@@ -27,31 +29,33 @@
 
     type Configuration=
         {
-            KillSourceUri: string;
-            NoCache: bool;
-            MongoServer: string;
-            DbName: string;
-            KillsDbCollection: string;
-            SessionsDbCollection: string;
-            MongoUserName: string;
-            MongoPassword: string;
-            WebServerPort: uint16;
-            BufferSize: int32;
-            SessionTimeout: TimeSpan;
+            KillSourceUri:          string;
+            ZkbApiBaseUri:          string;
+            NoCache:                bool;
+            MongoServer:            string;
+            DbName:                 string;
+            KillsDbCollection:      string;
+            SessionsDbCollection:   string;
+            MongoUserName:          string;
+            MongoPassword:          string;
+            WebServerPort:          uint16;
+            BufferSize:             int32;
+            SessionTimeout:         TimeSpan;
         }
         with             
             static member empty = 
                 {   
-                    KillSourceUri = ConfigurationDefaults.KillSourceUri;
-                    NoCache = false;
-                    MongoServer = ConfigurationDefaults.MongoServer; 
-                    DbName = ConfigurationDefaults.DbName; 
-                    KillsDbCollection = ConfigurationDefaults.KillsColName; 
-                    SessionsDbCollection = ConfigurationDefaults.SessionsColName;
-                    MongoUserName = ConfigurationDefaults.UserName; 
-                    MongoPassword = ConfigurationDefaults.UserPassword;
-                    WebServerPort = ConfigurationDefaults.WebServerPort; 
-                    BufferSize = ConfigurationDefaults.BufferSize;
-                    SessionTimeout = ConfigurationDefaults.SessionTimeout
+                    KillSourceUri =         ConfigurationDefaults.KillSourceUri;
+                    ZkbApiBaseUri =         ConfigurationDefaults.ZkbApiBaseUri;
+                    NoCache =               false;
+                    MongoServer =           ConfigurationDefaults.MongoServer; 
+                    DbName =                ConfigurationDefaults.DbName; 
+                    KillsDbCollection =     ConfigurationDefaults.KillsColName; 
+                    SessionsDbCollection =  ConfigurationDefaults.SessionsColName;
+                    MongoUserName =         ConfigurationDefaults.UserName; 
+                    MongoPassword =         ConfigurationDefaults.UserPassword;
+                    WebServerPort =         ConfigurationDefaults.WebServerPort; 
+                    BufferSize =            ConfigurationDefaults.BufferSize;
+                    SessionTimeout =        ConfigurationDefaults.SessionTimeout
                 }
 

src/zKbProxy/Program.fs

@@ -22,6 +22,7 @@ module Program=
         let defaultConfig = Configuration.empty
 
         { Configuration.KillSourceUri = j.["killSourceUri"] |> toString |> Option.defaultValue defaultConfig.KillSourceUri;
+            ZkbApiBaseUri =             j.["zkbApiBaseUri"] |> toString |> Option.defaultValue defaultConfig.ZkbApiBaseUri;
             NoCache =                   j.["noCache"] |> toBool |> Option.defaultValue defaultConfig.NoCache;
             MongoServer =               j.["mongoServer"] |> toString |> Option.defaultValue defaultConfig.MongoServer;
             DbName =                    j.["dbName"] |> toString |> Option.defaultValue defaultConfig.DbName;

src/zKbProxy/ServiceFactory.fs

@@ -11,6 +11,7 @@
                 sprintf "Kills collection:    %s" config.KillsDbCollection;
                 sprintf "Sessions collection: %s" config.SessionsDbCollection;
                 sprintf "Kill source:         %s" config.KillSourceUri;
+                sprintf "Zkb Api base:        %s" config.ZkbApiBaseUri;
                 sprintf "No Caching:          %b" config.NoCache;
                 sprintf "buffer size:         %i" config.BufferSize;
                 sprintf "Session timeout:     %f" config.SessionTimeout.TotalMinutes;
@@ -26,7 +27,7 @@
         do reportConfig postLog config |> ignore
 
         let importActor = KillImportActor(postLog, config) :> IActor
-        let zkbApiPassthrough = ZkbApiPassthroughActor(postLog)
+        let zkbApiPassthrough = ZkbApiPassthroughActor(config, postLog)
 
         let statsActor = StatisticsActor(postLog) 
         let postStats = (statsActor :> IActor).Post

src/zKbProxy/WebServices.fs

@@ -191,10 +191,10 @@ module WebServices=
 
             return! match resp.Status with
                     | HttpStatus.OK -> Successful.OK resp.Message ctx
-                    | HttpStatus.Unauthorized -> RequestErrors.UNAUTHORIZED "" ctx
-                    | HttpStatus.Forbidden -> RequestErrors.FORBIDDEN "" ctx
-                    | HttpStatus.NotFound -> RequestErrors.NOT_FOUND "" ctx
-                    | HttpStatus.TooManyRequests -> RequestErrors.TOO_MANY_REQUESTS "" ctx
-                    | HttpStatus.Error -> RequestErrors.BAD_REQUEST "" ctx
+                    | HttpStatus.Unauthorized -> RequestErrors.UNAUTHORIZED resp.Message ctx
+                    | HttpStatus.Forbidden -> RequestErrors.FORBIDDEN resp.Message ctx
+                    | HttpStatus.NotFound -> RequestErrors.NOT_FOUND resp.Message ctx
+                    | HttpStatus.TooManyRequests -> RequestErrors.TOO_MANY_REQUESTS resp.Message ctx
+                    | HttpStatus.Error -> ServerErrors.INTERNAL_ERROR resp.Message ctx
 
         }

src/zKbProxy/ZkbApiPassthroughActor.fs

@@ -9,18 +9,17 @@ type private ZkbApiPassthroughActorState = {
 with 
     static member empty = { ZkbApiPassthroughActorState.lastZkbRequest = DateTime.MinValue }
 
-type ZkbApiPassthroughActor(log: PostMessage)=
+type ZkbApiPassthroughActor(config: Configuration, log: PostMessage)=
     let logSource = typeof<ZkbApiPassthroughActor>.Name
     let logException (ex: Exception) = (logSource,ex.Message) |> ActorMessage.Error |> log
     let logInfo (msg: string) = (logSource, msg) |> ActorMessage.Info |> log
-
+    
     let logGetRequest (path: string) =              sprintf "Sending GET %s" path |> logInfo
     let logGetResp (resp: HttpResponseMessage) =    sprintf "Received %A from %A" resp.StatusCode resp.RequestMessage.RequestUri |> logInfo
 
     let httpClient = Web.httpClient()
-
-    [<Literal>]
-    let zkbDomain = "https://zkillboard.com/api"
+    let zkbBaseUri =    if config.ZkbApiBaseUri.EndsWith("/") then config.ZkbApiBaseUri
+                        else config.ZkbApiBaseUri + "/"
 
     let getResponseString(resp: HttpResponseMessage)= resp.Content.ReadAsStringAsync() |> Async.AwaitTask |> Async.RunSynchronously
 
@@ -39,27 +38,45 @@ type ZkbApiPassthroughActor(log: PostMessage)=
                                 | System.Net.HttpStatusCode.NotFound -> { WebResponse.Status = HttpStatus.NotFound; Retry = None; Message = "" }
                                 | System.Net.HttpStatusCode.TooManyRequests -> { WebResponse.Status = HttpStatus.TooManyRequests; Retry = None; Message = "" }
                                 | System.Net.HttpStatusCode.Forbidden -> { WebResponse.Status = HttpStatus.Forbidden; Retry = None; Message = "" }
-                                | _ -> { WebResponse.Status = HttpStatus.Error; Retry = None; Message = "" }
+                                | _ -> { WebResponse.Status = HttpStatus.Error; Retry = None; Message = "Unknown error." }
 
                 return result
             with
             | ex -> logException ex     
-                    return { WebResponse.Status = HttpStatus.Error; Retry = None; Message = "" }
+                    return { WebResponse.Status = HttpStatus.Error; Retry = None; Message = ex.Message }
         }
 
     let getZkbApi (path: string) (lastZkbRequest: DateTime) =
-        async {
-            let diff = DateTime.UtcNow - lastZkbRequest
-            if diff < TimeSpan.FromSeconds(1.) then
-                diff |> sprintf "Last Zkb API request in %A: pausing." |> logInfo
-                do! Async.Sleep(1000)
-            
-            let domain =    if path.StartsWith("/") then    zkbDomain
-                            else                            zkbDomain + "/"
-            
-            return! sprintf "%s%s" domain path |> getAsync
+        let maxIterations = 10
+
+        let rec get (url: string) (lastZkbRequest: DateTime) iterations =
+            async {
+                let diff = DateTime.UtcNow - lastZkbRequest
+                if diff < TimeSpan.FromSeconds(1.) then
+                    diff |> sprintf "Last Zkb API request in %A: pausing." |> logInfo
+                    do! Async.Sleep(1000)
+                           
+                let! resp = getAsync url
+         
+                return!
+                    match resp.Status with
+                    | HttpStatus.TooManyRequests when iterations <= 0 -> 
+                                                    async { return { WebResponse.Status = HttpStatus.Error; Retry = None; 
+                                                                        Message = maxIterations |> sprintf "Retried after %i iterations, quitting." } }
+                    | HttpStatus.TooManyRequests -> 
+                                                    iterations |> sprintf "Retrying, %i iteration(s) left." |> logInfo
+                                                    get url DateTime.UtcNow (iterations - 1)
+                    | _ ->        async { return resp }
+                    
             }
 
+        let path =  if path.StartsWith("/") then    path.Substring(1)
+                    else                            path
+        let uri = sprintf "%s%s" zkbBaseUri path
+
+        get uri lastZkbRequest maxIterations
+
+
     let pipe = MessageInbox.Start(fun inbox ->
         let rec loop(state: ZkbApiPassthroughActorState) = async{           
             let! msg = inbox.Receive()