Merge pull request #495 from owasp-noir/issue-494

Improve deliver performance

src/deliver/send_proxy.cr

@@ -1,49 +1,59 @@
 require "crest"
+require "wait_group"
 require "../utils/http_symbols"
 require "../models/deliver"
 
 class SendWithProxy < Deliver
   def run(endpoints : Array(Endpoint))
     proxy_url = URI.parse(@proxy)
     applied_endpoints = apply_all(endpoints)
+    wg = WaitGroup.new
+
     applied_endpoints.each do |endpoint|
-      begin
-        if endpoint.params.size > 0
-          endpoint_hash = endpoint.params_to_hash
-          body = {} of String => String
-          is_json = false
-          if endpoint_hash["json"].size > 0
-            is_json = true
-            body = endpoint_hash["json"]
+      wg.add(1)
+      spawn do
+        begin
+          if endpoint.params.size > 0
+            endpoint_hash = endpoint.params_to_hash
+            body = {} of String => String
+            is_json = false
+            if endpoint_hash["json"].size > 0
+              is_json = true
+              body = endpoint_hash["json"]
+            else
+              body = endpoint_hash["form"]
+            end
+
+            Crest::Request.execute(
+              method: get_symbol(endpoint.method),
+              url: endpoint.url,
+              p_addr: proxy_url.host,
+              p_port: proxy_url.port,
+              tls: OpenSSL::SSL::Context::Client.insecure,
+              user_agent: "Noir/#{Noir::VERSION}",
+              params: endpoint_hash["query"],
+              headers: @headers,
+              form: body,
+              json: is_json
+            )
           else
-            body = endpoint_hash["form"]
+            Crest::Request.execute(
+              method: get_symbol(endpoint.method),
+              url: endpoint.url,
+              p_addr: proxy_url.host,
+              p_port: proxy_url.port,
+              headers: @headers,
+              tls: OpenSSL::SSL::Context::Client.insecure,
+              user_agent: "Noir/#{Noir::VERSION}"
+            )
           end
-
-          Crest::Request.execute(
-            method: get_symbol(endpoint.method),
-            url: endpoint.url,
-            p_addr: proxy_url.host,
-            p_port: proxy_url.port,
-            tls: OpenSSL::SSL::Context::Client.insecure,
-            user_agent: "Noir/#{Noir::VERSION}",
-            params: endpoint_hash["query"],
-            headers: @headers,
-            form: body,
-            json: is_json
-          )
-        else
-          Crest::Request.execute(
-            method: get_symbol(endpoint.method),
-            url: endpoint.url,
-            p_addr: proxy_url.host,
-            p_port: proxy_url.port,
-            headers: @headers,
-            tls: OpenSSL::SSL::Context::Client.insecure,
-            user_agent: "Noir/#{Noir::VERSION}"
-          )
+        rescue
+        ensure
+          wg.done
         end
-      rescue
       end
     end
+
+    wg.wait
   end
 end

src/deliver/send_req.cr

@@ -1,44 +1,54 @@
 require "crest"
+require "wait_group"
 require "../utils/http_symbols"
 require "../models/deliver"
 
 class SendReq < Deliver
   def run(endpoints : Array(Endpoint))
     applied_endpoints = apply_all(endpoints)
+    wg = WaitGroup.new
+
     applied_endpoints.each do |endpoint|
-      begin
-        if endpoint.params.size > 0
-          endpoint_hash = endpoint.params_to_hash
-          body = {} of String => String
-          is_json = false
-          if endpoint_hash["json"].size > 0
-            is_json = true
-            body = endpoint_hash["json"]
+      wg.add(1)
+      spawn do
+        begin
+          if endpoint.params.size > 0
+            endpoint_hash = endpoint.params_to_hash
+            body = {} of String => String
+            is_json = false
+            if endpoint_hash["json"].size > 0
+              is_json = true
+              body = endpoint_hash["json"]
+            else
+              body = endpoint_hash["form"]
+            end
+
+            Crest::Request.execute(
+              method: get_symbol(endpoint.method),
+              url: endpoint.url,
+              tls: OpenSSL::SSL::Context::Client.insecure,
+              user_agent: "Noir/#{Noir::VERSION}",
+              params: endpoint_hash["query"],
+              form: body,
+              headers: @headers,
+              json: is_json
+            )
           else
-            body = endpoint_hash["form"]
+            Crest::Request.execute(
+              method: get_symbol(endpoint.method),
+              url: endpoint.url,
+              headers: @headers,
+              tls: OpenSSL::SSL::Context::Client.insecure,
+              user_agent: "Noir/#{Noir::VERSION}"
+            )
           end
-
-          Crest::Request.execute(
-            method: get_symbol(endpoint.method),
-            url: endpoint.url,
-            tls: OpenSSL::SSL::Context::Client.insecure,
-            user_agent: "Noir/#{Noir::VERSION}",
-            params: endpoint_hash["query"],
-            form: body,
-            headers: @headers,
-            json: is_json
-          )
-        else
-          Crest::Request.execute(
-            method: get_symbol(endpoint.method),
-            url: endpoint.url,
-            headers: @headers,
-            tls: OpenSSL::SSL::Context::Client.insecure,
-            user_agent: "Noir/#{Noir::VERSION}"
-          )
+        rescue
+        ensure
+          wg.done
         end
-      rescue
       end
     end
+
+    wg.wait
   end
 end

src/detector/detector.cr

@@ -117,4 +117,4 @@ def detect_techs(base_path : String, options : Hash(String, YAML::Any), passive_
 
   wg.wait
   {techs.uniq, passive_result}
-end
+end