Skip to content

Commit 02cf133

Browse files
hahwulhahwul
authored
Merge pull request #491 from owasp-noir/issue-490
Optimize Concurrency with WaitGroup
2 parents e1d4329 + 5e2e19a commit 02cf133

File tree

22 files changed

+942
-596
lines changed

22 files changed

+942
-596
lines changed

‎.github/workflows/ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ jobs:
99
runs-on: ubuntu-latest
1010
strategy:
1111
matrix:
12-
crystal-version: [1.10.1, 1.11.2, 1.12.2, 1.13.3, 1.14.0]
12+
crystal-version: [1.14.1, 1.15.0]
1313
steps:
1414
- uses: actions/checkout@v4
1515
- uses: MeilCli/setup-crystal-action@v4

‎src/analyzer/analyzers/crystal/kemal.cr

Lines changed: 54 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -6,47 +6,67 @@ module Analyzer::Crystal
66
# Variables
77
is_public = true
88
public_folders = [] of String
9+
channel = Channel(String).new
910

1011
# Source Analysis
1112
begin
12-
Dir.glob("#{@base_path}/**/*") do |path|
13-
next if File.directory?(path)
14-
if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib")
15-
File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
16-
last_endpoint = Endpoint.new("", "")
17-
file.each_line.with_index do |line, index|
18-
endpoint = line_to_endpoint(line)
19-
if endpoint.method != ""
20-
details = Details.new(PathInfo.new(path, index + 1))
21-
endpoint.details = details
22-
result << endpoint
23-
last_endpoint = endpoint
24-
end
25-
26-
param = line_to_param(line)
27-
if param.name != ""
28-
if last_endpoint.method != ""
29-
last_endpoint.push_param(param)
30-
end
31-
end
32-
33-
if line.includes? "serve_static false" || "serve_static(false)"
34-
is_public = false
35-
end
36-
37-
if line.includes? "public_folder"
38-
begin
39-
splited = line.split("public_folder")
40-
public_folder = ""
13+
spawn do
14+
Dir.glob("#{@base_path}/**/*") do |file|
15+
channel.send(file)
16+
end
17+
channel.close
18+
end
4119

42-
if splited.size > 1
43-
public_folder = splited[1].gsub("(", "").gsub(")", "").gsub(" ", "").gsub("\"", "").gsub("'", "")
44-
if public_folder != ""
45-
public_folders << public_folder
20+
WaitGroup.wait do |wg|
21+
@options["concurrency"].to_s.to_i.times do
22+
wg.spawn do
23+
loop do
24+
begin
25+
path = channel.receive?
26+
break if path.nil?
27+
next if File.directory?(path)
28+
if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib")
29+
File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
30+
last_endpoint = Endpoint.new("", "")
31+
file.each_line.with_index do |line, index|
32+
endpoint = line_to_endpoint(line)
33+
if endpoint.method != ""
34+
details = Details.new(PathInfo.new(path, index + 1))
35+
endpoint.details = details
36+
result << endpoint
37+
last_endpoint = endpoint
38+
end
39+
40+
param = line_to_param(line)
41+
if param.name != ""
42+
if last_endpoint.method != ""
43+
last_endpoint.push_param(param)
44+
end
45+
end
46+
47+
if line.includes?("serve_static false") || line.includes?("serve_static(false)")
48+
is_public = false
49+
end
50+
51+
if line.includes?("public_folder")
52+
begin
53+
splited = line.split("public_folder")
54+
public_folder = ""
55+
56+
if splited.size > 1
57+
public_folder = splited[1].gsub("(", "").gsub(")", "").gsub(" ", "").gsub("\"", "").gsub("'", "")
58+
if public_folder != ""
59+
public_folders << public_folder
60+
end
61+
end
62+
rescue
63+
end
64+
end
4665
end
4766
end
48-
rescue
4967
end
68+
rescue e : File::NotFoundError
69+
logger.debug "File not found: #{path}"
5070
end
5171
end
5272
end

‎src/analyzer/analyzers/crystal/lucky.cr

Lines changed: 39 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,27 +15,48 @@ module Analyzer::Crystal
1515
logger.debug e
1616
end
1717

18+
channel = Channel(String).new
19+
20+
spawn do
21+
Dir.glob("#{@base_path}/**/*") do |file|
22+
channel.send(file)
23+
end
24+
channel.close
25+
end
26+
1827
# Source Analysis
1928
begin
20-
Dir.glob("#{@base_path}/**/*") do |path|
21-
next if File.directory?(path)
22-
if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib")
23-
File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
24-
last_endpoint = Endpoint.new("", "")
25-
file.each_line.with_index do |line, index|
26-
endpoint = line_to_endpoint(line)
27-
if endpoint.method != ""
28-
details = Details.new(PathInfo.new(path, index + 1))
29-
endpoint.details = details
30-
result << endpoint
31-
last_endpoint = endpoint
32-
end
33-
34-
param = line_to_param(line)
35-
if param.name != ""
36-
if last_endpoint.method != ""
37-
last_endpoint.push_param(param)
29+
WaitGroup.wait do |wg|
30+
@options["concurrency"].to_s.to_i.times do
31+
wg.spawn do
32+
loop do
33+
begin
34+
path = channel.receive?
35+
break if path.nil?
36+
next if File.directory?(path)
37+
if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib")
38+
File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
39+
last_endpoint = Endpoint.new("", "")
40+
file.each_line.with_index do |line, index|
41+
endpoint = line_to_endpoint(line)
42+
if endpoint.method != ""
43+
details = Details.new(PathInfo.new(path, index + 1))
44+
endpoint.details = details
45+
result << endpoint
46+
last_endpoint = endpoint
47+
end
48+
49+
param = line_to_param(line)
50+
if param.name != ""
51+
if last_endpoint.method != ""
52+
last_endpoint.push_param(param)
53+
end
54+
end
55+
end
56+
end
3857
end
58+
rescue e : File::NotFoundError
59+
logger.debug "File not found: #{path}"
3960
end
4061
end
4162
end

‎src/analyzer/analyzers/elixir/elixir_phoenix.cr

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,40 @@ module Analyzer::Elixir
44
class Phoenix < Analyzer
55
def analyze
66
# Source Analysis
7+
channel = Channel(String).new
8+
79
begin
8-
Dir.glob("#{@base_path}/**/*") do |path|
9-
next if File.directory?(path)
10-
if File.exists?(path) && File.extname(path) == ".ex"
11-
File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
12-
file.each_line.with_index do |line, index|
13-
endpoints = line_to_endpoint(line)
14-
endpoints.each do |endpoint|
15-
if endpoint.method != ""
16-
details = Details.new(PathInfo.new(path, index + 1))
17-
endpoint.details = details
18-
@result << endpoint
10+
spawn do
11+
Dir.glob("#{@base_path}/**/*") do |file|
12+
channel.send(file)
13+
end
14+
channel.close
15+
end
16+
17+
WaitGroup.wait do |wg|
18+
@options["concurrency"].to_s.to_i.times do
19+
wg.spawn do
20+
loop do
21+
begin
22+
path = channel.receive?
23+
break if path.nil?
24+
next if File.directory?(path)
25+
if File.exists?(path) && File.extname(path) == ".ex"
26+
File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
27+
file.each_line.with_index do |line, index|
28+
endpoints = line_to_endpoint(line)
29+
endpoints.each do |endpoint|
30+
if endpoint.method != ""
31+
details = Details.new(PathInfo.new(path, index + 1))
32+
endpoint.details = details
33+
@result << endpoint
34+
end
35+
end
36+
end
37+
end
1938
end
39+
rescue e : File::NotFoundError
40+
logger.debug "File not found: #{path}"
2041
end
2142
end
2243
end

0 commit comments

Comments
 (0)