Implement ruleRemoveByMsg and ruleRemoveByTag actions

Use a per-worker lookup table to append rule IDs to the _ignore_rules
waf object table for ctl:ruleRemoveBy* actions. This lookup table
is re-generated when a new key is added to the ruleset definitions
table. Both RemoveByTag and RemoveByMsg actions are merged into a
nondisruptive action 'rule_removed_by_meta', which consults the
pregenerated exceptions lookup table.

This commit addresses part of issue #257.

Author: p0pr0ck5

lib/resty/waf.lua

@@ -39,6 +39,30 @@ local _global_rulesets = {
 local _ruleset_defs = {}
 local _ruleset_def_cnt = 0
 
+-- lookup tables for msg and tag exceptions
+-- public so it can be accessed via metatable
+_M._meta_exception = {
+	msgs = {},
+	tags = {},
+	meta_ids = {},
+}
+
+-- this function runs when a new ruleset has been introduced to
+-- _ruleset_defs. it reads over all the rules, looking for msg and tag
+-- elements, and building a lookup table for exceptions
+local function _build_exception_table()
+	-- build a rule.id -> { rule.id, ... } lookup table based on the exception
+	-- action for the rule
+
+	for r, ruleset in pairs(_ruleset_defs) do
+		for phase, rules in pairs(ruleset) do
+			for i, rule in ipairs(rules) do
+				util.rule_exception(_M._meta_exception, rule)
+			end
+		end
+	end
+end
+
 -- get a subset or superset of request data collection
 local function _parse_collection(self, collection, var)
 	local parse = var.parse
@@ -317,7 +341,7 @@ end
 local function _calculate_offset(ruleset)
 	for phase, i in pairs(phase_t.phases) do
 		if (ruleset[phase]) then
-			calc.calculate(ruleset[phase])
+			calc.calculate(ruleset[phase], _M._meta_exception)
 		else
 			ruleset[phase] = {}
 		end
@@ -333,6 +357,8 @@ local function _merge_rulesets(self)
 		t[v] = true
 	end
 
+	local rebuild_exception_table = false
+
 	if (self) then
 		local added   = self._add_ruleset
 		local added_s = self._add_ruleset_string
@@ -356,6 +382,9 @@ local function _merge_rulesets(self)
 					_calculate_offset(rs)
 
 					_ruleset_defs[k] = rs
+					_ruleset_def_cnt = _ruleset_def_cnt + 1
+
+					rebuild_exception_table = true
 				end
 			end
 
@@ -368,6 +397,8 @@ local function _merge_rulesets(self)
 		end
 	end
 
+	if rebuild_exception_table then _build_exception_table() end
+
 	t = util.table_keys(t)
 
 	-- rulesets will be processed in numeric order
@@ -475,6 +506,9 @@ function _M.exec(self, opts)
 				_calculate_offset(rs)
 
 				_ruleset_defs[ruleset] = rs
+				_ruleset_def_cnt = _ruleset_def_cnt + 1
+
+				_build_exception_table()
 			end
 		end
 
@@ -627,8 +661,9 @@ function _M.init()
 			_calculate_offset(rs)
 
 			_ruleset_defs[ruleset] = rs
-
 			_ruleset_def_cnt = _ruleset_def_cnt + 1
+
+			_build_exception_table()
 		end
 	end
 end

lib/resty/waf/actions.lua

@@ -81,7 +81,18 @@ _M.nondisruptive_lookup = {
 		--_LOG_"Runtime ignoring rule " .. rule
 
 		waf._ignore_rule[rule] = true
-	end
+	end,
+	rule_remove_by_meta = function(waf, data, ctx)
+		--_LOG_"Runtime ignoring rules by meta"
+
+		-- this lookup table holds
+		local meta_rules = waf._meta_exception.meta_ids[ctx.id] or {}
+
+		for i, id in ipairs(meta_rules) do
+			--_LOG_"Runtime ignoring rule " .. id
+			waf._ignore_rule[id] = true
+		end
+	end,
 }
 
 return _M

lib/resty/waf/rule_calc.lua

@@ -81,7 +81,7 @@ local function _write_skip_offset(rule, max, cur_offset)
 	end
 end
 
-function _M.calculate(ruleset)
+function _M.calculate(ruleset, meta_lookup)
 	local max = #ruleset
 	local chain = {}
 
@@ -122,6 +122,29 @@ function _M.calculate(ruleset)
 
 			chain = {}
 		end
+
+		-- meta table lookups for exceptions
+		if (meta_lookup) then
+			if (rule.msg) then
+				local msg = rule.msg
+
+				if (not meta_lookup.msgs[msg]) then
+					meta_lookup.msgs[msg] = { rule.id }
+				else
+					table_insert(meta_lookup.msgs[msg], rule.id)
+				end
+			end
+
+			if (rule.tag) then
+				for _, tag in ipairs(rule.tag) do
+					if (not meta_lookup.tags[tag]) then
+						meta_lookup.tags[tag] = { rule.id }
+					else
+						table_insert(meta_lookup.tags[tag], rule.id)
+					end
+				end
+			end
+		end
 	end
 end
 

lib/resty/waf/util.lua

@@ -5,7 +5,7 @@ _M.version = "0.9"
 local cjson  = require "cjson"
 local logger = require "resty.waf.log"
 
-
+local re_find       = ngx.re.find
 local string_byte   = string.byte
 local string_char   = string.char
 local string_format = string.format
@@ -101,6 +101,19 @@ function _M.table_has_value(needle, haystack)
 	return false
 end
 
+-- append the contents of (array-like) table b into table a
+function _M.table_append(a, b)
+	-- handle some ugliness
+	local c = type(b) == 'table' and b or { b }
+
+	local a_count = #a
+
+	for i = 1, #c do
+		a_count = a_count + 1
+		a[a_count] = c[i]
+	end
+end
+
 -- pick out dynamic data from storage key definitions
 function _M.parse_dynamic_value(waf, key, collections)
 	local lookup = function(m)
@@ -274,4 +287,38 @@ _M.sieve_collection = {
 	end,
 }
 
+-- build the msg/tag exception table for a given rule
+function _M.rule_exception(exception_table, rule)
+	if not rule.exceptions then
+		return
+	end
+
+	local ids   = {}
+	local count = 0
+
+	for i, exception in ipairs(rule.exceptions) do
+		for key, rules in pairs(exception_table.msgs) do
+			if (re_find(key, exception, 'jo')) then
+				for j, id in ipairs(rules) do
+					count = count + 1
+					ids[count] = id
+				end
+			end
+		end
+
+		for key, rules in pairs(exception_table.tags) do
+			if (re_find(key, exception, 'jo')) then
+				for j, id in ipairs(rules) do
+					count = count + 1
+					ids[count] = id
+				end
+			end
+		end
+	end
+
+	if count > 0 then
+		exception_table.meta_ids[rule.id] = ids
+	end
+end
+
 return _M

t/acceptance/ctl/02_remove_rule_by_meta.t

@@ -0,0 +1,140 @@
+use Test::Nginx::Socket::Lua;
+use Cwd qw(cwd);
+
+my $pwd = cwd();
+
+our $HttpConfig = qq{
+	lua_package_path "$pwd/lib/?.lua;$pwd/t/?.lua;;";
+	lua_package_cpath "$pwd/lib/?.lua;;";
+};
+
+repeat_each(3);
+plan tests => repeat_each() * 5 * blocks();
+
+no_shuffle();
+run_tests();
+
+__DATA__
+
+=== TEST 1: Ignore rules at runtime based on tag
+--- http_config eval: $::HttpConfig
+--- config
+	location /t {
+		access_by_lua_block {
+			local lua_resty_waf = require "resty.waf"
+			local waf           = lua_resty_waf:new()
+
+			waf:set_option("debug", true)
+			waf:set_option("mode", "ACTIVE")
+			waf:set_option("add_ruleset", "10000_ctl_meta")
+
+--[[
+rules for these tests:
+
+SecAction "id:12345,phase:1,pass,nolog,ctl:ruleRemoveByTag=mocktag"
+SecAction "id:12355,phase:1,pass,nolog,ctl:ruleRemoveByMsg=mockmsg"
+
+SecRule ARGS bar "id:12346,phase:1,deny,tag:'mocktag'"
+SecRule ARGS baz "id:12347,phase:1,deny,tag:'mocktag',tag:'othertag'"
+SecRule ARGS bat "id:12348,phase:1,deny,tag:'othertag'"
+
+SecRule ARGS foo "id:12356,phase:1,deny,msg:'mockmsg'"
+SecRule ARGS frob "id:12357,phase:1,deny,msg:'othermsg'"
+--]]
+
+			waf:exec()
+		}
+
+		content_by_lua_block { ngx.exit(ngx.HTTP_OK) }
+	}
+--- request
+GET /t
+--- error_code: 200
+--- error_log
+Runtime ignoring rules by meta
+Runtime ignoring rule 12346
+Runtime ignoring rule 12347
+Ignoring rule 12346
+Ignoring rule 12347
+--- no_error_log
+[error]
+Runtime ignoring rule 12348
+Ignoring rule 12348
+
+=== TEST 2: Matched rules not in an exception group still execute
+--- http_config eval: $::HttpConfig
+--- config
+	location /t {
+		access_by_lua_block {
+			local lua_resty_waf = require "resty.waf"
+			local waf           = lua_resty_waf:new()
+
+			waf:set_option("debug", true)
+			waf:set_option("mode", "ACTIVE")
+			waf:set_option("add_ruleset", "10000_ctl_meta")
+
+			waf:exec()
+		}
+
+		content_by_lua_block { ngx.exit(ngx.HTTP_OK) }
+	}
+--- request
+GET /t?a=bat
+--- error_code: 403
+--- error_log
+Match of rule 12348
+--- no_error_log
+[error]
+
+=== TEST 3: Ignore a rule at runtime based on msg
+--- http_config eval: $::HttpConfig
+--- config
+	location /t {
+		access_by_lua_block {
+			local lua_resty_waf = require "resty.waf"
+			local waf           = lua_resty_waf:new()
+
+			waf:set_option("debug", true)
+			waf:set_option("mode", "ACTIVE")
+			waf:set_option("add_ruleset", "10000_ctl_meta")
+
+			waf:exec()
+		}
+
+		content_by_lua_block { ngx.exit(ngx.HTTP_OK) }
+	}
+--- request
+GET /t
+--- error_code: 200
+--- error_log
+Runtime ignoring rules by meta
+Ignoring rule 12356
+--- no_error_log
+[error]
+Ignoring rule 12357
+
+=== TEST 4: Matched rules not in an exception group still execute
+--- http_config eval: $::HttpConfig
+--- config
+	location /t {
+		access_by_lua_block {
+			local lua_resty_waf = require "resty.waf"
+			local waf           = lua_resty_waf:new()
+
+			waf:set_option("debug", true)
+			waf:set_option("mode", "ACTIVE")
+			waf:set_option("add_ruleset", "10000_ctl_meta")
+
+			waf:exec()
+		}
+
+		content_by_lua_block { ngx.exit(ngx.HTTP_OK) }
+	}
+--- request
+GET /t?a=frob
+--- error_code: 403
+--- error_log
+Match of rule 12357
+--- no_error_log
+[error]
+