detect: move non-pf rules into special prefilter engines

Instead of having a per detection engine list of rule that couldn't be
prefiltered, put those into special "prefilter" engines.

For packet and frame rules this doesn't change much, it just removes
some hard coded logic from the detect engine.

For the packet non-prefilter rules in the "non-prefilter" special prefilter
engine, add additional filtering for the packet variant. It can prefilter on
alproto, dsize and dest port.

The frame non-prefilter rules are added to a single engine, that per
rule checks the alproto and the type.

For app-layer, there is an engine per progress value, per app-layer
protocol and per direction. This hooks app-layer non-prefilter rules
into the app inspect logic at the correct "progress" hook.

e.g. a rule like
        dns.query; bsize:1;

Negated MPM rules will also fall into this category:
        dns.query; content:!"abc";

Are part of a special "generic list" app engine for dns, at the
same progress hook as `dns.query`.

This all results in a lot fewer checks:

previous:

  --------------------------------------------------------------------------
  Date: 1/29/2025 -- 10:22:25. Sorted by: number of checks.
  --------------------------------------------------------------------------
   Num      Rule         Gid      Rev      Ticks        %      Checks   Matches  Max Ticks   Avg Ticks   Avg Match   Avg No Match
  -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- --------------
  1        20           1        0        181919672    11.85  588808   221      60454       308.96      2691.46     308.07
  2        50           1        0        223455914    14.56  453104   418      61634       493.17      3902.59     490.02
  3        60           1        0        185990683    12.12  453104   418      60950       410.48      1795.40     409.20
  4        51           1        0        192436011    12.54  427028   6084     61223       450.64      2749.12     417.42
  5        61           1        0        180401533    11.75  427028   6084     61093       422.46      2177.04     397.10
  6        70           1        0        153899099    10.03  369836   0        61282       416.13      0.00        416.13
  7        71           1        0        123389405    8.04   369836   12833    44921       333.63      2430.23     258.27
  8        41           1        0        63889876     4.16   155824   12568    39138       410.01      1981.97     272.10
  9        40           1        0        64149724     4.18   155818   210      39792       411.70      4349.57     406.38
  10       10           1        0        70848850     4.62   65558    0        39544       1080.70     0.00        1080.70
  11       11           1        0        94743878     6.17   65558    32214    60547       1445.19     2616.14     313.92

this commit:

  --------------------------------------------------------------------------
  Date: 1/29/2025 -- 10:15:46. Sorted by: number of checks.
  --------------------------------------------------------------------------
   Num      Rule         Gid      Rev      Ticks        %      Checks   Matches  Max Ticks   Avg Ticks   Avg Match   Avg No Match
  -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- --------------
  1        50           1        0        138776766    19.23  95920    418      167584      1446.80     3953.11     1435.83
  2        60           1        0        97988084     13.58  95920    418      182817      1021.56     1953.63     1017.48
  3        51           1        0        105318318    14.60  69838    6084     65649       1508.04     2873.38     1377.74
  4        61           1        0        89571260     12.41  69838    6084     164632      1282.56     2208.41     1194.20
  5        11           1        0        91132809     12.63  32779    32214    373569      2780.22     2785.58     2474.45
  6        10           1        0        66095303     9.16   32779    0        56704       2016.39     0.00        2016.39
  7        70           1        0        48107573     6.67   12928    0        42832       3721.19     0.00        3721.19
  8        71           1        0        32308792     4.48   12928    12833    39565       2499.13     2510.05     1025.09
  9        41           1        0        25546837     3.54   12886    12470    41479       1982.53     1980.84     2033.05
  10       40           1        0        26069992     3.61   12886    210      38495       2023.13     4330.05     1984.91
  11       20           1        0        639025       0.09   221      221      14750       2891.52     2891.52     0.00

Author: victorjulien

src/app-layer-frames.h

@@ -26,6 +26,8 @@
 
 #include "rust.h"
 
+/** special value for matching any type */
+#define FRAME_ANY_TYPE 62
 /** max 63 to fit the 64 bit per protocol space */
 #define FRAME_STREAM_TYPE 63
 

src/decode.h

@@ -94,6 +94,9 @@ enum PktSrcEnum {
 
 #include "util-validate.h"
 
+/* for now a uint8_t is enough -- here in decode as it's part of the packet */
+#define SignatureMask uint8_t
+
 /* forward declarations */
 struct DetectionEngineThreadCtx_;
 typedef struct AppLayerThreadCtx_ AppLayerThreadCtx;
@@ -508,6 +511,7 @@ typedef struct Packet_
     /* coccinelle: Packet:flowflags:FLOW_PKT_ */
 
     uint8_t app_update_direction; // enum StreamUpdateDir
+    SignatureMask sig_mask;
 
     /* Pkt Flags */
     uint32_t flags;

src/detect-engine-build.c

@@ -1990,8 +1990,6 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx)
 
         PrefilterSetupRuleGroup(de_ctx, sgh);
 
-        SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh);
-
         sgh->id = idx;
         cnt++;
     }
@@ -2002,7 +2000,7 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx)
     if (de_ctx->decoder_event_sgh != NULL) {
         /* no need to set filestore count here as that would make a
          * signature not decode event only. */
-        SigGroupHeadBuildNonPrefilterArray(de_ctx, de_ctx->decoder_event_sgh);
+        PrefilterSetupRuleGroup(de_ctx, de_ctx->decoder_event_sgh);
     }
 
     int dump_grouping = 0;

src/detect-engine-frame.c

@@ -76,8 +76,9 @@ void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead
     SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt);
     PrefilterEngine *engine = sgh->frame_engines;
     do {
-        BUG_ON(engine->alproto == ALPROTO_UNKNOWN);
-        if (engine->alproto == alproto && engine->ctx.frame_type == frame->type) {
+        if ((engine->alproto == alproto || engine->alproto == ALPROTO_UNKNOWN) &&
+                (engine->ctx.frame_type == frame->type ||
+                        engine->ctx.frame_type == FRAME_ANY_TYPE)) {
             SCLogDebug("frame %p engine %p", frame, engine);
             PREFILTER_PROFILING_START(det_ctx);
             engine->cb.PrefilterFrame(det_ctx, engine->pectx, p, frames, frame);