config.yaml

logging:
  base:
    debug: false
  modules:
    log_storage.logserver:
      debug: false
    log_collection.collector:
      debug: false
    log_collection.batch_handler:
      debug: false
    log_filtering.prefilter:
      debug: false
    data_inspection.inspector:
      debug: false
    data_analysis.detector:
      debug: false

pipeline:
  log_storage:
    logserver:
      input_kafka_topic: "LogServer"
      input_file: "/opt/file.txt"
      max_number_of_connections: 1000

  log_collection:
    collector:
      logline_format:
        - [ "timestamp", RegEx, '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$' ]
        - [ "status_code", ListItem, [ "NOERROR", "NXDOMAIN" ], [ "NXDOMAIN" ] ]
        - [ "client_ip", IpAddress ]
        - [ "dns_server_ip", IpAddress ]
        - [ "domain_name", RegEx, '^(?=.{1,253}$)((?!-)[A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,63}$' ]
        - [ "record_type", ListItem, [ "A", "AAAA" ] ]
        - [ "response_ip", IpAddress ]
        - [ "size", RegEx, '^\d+b$' ]
    batch_handler:
      batch_size: 1000
      batch_timeout: 20.0
      subnet_id:
        ipv4_prefix_length: 24
        ipv6_prefix_length: 64

  data_inspection:
    inspector:
      mode: univariate # multivariate, ensemble
      # Only used when mode is set to ensemble
      ensemble:
        model: WeightEnsemble
        module: streamad.process
        model_args:
      models:
        - model: ZScoreDetector
          module: streamad.model
          model_args:
            is_global: false
      anomaly_threshold: 0.01
      score_threshold: 0.5
      time_type: ms
      time_range: 20

  data_analysis:
    detector:
      model: rf # XGBoost
      checksum: ba1f718179191348fe2abd51644d76191d42a5d967c6844feb3371b6f798bf06
      base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/
      threshold: 0.5

environment:
  timestamp_format: "%Y-%m-%dT%H:%M:%S.%fZ"
  kafka_brokers:
    - hostname: 172.27.0.3
      port: 8097
    - hostname: 172.27.0.4
      port: 8098
    - hostname: 172.27.0.5
      port: 8099
  logserver:
    hostname: 172.27.0.8
    port_in: 9998
    port_out: 9999