logsearch-jobs.yml

instance_groups:
  #######################################################
  #First deploy group - elasticsearch_master, maintenance
  #######################################################
  - name: elasticsearch_master
    instances: 3
    jobs:
      - name: bpm
        release: bpm
      - name: elasticsearch
        release: logsearch
        provides:
          elasticsearch: {as: elasticsearch_master}
        consumes:
          elasticsearch: {from: elasticsearch_master}
        properties:
          elasticsearch:
            cluster_name: logsearch
            jvm_options:
              - "-Dlog4j2.formatMsgNoLookups=true"
            node:
              allow_master: true
              allow_data: false
            limits:
              fd: 131072 # 2 ** 17
            health:
              timeout: 900
            recovery:
              delay_allocation_restart: "15m"
            config_options:
              indices.query.bool.max_clause_count: 2048
      - name: snort-config
        release: jammy-snort
        properties:
          snort:
            rules:
              - 'alert tcp any any -> any 9200 (msg:"Unexpected logsearch action"; content:"POST"; http_method; content: "logs-app"; http_uri; content:"/_update"; http_uri; classtype:web-application-attack; sid:343080002; rev:1;)'
              - 'alert tcp any any -> any 9200 (msg:"Unexpected logsearch action"; content:"DELETE"; http_method; content: "logs-app"; http_uri; classtype:web-application-attack; sid:343080004; rev:1;)'
              - (( concat "suppress gen_id 1, sig_id 343080004, track by_src, ip " instance_groups.maintenance.networks.services.static_ips.[0] ))
              - 'suppress gen_id 1, sig_id 343080004, track by_src, ip 127.0.0.1'
              - 'suppress gen_id 1, sig_id 57907, track by_src, ip 127.0.0.1'
              - 'suppress gen_id 1, sig_id 26275, track by_src, ip 127.0.0.1'
              - 'suppress gen_id 1, sig_id 41495, track by_src, ip 127.0.0.1'
              - 'suppress gen_id 1, sig_id 58741'
              - 'drop tcp $EXTERNAL_NET any -> $HOME_NET $HTTP_PORTS (msg:"SERVER-OTHER Apache Log4j logging remote code execution attempt"; flow:to_server,established; content:"${"; fast_pattern:only; http_client_body; pcre:"/\x24\x7b.{0,200}(%(25)?24|\x24)(%(25)?7b|\x7b).{0,200}(%(25)?3a|\x3a)(%(25)?(27|2d|5c|22)|[\x27\x2d\x5c\x22])*([jndi\x7d\x3a\x2d]|(%(25)?(7d|3a|2d))|(%(25)?5c|\x5c)u00[a-f0-9]{2}){1,4}(%(25)?(22|27)|[\x22\x27])?(%(25)?(3a|7d)|[\x3a\x7djndi])/Pi"; metadata:policy balanced-ips drop, policy connectivity-ips drop, policy max-detect-ips drop, policy security-ips drop, ruleset community, service http; reference:cve,2021-44228; reference:cve,2021-44832; reference:cve,2021-45046; reference:cve,2021-45105; classtype:attempted-user; sid:58741000; rev:6;)'
    persistent_disk_type: logsearch_es_master
    stemcell: default
    azs: [z1]
    networks:
      - name: services
        static_ips:
          - (( grab terraform_outputs.logsearch_static_ips.[0]))
          - (( grab terraform_outputs.logsearch_static_ips.[1]))
          - (( grab terraform_outputs.logsearch_static_ips.[2]))
    vm_extensions: [logsearch-lb]
    update:
      max_in_flight: 1 # Should never update more than one ES master node at a time or cluster will go down
  - name: redis
    instances: 1
    jobs:
      - name: bpm
        release: bpm
      - {name: redis, release: logsearch-for-cloudfoundry}
    persistent_disk_type: logsearch_redis
    stemcell: default
    azs: [z1]
    networks:
      - name: services
  - name: maintenance
    instances: 1
    vm_extensions: [errand-profile]
    jobs:
      - name: bpm
        release: bpm
      - name: elasticsearch
        release: logsearch
        consumes:
          elasticsearch: {from: elasticsearch_master}
        properties:
          elasticsearch:
            jvm_options:
              - "-Dlog4j2.formatMsgNoLookups=true"
      - name: curator
        release: logsearch
        properties:
          curator:
            execute:
              daily: true
              hourly: false
            purge_logs:
              retention_period: 180
            actions:
              - action: forcemerge
                description: >-
                  Force merging of older indices to make their shards more compact and search-optimized
                options:
                  ignore_empty_list: true
                  max_num_segments: 1
                filters:
                  - filtertype: pattern
                    kind: regex
                    value: logs*
                  - filtertype: age
                    source: creation_date
                    direction: older
                    unit: days
                    unit_count: 45
                  - filtertype: forcemerged
                    max_num_segments: 1
                    exclude: true
              - action: index_settings
                description: >-
                  Sets indices older than 1 day to be read only
                options:
                  index_settings:
                    index:
                      blocks:
                        write: true
                filters:
                  - filtertype: pattern
                    kind: regex
                    value: logs*
                  - filtertype: age
                    source: creation_date
                    direction: older
                    unit: days
                    unit_count: 1
      - name: elasticsearch_config
        release: logsearch
        properties:
          elasticsearch_config:
            component_templates:
              - shards-and-replicas: /var/vcap/jobs/elasticsearch_config/index-templates/shards-and-replicas.json
              - index-settings: /var/vcap/jobs/elasticsearch_config/index-templates/index-settings.json
              - index-mappings: /var/vcap/jobs/elasticsearch_config/index-templates/index-mappings.json
              - index-mappings-lfc: /var/vcap/jobs/elasticsearch-config-lfc/component-index-mappings.json
              - index-mappings-app-lfc: /var/vcap/jobs/elasticsearch-config-lfc/component-index-mappings-app.json
              - index-mappings-platform-lfc: /var/vcap/jobs/elasticsearch-config-lfc/component-index-mappings-platform.json
            index_templates:
              - index-mappings-lfc: /var/vcap/jobs/elasticsearch-config-lfc/index-mappings.json
              - index-mappings-app-lfc: /var/vcap/jobs/elasticsearch-config-lfc/index-mappings-app.json
              - index-mappings-platform-lfc: /var/vcap/jobs/elasticsearch-config-lfc/index-mappings-platform.json
      - name: elasticsearch-config-lfc
        release: logsearch-for-cloudfoundry
        properties:
          elasticsearch_config:
            app_index_settings:
              index.mapping.total_fields.limit: 2000
              index.queries.cache.enabled: "false"
              index.max_docvalue_fields_search: 800
            base_index_component_name: index-mappings-lfc
            app_index_component_name: index-mappings-app-lfc
            platform_index_component_name: index-mappings-platform-lfc
            index_mappings_component_name: index-mappings
            index_settings_component_name: index-settings
            shards_and_replicas_component_name: shards-and-replicas
      - name: elasticsearch_exporter
        release: prometheus
        properties:
          elasticsearch_exporter:
            es:
              uri: http://localhost:9200
              all: true
      - name: upload-kibana-objects
        release: logsearch-for-cloudfoundry
        properties:
          cloudfoundry:
            firehose_events:
              - LogMessage
              - ContainerMetric
            system_domain: (( grab $CF_SYSTEM_DOMAIN ))
            logs_hostname: logs-deprecated-waiting-for-delete
            user: (( grab $CF_USERNAME ))
            password: (( grab $CF_PASSWORD ))
          kibana_objects:
            upload_patterns:
              - {type: index-pattern, pattern: "/var/vcap/jobs/upload-kibana-objects/kibana-objects/index-pattern/logs-app.json"}
              - {type: search, pattern: "/var/vcap/jobs/upload-kibana-objects/kibana-objects/search/app-*.json"}
              - {type: visualization, pattern: "/var/vcap/jobs/upload-kibana-objects/kibana-objects/visualization/App-*.json"}
              - {type: dashboard, pattern: "/var/vcap/jobs/upload-kibana-objects/kibana-objects/dashboard/App-*.json"}
      # TODO: Drop after https://github.com/cloudfoundry-community/logsearch-for-cloudfoundry/pull/267 is merged
      - name: cron
        release: cron
        properties:
          cron:
            entries:
              - script:
                  name: slow-logs
                  contents: (( file "cronjobs/slow-logs.sh" ))
                variables:
                  HOST: (( grab instance_groups.elasticsearch_master.networks.services.static_ips.[0] ))
                minute: "0"
                hour: "0"
                day: "*"
                month: "*"
                wday: "*"
                user: root
    stemcell: default
    azs: [z1]
    networks:
      - name: services
        static_ips:
          - (( grab terraform_outputs.logsearch_static_ips.[5] ))
    update:
      serial: true # Block on this job to create deploy group 1
  #########################################################
  #2nd deploy group - elasticsearch_data, kibana, ingestors
  #########################################################
  - name: elasticsearch_data
    instances: 11
    jobs:
      - name: bpm
        release: bpm
      - name: elasticsearch
        release: logsearch
        consumes:
          elasticsearch: {from: elasticsearch_master}
        properties:
          elasticsearch:
            node:
              allow_master: false
              allow_data: true
            limits:
              fd: 131072 # 2 ** 17
            health:
              timeout: 900
              disable_post_start: true
            recovery:
              delay_allocation_restart: "15m"
            config_options:
              indices.query.bool.max_clause_count: 2048
            jvm_options:
              - "-Dlog4j2.formatMsgNoLookups=true"
    persistent_disk_type: logsearch_es_data
    stemcell: default
    azs: [z1]
    networks:
      - name: services
    update:
      max_in_flight: 1 # Only update 1 ES data node at a time or risk downtime
    env:
      bosh:
        swap_size: 0
  - name: kibana
    instances: 2
    jobs:
      - name: bpm
        release: bpm
      - name: elasticsearch
        release: logsearch
        consumes:
          elasticsearch: {from: elasticsearch_master}
        properties:
          elasticsearch:
            heap_size: 2G
            jvm_options:
              - "-Dlog4j2.formatMsgNoLookups=true"
      - name: kibana
        release: logsearch
        provides:
          kibana: {as: kibana_link}
        consumes:
          elasticsearch: {from: elasticsearch_master}
        properties:
          kibana:
            memory_limit: 75
            default_app_id: "dashboard/App-Overview"
            plugins:
              - auth: /var/vcap/packages/kibana-auth-plugin/kibana-auth-plugin.zip
            console.enabled: false
            config_options:
              server.maxPayloadBytes: 6291456
              # max CSV report size ~20M
              # should never exceed http.max_content_length, which defaults to 100M
              xpack.reporting.csv.maxSizeBytes: 20971520
            env:
              - NODE_ENV: production
            source_files: [/var/vcap/jobs/kibana-auth-plugin/config/config.sh]
            health:
              timeout: 600
      - name: kibana-auth-plugin
        release: logsearch-for-cloudfoundry
        properties:
          kibana-auth:
            cloudfoundry:
              skip_ssl_validation: false
              client_id: kibana_oauth2_client
              system_org: cloud-gov-operators # Org Managers of this org get admin access
            session_key: (( param "specify kibana session key" ))
      - name: route_registrar
        release: routing
        properties:
          route_registrar:
            routes:
              - name: logsearch
                registration_interval: 2s
                port: 5601
                health_check:
                  name: kibana-up
                  script_path: /var/vcap/jobs/kibana/bin/post-start
                timeout: 1s
    stemcell: default
    azs: [z1]
    networks:
      - name: services
    vm_extensions:
      - 50GB_ephemeral_disk
    env:
      bosh:
        swap_size: 0
  - name: archiver
    jobs:
      - name: bpm
        release: bpm
      - name: elasticsearch
        release: logsearch
        consumes:
          elasticsearch: {from: elasticsearch_master}
        properties:
          elasticsearch:
            jvm_options:
              - "-Dlog4j2.formatMsgNoLookups=true"
            migrate_data_path: true
      - name: archiver_syslog
        release: logsearch
        properties:
          logstash:
            queue:
              max_bytes: 30gb
          logstash_archiver:
            files: 16384
          logstash_ingestor:
            outputs:
              - plugin: s3
                options:
                  region: (( grab terraform_outputs.vpc_region ))
                  bucket: (( grab terraform_outputs.logsearch_archive_bucket_name ))
                  validate_credentials_on_root_bucket: false # https://github.com/logstash-plugins/logstash-output-s3/issues/132
                  server_side_encryption: true
                  time_file: 5
                  prefix: "%{+yyyy/MM/dd/HH/mm}"
                  encoding: "gzip"
      - name: ingestor_cloudfoundry-firehose
        release: logsearch-for-cloudfoundry
        properties:
          cloudfoundry:
            firehose_subscription_id: logsearch-archiver
            firehose_events:
              - LogMessage
              - ContainerMetric
            firehose_client_id: logsearch_firehose_ingestor
            firehose_cc_pull_interval: 300s
            skip_ssl_validation: false
          syslog:
            host: 127.0.0.1
            port: 5514
    persistent_disk_type: logsearch_ingestor
    stemcell: default
    azs: [z1]
    networks:
      - name: services
    vm_extensions:
      - logsearch-ingestor-profile
      - 50GB_ephemeral_disk
  - name: ingestor
    jobs:
      - name: bpm
        release: bpm
      - name: elasticsearch
        release: logsearch
        consumes:
          elasticsearch: {from: elasticsearch_master}
        properties:
          elasticsearch:
            heap_size: 1G
            jvm_options:
              - "-Dlog4j2.formatMsgNoLookups=true"
      - name: ingestor_syslog
        release: logsearch
        consumes:
          elasticsearch: {from: elasticsearch_master}
        provides:
          ingestor: {as: ingestor_link}
        properties:
          logstash:
            jvm_options:
              - "-Dlog4j2.formatMsgNoLookups=true"
            queue:
              max_bytes: 30gb
          logstash_parser:
            elasticsearch:
              # Use per-day indexing strategy
              index: "logs-app-%{+YYYY.MM.dd}"
              index_type: "%{@type}"
              data_hosts: [127.0.0.1]
            filters:
              - logsearch-for-cf: /var/vcap/packages/logsearch-config-logstash-filters/logstash-filters-default.conf
            deployment_dictionary:
              - /var/vcap/packages/logsearch-config/deployment_lookup.yml
              - /var/vcap/jobs/parser-config-lfc/config/deployment_lookup.yml
      - name: ingestor_cloudfoundry-firehose
        release: logsearch-for-cloudfoundry
        properties:
          cloudfoundry:
            firehose_subscription_id: logsearch-ingestor
            firehose_events:
              - LogMessage
              - ContainerMetric
            firehose_client_id: logsearch_firehose_ingestor
            firehose_cc_pull_interval: 300s
            skip_ssl_validation: false
          syslog:
            host: 127.0.0.1
            port: 5514
      - name: parser-config-lfc
        release: logsearch-for-cloudfoundry
    persistent_disk_type: logsearch_ingestor
    stemcell: default
    azs: [z1]
    networks:
      - name: services
  ###########################
  #3nd deploy group - errands
  ###########################
  - name: smoke-tests
    instances: 1
    vm_extensions: [errand-profile]
    stemcell: default
    azs: [z1]
    networks:
      - name: services
    lifecycle: errand
    release: logsearch
    jobs:
      - name: smoke_tests
        release: logsearch
        consumes:
          elasticsearch: {from: elasticsearch_master}
          ingestor_link: {from: ingestor_syslog}