From 38d7601d86cfd6139d433e922bd0aaa31aa76c99 Mon Sep 17 00:00:00 2001 From: May Lee Date: Wed, 26 Nov 2025 14:13:13 -0500 Subject: [PATCH] delete old setup docs --- .../set_up_pipelines/archive_logs/_index.md | 51 -- .../archive_logs/amazon_data_firehose.md | 459 ----------------- .../archive_logs/amazon_s3.md | 454 ----------------- .../archive_logs/datadog_agent.md | 475 ------------------ .../set_up_pipelines/archive_logs/fluent.md | 464 ----------------- .../archive_logs/google_pubsub.md | 455 ----------------- .../archive_logs/http_client.md | 458 ----------------- .../archive_logs/http_server.md | 454 ----------------- .../set_up_pipelines/archive_logs/kafka.md | 458 ----------------- .../set_up_pipelines/archive_logs/logstash.md | 459 ----------------- .../set_up_pipelines/archive_logs/socket.md | 439 ---------------- .../archive_logs/splunk_hec.md | 460 ----------------- .../archive_logs/splunk_tcp.md | 459 ----------------- .../sumo_logic_hosted_collector.md | 459 ----------------- .../set_up_pipelines/archive_logs/syslog.md | 463 ----------------- .../set_up_pipelines/dual_ship_logs/_index.md | 47 -- .../dual_ship_logs/amazon_data_firehose.md | 409 --------------- .../dual_ship_logs/amazon_s3.md | 403 --------------- .../dual_ship_logs/datadog_agent.md | 423 ---------------- .../set_up_pipelines/dual_ship_logs/fluent.md | 411 --------------- .../dual_ship_logs/google_pubsub.md | 402 --------------- .../dual_ship_logs/http_client.md | 406 --------------- .../dual_ship_logs/http_server.md | 403 --------------- .../set_up_pipelines/dual_ship_logs/kafka.md | 407 --------------- .../dual_ship_logs/logstash.md | 408 --------------- .../set_up_pipelines/dual_ship_logs/socket.md | 384 -------------- .../dual_ship_logs/splunk_hec.md | 418 --------------- .../dual_ship_logs/splunk_tcp.md | 414 --------------- .../sumo_logic_hosted_collector.md | 414 --------------- .../set_up_pipelines/dual_ship_logs/syslog.md | 411 --------------- .../generate_metrics/_index.md | 55 -- .../generate_metrics/amazon_data_firehose.md | 408 --------------- .../generate_metrics/amazon_s3.md | 403 --------------- .../generate_metrics/datadog_agent.md | 421 ---------------- .../generate_metrics/fluent.md | 409 --------------- .../generate_metrics/google_pubsub.md | 402 --------------- .../generate_metrics/http_client.md | 404 --------------- .../generate_metrics/http_server.md | 403 --------------- .../generate_metrics/kafka.md | 407 --------------- .../generate_metrics/logstash.md | 408 --------------- .../generate_metrics/socket.md | 384 -------------- .../generate_metrics/splunk_hec.md | 414 --------------- .../generate_metrics/splunk_tcp.md | 412 --------------- .../sumo_logic_hosted_collector.md | 412 --------------- .../generate_metrics/syslog.md | 409 --------------- .../set_up_pipelines/log_enrichment/_index.md | 52 -- .../log_enrichment/amazon_data_firehose.md | 408 --------------- .../log_enrichment/amazon_s3.md | 403 --------------- .../log_enrichment/datadog_agent.md | 423 ---------------- .../set_up_pipelines/log_enrichment/fluent.md | 411 --------------- .../log_enrichment/google_pubsub.md | 402 --------------- .../log_enrichment/http_client.md | 406 --------------- .../log_enrichment/http_server.md | 404 --------------- .../set_up_pipelines/log_enrichment/kafka.md | 407 --------------- .../log_enrichment/logstash.md | 408 --------------- .../set_up_pipelines/log_enrichment/socket.md | 384 -------------- .../log_enrichment/splunk_hec.md | 409 --------------- .../log_enrichment/splunk_tcp.md | 408 --------------- .../sumo_logic_hosted_collector.md | 404 --------------- .../set_up_pipelines/log_enrichment/syslog.md | 411 --------------- .../log_volume_control/_index.md | 61 --- .../amazon_data_firehose.md | 408 --------------- .../log_volume_control/amazon_s3.md | 403 --------------- .../log_volume_control/datadog_agent.md | 423 ---------------- .../log_volume_control/fluent.md | 411 --------------- .../log_volume_control/google_pubsub.md | 402 --------------- .../log_volume_control/http_client.md | 406 --------------- .../log_volume_control/http_server.md | 403 --------------- .../log_volume_control/kafka.md | 407 --------------- .../log_volume_control/logstash.md | 408 --------------- .../log_volume_control/socket.md | 384 -------------- .../log_volume_control/splunk_hec.md | 409 --------------- .../log_volume_control/splunk_tcp.md | 408 --------------- .../sumo_logic_hosted_collector.md | 404 --------------- .../log_volume_control/syslog.md | 411 --------------- .../sensitive_data_redaction/_index.md | 49 -- .../amazon_data_firehose.md | 420 ---------------- .../sensitive_data_redaction/amazon_s3.md | 405 --------------- .../sensitive_data_redaction/datadog_agent.md | 425 ---------------- .../sensitive_data_redaction/fluent.md | 413 --------------- .../sensitive_data_redaction/google_pubsub.md | 404 --------------- .../sensitive_data_redaction/http_client.md | 408 --------------- .../sensitive_data_redaction/http_server.md | 405 --------------- .../sensitive_data_redaction/kafka.md | 409 --------------- .../sensitive_data_redaction/logstash.md | 410 --------------- .../sensitive_data_redaction/socket.md | 386 -------------- .../sensitive_data_redaction/splunk_hec.md | 411 --------------- .../sensitive_data_redaction/splunk_tcp.md | 410 --------------- .../sumo_logic_hosted_collector.md | 412 --------------- .../sensitive_data_redaction/syslog.md | 413 --------------- .../set_up_pipelines/split_logs/_index.md | 47 -- .../split_logs/amazon_data_firehose.md | 408 --------------- .../set_up_pipelines/split_logs/amazon_s3.md | 403 --------------- .../split_logs/datadog_agent.md | 436 ---------------- .../set_up_pipelines/split_logs/fluent.md | 411 --------------- .../split_logs/google_pubsub.md | 402 --------------- .../split_logs/http_client.md | 406 --------------- .../split_logs/http_server.md | 403 --------------- .../set_up_pipelines/split_logs/kafka.md | 407 --------------- .../set_up_pipelines/split_logs/logstash.md | 408 --------------- .../set_up_pipelines/split_logs/socket.md | 384 -------------- .../set_up_pipelines/split_logs/splunk_hec.md | 415 --------------- .../set_up_pipelines/split_logs/splunk_tcp.md | 414 --------------- .../split_logs/sumo_logic_hosted_collector.md | 414 --------------- .../set_up_pipelines/split_logs/syslog.md | 411 --------------- 105 files changed, 41015 deletions(-) delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/_index.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_s3.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/datadog_agent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/fluent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/google_pubsub.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/http_client.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/http_server.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/kafka.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/logstash.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/socket.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_hec.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/archive_logs/syslog.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/_index.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/datadog_agent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/fluent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_client.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_server.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/kafka.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/logstash.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/socket.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/syslog.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/_index.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/fluent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_client.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_server.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/kafka.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/logstash.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/socket.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/generate_metrics/syslog.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/_index.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/datadog_agent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/fluent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_client.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_server.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/kafka.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/logstash.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/socket.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_enrichment/syslog.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/_index.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/datadog_agent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/fluent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_client.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_server.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/kafka.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/logstash.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/socket.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/log_volume_control/syslog.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/_index.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/datadog_agent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/fluent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_client.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/_index.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_s3.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/datadog_agent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/fluent.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/google_pubsub.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/http_client.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/http_server.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/kafka.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/logstash.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/socket.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_hec.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_tcp.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector.md delete mode 100644 content/en/observability_pipelines/set_up_pipelines/split_logs/syslog.md diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/_index.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/_index.md deleted file mode 100644 index 0a56dfc956f..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/_index.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Archive Logs to Datadog Archives -disable_toc: false -aliases: - - /observability_pipelines/archive_logs/ -private: true -cascade: - private: true ---- - -## Overview - -Use Observability Pipelines to route ingested logs to a cloud storage solution (Amazon S3, Google Cloud Storage, or Azure Storage) in Datadog-rehydratable format. You can then rehydrate the archive in Datadog ad hoc whenever you need to analyze and investigate them. This is useful when: - -- You are migrating from another log vendor to Datadog Log Management, and want to ensure you have access to historical logs when you finish migrating. -- You have a high volume of noisy logs, but you may need to index them in Log Management ad hoc. -- You have a retention policy. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -Select a source to get started: - -- [Amazon Data Firehose][12] -- [Amazon S3][11] -- [Datadog Agent][1] -- [Fluentd or Fluent Bit][2] -- [Google Pub/Sub][3] -- [HTTP Client][4] -- [HTTP Server][5] -- [Kafka][13] -- [Logstash][6] -- [Splunk HTTP Event Collector (HEC)][7] -- [Splunk Heavy or Universal Forwarders (TCP)][8] -- [Socket (TCP or UDP)][14] -- [Sumo Logic Hosted Collector][9] -- [rsylsog or syslog-ng][10] - -[1]: /observability_pipelines/archive_logs/datadog_agent -[2]: /observability_pipelines/archive_logs/fluent -[3]: /observability_pipelines/set_up_pipelines/archive_logs/google_pubsub -[4]: /observability_pipelines/archive_logs/http_client -[5]: /observability_pipelines/set_up_pipelines/archive_logs/http_server -[6]: /observability_pipelines/set_up_pipelines/archive_logs/logstash -[7]: /observability_pipelines/archive_logs/splunk_hec -[8]: /observability_pipelines/archive_logs/splunk_tcp -[9]: /observability_pipelines/archive_logs/sumo_logic_hosted_collector -[10]: /observability_pipelines/archive_logs/syslog -[11]: /observability_pipelines/set_up_pipelines/archive_logs/amazon_s3 -[12]: /observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose -[13]: /observability_pipelines/set_up_pipelines/archive_logs/kafka -[14]: /observability_pipelines/set_up_pipelines/archive_logs/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose.md deleted file mode 100644 index e9bc203d607..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose.md +++ /dev/null @@ -1,459 +0,0 @@ ---- -title: Archive Logs for Amazon Data Firehose -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to format your Amazon Data Firehose logs into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following steps: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_data_firehose %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See the [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select the **Amazon Data Firehose** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_data_firehose %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Amazon Data Firehose address. The Observability Pipelines Worker listens to this address and port for incoming logs from Amazon Data Firehose. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker - -{{% observability_pipelines/log_source_configuration/amazon_data_firehose %}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_s3.md deleted file mode 100644 index 57798ee147d..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_s3.md +++ /dev/null @@ -1,454 +0,0 @@ ---- -title: Archive Logs for Amazon S3 -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to format your Amazon S3 logs into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_s3 %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See the [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select the **Amazon S3** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_s3 %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. In the **AWS S3 SQS URL** field, enter the URL of the SQS queue to which the S3 bucket sends notification events. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/datadog_agent.md deleted file mode 100644 index a2c43c2d070..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/datadog_agent.md +++ /dev/null @@ -1,475 +0,0 @@ ---- -title: Archive Logs for the Datadog Agent -disable_toc: false -aliases: - - /observability_pipelines/archive_logs/datadog_agent/ ---- - -## Overview - -Configure your Datadog Agent so that the Observability Pipelines Worker formats the logs collected into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting the Datadog Agent to the Observability Pipelines Worker](#connect-the-datadog-agent-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/datadog_agent %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select **Datadog Agent** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/datadog_agent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the listener address, which is the address and port the Observability Pipelines Worker listens on for incoming logs from the Datadog Agent. For example, `0.0.0.0:`. -1. Provide the environment variables for each of your selected destinations. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Connect the Datadog Agent to the Observability Pipelines Worker - -Use the Agent configuration file or the Agent Helm chart values file to connect the Datadog Agent to the Observability Pipelines Worker. - -{{< tabs >}} -{{% tab "Agent configuration file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent %}} - -{{% /tab %}} -{{% tab "Agent Helm values file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent_kubernetes %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/fluent.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/fluent.md deleted file mode 100644 index bbf2851d7d6..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/fluent.md +++ /dev/null @@ -1,464 +0,0 @@ ---- -title: Archive Logs for Fluent -disable_toc: false -aliases: - - /observability_pipelines/archive_logs/fluent/ ---- - -## Overview - -Configure Fluentd or Fluent Bit so that the Observability Pipelines Worker formats the logs collected into a Datadog-rehydratable format before routing them to Datadog Log Archives. - - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/fluent %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select **Fluentd** or **Fluent Bit** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/fluent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Fluent socket address and port. The Observability Pipelines Worker listens on this address for incoming log messages. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker - -{{% observability_pipelines/log_source_configuration/fluent %}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/google_pubsub.md deleted file mode 100644 index 929fe55fa7f..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/google_pubsub.md +++ /dev/null @@ -1,455 +0,0 @@ ---- -title: Archive Logs for Google Pub/Sub -disable_toc: false -aliases: - - /observability_pipelines/set_up_pipelines/archive_logs/google_pub_sub/ ---- - -## Overview - -Configure Google Pub/Sub so that the Observability Pipelines Worker formats the logs collected into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/google_pubsub %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See the [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select the **Google Pub/Sub** source. - -### Set up the source - -{{% observability_pipelines/source_settings/google_pubsub %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_client.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_client.md deleted file mode 100644 index a4f76b1e65c..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_client.md +++ /dev/null @@ -1,458 +0,0 @@ ---- -title: Archive Logs for HTTP Client -disable_toc: false -aliases: - - /observability_pipelines/archive_logs/http_client/ ---- - -## Overview - -Use the Observability Pipelines Worker to format your HTTP server logs into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_client %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select **HTTP Client** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_client %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the full path of the HTTP/S endpoint URL. For example, `https://127.0.0.8/logs`. The Observability Pipelines Worker collects logs events from this endpoint. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_server.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_server.md deleted file mode 100644 index bf946fcb45b..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_server.md +++ /dev/null @@ -1,454 +0,0 @@ ---- -title: Archive Logs for HTTP Server -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to format HTTP client logs into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_server %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See the [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select the **HTTP Server** source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_server %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the HTTP/S server address, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens to this socket address for your HTTP client logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/kafka.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/kafka.md deleted file mode 100644 index 568a8f79df1..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/kafka.md +++ /dev/null @@ -1,458 +0,0 @@ ---- -title: Archive Logs for Kafka -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to format your logs from Kafka topics into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/kafka %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See the [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select the **Kafka** source. - -### Set up the source - -{{% observability_pipelines/source_settings/kafka %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the host and port of the Kafka bootstrap servers, which clients should use to connect to the Kafka cluster and discover all the other hosts in the cluster. Must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. - - If you enabled SASL, enter the Kafka SASL username and Kafka SASL password. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/logstash.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/logstash.md deleted file mode 100644 index 3f4d63ba2b2..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/logstash.md +++ /dev/null @@ -1,459 +0,0 @@ ---- -title: Archive Logs for Logstash -disable_toc: false ---- - -## Overview - -Configure Logstash so that the Observability Pipelines Worker formats the logs collected into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/logstash %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See the [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select the **Logstash** source. - -### Set up the source - -{{% observability_pipelines/source_settings/logstash %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Logstash address and port, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens on this address for incoming log messages. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker - -{{% observability_pipelines/log_source_configuration/logstash %}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/socket.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/socket.md deleted file mode 100644 index e1d8b9c369f..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/socket.md +++ /dev/null @@ -1,439 +0,0 @@ ---- -title: Archive Logs for the Socket Source (TCP or UDP) -disable_toc: false ---- - -## Overview - -Send your logs over a socket connection to the Observability Pipelines Worker to format your logs into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/socket %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See the [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select the **Socket** source. - -### Set up the source - -{{% observability_pipelines/source_settings/socket %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the socket address and port, such as `0.0.0.0:9000`. This is the address and port the Observability Pipelines Worker listens on for incoming logs. The socket address must include a port. -1. If you enabled TLS, enter the TLS passphrase. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_hec.md deleted file mode 100644 index e675270e2fa..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_hec.md +++ /dev/null @@ -1,460 +0,0 @@ ---- -title: Archive Logs for the Splunk HTTP Event Collector (HEC) -disable_toc: false -aliases: - - /observability_pipelines/archive_logs/splunk_hec/ ---- - -## Overview - -Configure your Splunk HTTP Event Collector (HEC) so that the Observability Pipelines Worker formats the logs collected into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks through the following steps: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Worker over Splunk HEC](#send-logs-to-the-observability-pipelines-worker-over-splunk-hec) - -## Prerequisites - -{{% observability_pipelines/prerequisites/splunk_hec %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select **Splunk HEC** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk HEC address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_hec %}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp.md deleted file mode 100644 index 824b39773aa..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp.md +++ /dev/null @@ -1,459 +0,0 @@ ---- -title: Archive Logs for Splunk Heavy or Universal Forwarders (TCP) -disable_toc: false -aliases: - - /observability_pipelines/archive_logs/splunk_tcp/ ---- - -## Overview - -Configure your Splunk Heavy and Universal Forwarder so that the Observability Pipelines Worker formats the logs collected into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting Splunk Forwarder to the Observability Pipelines Worker](#connect-splunk-forwarder-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/splunk_tcp %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select **Splunk TCP** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_tcp %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk TCP address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_tcp %}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector.md deleted file mode 100644 index 9c5f015654f..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector.md +++ /dev/null @@ -1,459 +0,0 @@ ---- -title: Archive Logs for the Sumo Logic Hosted Collector HTTP Logs Source -disable_toc: false -aliases: - - /observability_pipelines/archive_logs/sumo_logic_hosted_collector/ ---- - -## Overview - -Configure your Sumo Logic Hosted Collector HTTP Logs source so that the Observability Pipelines Worker formats the logs collected into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker over Sumo Logic HTTP Source](#send-logs-to-the-observability-pipelines-worker-over-sumo-logic-http-source) - -## Prerequisites - -{{% observability_pipelines/prerequisites/sumo_logic %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select **Sumo Logic** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/sumo_logic %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Sumo Logic address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/sumo_logic %}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/syslog.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/syslog.md deleted file mode 100644 index bde56732e12..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/syslog.md +++ /dev/null @@ -1,463 +0,0 @@ ---- -title: Archive Logs for Syslog -disable_toc: false -aliases: - - /observability_pipelines/archive_logs/syslog/ ---- - -## Overview - -Configure your rsyslog or syslog-ng source so that the Observability Pipelines Worker formats the logs collected into a Datadog-rehydratable format before routing them to Datadog Log Archives. - -{{% observability_pipelines/use_case_images/archive_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Configuring a Log Archive](#configure-a-log-archive) -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/syslog %}} - -## Configure Log Archives - -If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). - -You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. - -Select the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} - -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Google Cloud Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h4" %}} - -{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} - -{{% /collapse-content %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][4]. -1. Select the **Archive Logs** template to create a new pipeline. -1. Select **rsyslog** or **syslog-ng** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/syslog%}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Syslog address. This is a Syslog-compatible endpoint, exposed by the Worker, that your applications send logs to. The Observability Pipelines Worker listens on this address for incoming logs. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker - -{{% observability_pipelines/log_source_configuration/syslog %}} - -[1]: /integrations/amazon_web_services/#setup -[2]: /integrations/google_cloud_platform/#setup -[3]: /integrations/azure/#setup -[4]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/_index.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/_index.md deleted file mode 100644 index cc445e868c2..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/_index.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Dual Ship Logs -disable_toc: false -aliases: - - /observability_pipelines/dual_ship_logs/ -private: true -cascade: - private: true ---- - -## Overview - -As your infrastructure and your organization scales, so does your log volume, the complexity of your data, and your observability architecture. To optimize how you manage your logs, you might need to experiment with different log management tools and routing workflows. Use Observability Pipelines to send your logs to different destinations, so you can evaluate different tools and workflows with minimal disruption to your production environment. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -Select a source to get started: - -- [Amazon Data Firehose][12] -- [Amazon S3][11] -- [Datadog Agent][1] -- [Fluentd or Fluent Bit][2] -- [Google Pub/Sub][3] -- [HTTP Client][4] -- [HTTP Server][5] -- [Kafka][13] -- [Logstash][6] -- [Splunk HTTP Event Collector (HEC)][7] -- [Splunk Heavy or Universal Forwarders (TCP)][8] -- [Socket (TCP or UDP)][14] -- [Sumo Logic Hosted Collector][9] -- [rsyslog or syslog-ng][10] - -[1]: /observability_pipelines/dual_ship_logs/datadog_agent -[2]: /observability_pipelines/dual_ship_logs/fluent -[3]: /observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub -[4]: /observability_pipelines/dual_ship_logs/http_client -[5]: /observability_pipelines/set_up_pipelines/dual_ship_logs/http_server -[6]: /observability_pipelines/set_up_pipelines/dual_ship_logs/logstash -[7]: /observability_pipelines/dual_ship_logs/splunk_hec -[8]: /observability_pipelines/dual_ship_logs/splunk_tcp -[9]: /observability_pipelines/dual_ship_logs/sumo_logic_hosted_collector -[10]: /observability_pipelines/dual_ship_logs/syslog -[11]: /observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3 -[12]: /observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose -[13]: /observability_pipelines/set_up_pipelines/dual_ship_logs/kafka -[14]: /observability_pipelines/set_up_pipelines/dual_ship_logs/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose.md deleted file mode 100644 index 10e204ae47f..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose.md +++ /dev/null @@ -1,409 +0,0 @@ ---- -title: Dual Ship Logs for Amazon Data Firehose -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to aggregate and process your Amazon Data Firehose logs before routing them to various applications. - - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-amazon_data_firehose) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_data_firehose %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select the **Amazon Data Firehose** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_data_firehose %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Amazon Data Firehose address. The Observability Pipelines Worker listens to this address and port for incoming logs from Amazon Data Firehose. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Amazon Data Firehose - -{{% observability_pipelines/log_source_configuration/amazon_data_firehose %}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3.md deleted file mode 100644 index a86c18f1856..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Dual Ship Logs for Amazon S3 -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to aggregate and process your Amazon S3 logs before routing them to various applications. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_s3 %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select the **Amazon S3** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_s3 %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. In the **AWS S3 SQS URL** field, enter the URL of the SQS queue to which the S3 bucket sends notification events. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/datadog_agent.md deleted file mode 100644 index 8b96674a5a8..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/datadog_agent.md +++ /dev/null @@ -1,423 +0,0 @@ ---- -title: Dual Ship Logs for the Datadog Agent -disable_toc: false -aliases: - - /observability_pipelines/dual_ship_logs/datadog_agent/ ---- - -## Overview - -Configure the Datadog Agent and set up Observability Pipelines so that the Observability Pipelines Worker aggregates and processes the logs coming from your upstream sources before routing them to various applications. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting the Datadog Agent to the Observability Pipelines Worker](#connect-the-datadog-agent-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/datadog_agent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship** template to create a new pipeline. -1. Select **Datadog Agent** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/datadog_agent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the listener address, which is the address and port the Observability Pipelines Worker listens on for incoming logs from the Datadog Agent. For example, `0.0.0.0:`. -1. Provide the environment variables for each of your selected destinations. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Connect the Datadog Agent to the Observability Pipelines Worker - -Use the Agent configuration file or the Agent Helm chart values file to connect the Datadog Agent to the Observability Pipelines Worker. - -{{< tabs >}} -{{% tab "Agent configuration file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent %}} - -{{% /tab %}} -{{% tab "Agent Helm values file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent_kubernetes %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/fluent.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/fluent.md deleted file mode 100644 index 421962c1dc3..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/fluent.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Dual Ship Logs for Fluent -disable_toc: false -aliases: - - /observability_pipelines/dual_ship_logs/fluent/ ---- - -## Overview - -Configure Fluentd or Fluent Bit and set up Observability Pipelines so that the Observability Pipelines Worker aggregates and processes the logs coming from your upstream sources before routing them to various applications. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-fluent) - -## Prerequisites - -{{% observability_pipelines/prerequisites/fluent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select **Fluentd** or **Fluent Bit** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/fluent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Fluent socket address and port. The Observability Pipelines Worker listens on this address for incoming log messages. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Fluent - -{{% observability_pipelines/log_source_configuration/fluent %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub.md deleted file mode 100644 index 2aa1c99c855..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub.md +++ /dev/null @@ -1,402 +0,0 @@ ---- -title: Dual Ship Logs for Google Pub/Sub -disable_toc: false ---- - -## Overview - -Configure Google Pub/Sub and set up Observability Pipelines so that the Observability Pipelines Worker aggregates and processes the logs coming from your upstream sources before routing them to various applications. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/google_pubsub %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select the **Google Pub/Sub** source. - -### Set up the source - -{{% observability_pipelines/source_settings/google_pubsub %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_client.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_client.md deleted file mode 100644 index 18e526e1801..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_client.md +++ /dev/null @@ -1,406 +0,0 @@ ---- -title: Dual Ship Logs for HTTP Client -disable_toc: false -aliases: - - /observability_pipelines/dual_ship_logs/http_client/ ---- - -## Overview - -Use the Observability Pipelines Worker to aggregate and process your HTTP server logs before routing them to various applications. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_client %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select **HTTP Client** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_client %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the full path of the HTTP/S endpoint URL. For example, `https://127.0.0.8/logs`. The Observability Pipelines Worker collects logs events from this endpoint. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_server.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_server.md deleted file mode 100644 index b4de3e63450..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_server.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Dual Ship Logs for HTTP Server -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to aggregate and processes your HTTP client logs before routing them to various applications. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_server %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select the **HTTP Server** source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_server %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the HTTP/S server address, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens to this socket address for your HTTP client logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/kafka.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/kafka.md deleted file mode 100644 index 7aeca8627ad..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/kafka.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -title: Dual Ship Logs for Kafka -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to aggregate and process your logs from Kafka topics before routing them to various applications. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-kafka) - -## Prerequisites - -{{% observability_pipelines/prerequisites/kafka %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select the **Kafka** source. - -### Set up the source - -{{% observability_pipelines/source_settings/kafka %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the host and port of the Kafka bootstrap servers, which clients should use to connect to the Kafka cluster and discover all the other hosts in the cluster. Must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. - - If you enabled SASL, enter the Kafka SASL username and Kafka SASL password. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/logstash.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/logstash.md deleted file mode 100644 index 1bc49cecebc..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/logstash.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Dual Ship Logs for Logstash -disable_toc: false ---- - -## Overview - -Configure Logstash and set up Observability Pipelines so that the Observability Pipelines Worker aggregates and processes the logs coming from your upstream sources before routing them to various applications. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-logstash) - -## Prerequisites - -{{% observability_pipelines/prerequisites/logstash %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select the **Logstash** source. - -### Set up the source - -{{% observability_pipelines/source_settings/logstash %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Logstash address and port, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens on this address for incoming log messages. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Logstash - -{{% observability_pipelines/log_source_configuration/logstash %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/socket.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/socket.md deleted file mode 100644 index d94b0d30d15..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/socket.md +++ /dev/null @@ -1,384 +0,0 @@ ---- -title: Dual Ship Logs for the Socket Source (TCP or UDP) -disable_toc: false ---- - -## Overview - -Send logs over a socket connection to the Observability Pipelines Worker to aggregate, process, and route them to different destinations. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This guide covers: -1. [Prerequisites](#prerequisites) for setting up Observability Pipelines -1. [How to set up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/socket %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select the **Socket** source. - -### Set up the source - -{{% observability_pipelines/source_settings/socket %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the socket address and port, such as `0.0.0.0:9000`. This is the address and port the Observability Pipelines Worker listens on for incoming logs. The socket address must include a port. -1. If you enabled TLS, enter the TLS passphrase. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec.md deleted file mode 100644 index 54a2ea9dfdc..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec.md +++ /dev/null @@ -1,418 +0,0 @@ ---- -title: Dual Ship Logs for the Splunk HTTP Event Collector (HEC) -disable_toc: false -aliases: - - /observability_pipelines/dual_ship_logs/splunk_hec/ ---- - -## Overview - -Configure your Splunk HTTP Event Collectors (HEC) and set up Observability Pipelines so that the Observability Pipelines Worker aggregates and processes the logs coming from your upstream sources before routing them to various applications, including Splunk. - -See [Dual Ship Logs from Splunk TCP][1] if you want to ingest logs from Splunk Heavy or Splunk Universal Forwarders. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following steps to set up dual shipping: - -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Worker over Splunk HEC](#send-logs-to-the-observability-pipelines-worker-over-splunk-hec) - -## Prerequisites - -### Splunk HEC -{{% observability_pipelines/prerequisites/splunk_hec %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][2]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select **Splunk HEC** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk HEC address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_hec %}} - -[1]: /observability_pipelines/dual_ship_logs/splunk_tcp/ -[2]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp.md deleted file mode 100644 index 8d7fee5880f..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp.md +++ /dev/null @@ -1,414 +0,0 @@ ---- -title: Dual Ship Logs for Splunk Heavy or Universal Forwarders (TCP) -disable_toc: false -aliases: - - /observability_pipelines/dual_ship_logs/splunk_tcp/ ---- - -## Overview - -Configure your Splunk Heavy or Universal Forwarders to send logs to the Observability Pipelines Worker to process and route them to downstream destinations. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting Splunk Forwarder to the Observability Pipelines Worker](#connect-splunk-forwarder-to-the-observability-pipelines-worker) - -## Prerequisites - -### Splunk Heavy or Universal Forwarders (TCP) - -{{% observability_pipelines/prerequisites/splunk_tcp %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship** template to create a new pipeline. -1. Select **Splunk TCP** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_tcp %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk TCP address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_tcp %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector.md deleted file mode 100644 index 72e6f048f7b..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector.md +++ /dev/null @@ -1,414 +0,0 @@ ---- -title: Dual Ship Logs for the Sumo Logic Hosted Collector HTTP Logs Source -disable_toc: false -aliases: - - /observability_pipelines/dual_ship_logs/sumo_logic_hosted_collector/ ---- - -## Overview - -Configure Sumo Logic Hosted Collector the HTTP Logs source to send logs to the Observability Pipelines Worker so that Observability Pipelines receives and routes logs from the Sumo Logic Hosted Collector to downstream destinations. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker over Sumo Logic HTTP Source](#send-logs-to-the-observability-pipelines-worker-over-sumo-logic-http-source) - -## Prerequisites - -### Sumo Logic Hosted Collectors - -{{% observability_pipelines/prerequisites/sumo_logic %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship** template to create a new pipeline. -1. Select **Sumo Logic** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Sumo Logic address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/sumo_logic %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/syslog.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/syslog.md deleted file mode 100644 index 6ffe85996f0..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/syslog.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Dual Ship Logs for Syslog -disable_toc: false -aliases: - - /observability_pipelines/dual_ship_logs/syslog/ ---- - -## Overview - -Configure rsyslog or syslog-ng and set up Observability Pipelines so that the Observability Pipelines Worker aggregates and processes the logs coming from your upstream sources before routing them to various applications, including Splunk. - -{{% observability_pipelines/use_case_images/dual_ship_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-syslog) - -## Prerequisites - -{{% observability_pipelines/prerequisites/syslog %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Dual Ship Logs** template to create a new pipeline. -1. Select **rsyslog** or **syslog-ng** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/syslog %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Syslog address. This is a Syslog-compatible endpoint, exposed by the Worker, that your applications send logs to. The Observability Pipelines Worker listens on this address for incoming logs. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Syslog - -{{% observability_pipelines/log_source_configuration/syslog %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/_index.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/_index.md deleted file mode 100644 index 9adb547e0c2..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/_index.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: Generate Metrics -disable_toc: false -private: true -cascade: - private: true ---- - -## Overview - -
The solutions outlined in this documentation are specific to on-premises logging environments. To generate metrics from cloud-based logs, see the Observability Pipelines documentation.
- -Some log sources, such as firewalls and network appliances, generate a large volume of log events that contain data that don't necessarily need to be stored. Often, you just want to see a summary of the logs and compare it to historical data. Use the Generate Metrics template to generate a count metric of logs that match a query or a distribution metric of a numeric value contained in the logs, such as a request duration. The template starts you off with the following processors: - -- **Filter**: Add a query to send only a subset of logs based on your conditions. -- **Grok Parser**: Parse your logs using grok parsing rules that are available for a set of sources or add custom parsing rules. -- **Generate metrics**: Generate metrics for your logs or a subset of them. See [Metrics types](#metrics-types) for the types of metrics you can generate. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -Select a source to get started: - -- [Amazon Data Firehose][12] -- [Amazon S3][11] -- [Datadog Agent][1] -- [Fluentd or Fluent Bit][2] -- [Google Pub/Sub][3] -- [HTTP Client][4] -- [HTTP Server][5] -- [Kafka][13] -- [Logstash][6] -- [Socket][14] -- [Splunk HTTP Event Collector (HEC)][7] -- [Splunk Heavy or Universal Forwarders (TCP)][8] -- [Sumo Logic Hosted Collector][9] -- [rsyslog or syslog-ng][10] - -[1]: /observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent -[2]: /observability_pipelines/set_up_pipelines/generate_metrics/fluent -[3]: /observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub -[4]: /observability_pipelines/set_up_pipelines/generate_metrics/http_client -[5]: /observability_pipelines/set_up_pipelines/generate_metrics/http_server -[6]: /observability_pipelines/set_up_pipelines/generate_metrics/logstash -[7]: /observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec -[8]: /observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp -[9]: /observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector -[10]: /observability_pipelines/set_up_pipelines/generate_metrics/syslog -[11]: /observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3 -[12]: /observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose -[13]: /observability_pipelines/set_up_pipelines/generate_metrics/kafka -[14]: /observability_pipelines/set_up_pipelines/generate_metrics/socket - -## Metrics types - -{{% observability_pipelines/metrics_types %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose.md deleted file mode 100644 index 43dd2552fcf..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Generate Metrics for Amazon Data Firehose -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to generate metrics from your Amazon Data Firehose logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-amazon_data_firehose) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_data_firehose %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select the **Amazon Data Firehose** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_data_firehose %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Amazon Data Firehose address. The Observability Pipelines Worker listens to this address and port for incoming logs from Amazon Data Firehose. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Amazon Data Firehose - -{{% observability_pipelines/log_source_configuration/amazon_data_firehose %}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3.md deleted file mode 100644 index 5aa621e34a1..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Generate Metrics for Amazon S3 -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to generate metrics from your Amazon S3 logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_s3 %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select the **Amazon S3** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_s3 %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. In the **AWS S3 SQS URL** field, enter the URL of the SQS queue to which the S3 bucket sends notification events. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent.md deleted file mode 100644 index 235c33c535a..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent.md +++ /dev/null @@ -1,421 +0,0 @@ ---- -title: Generate Metrics for the Datadog Agent -disable_toc: false ---- - -## Overview - -Configure the Datadog Agent to send logs to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting the Datadog Agent to the Observability Pipelines Worker](#connect-the-datadog-agent-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/datadog_agent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select **Datadog Agent** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/datadog_agent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the listener address, which is the address and port the Observability Pipelines Worker listens on for incoming logs from the Datadog Agent. For example, `0.0.0.0:`. -1. Provide the environment variables for each of your selected destinations. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Connect the Datadog Agent to the Observability Pipelines Worker - -Use the Agent configuration file or the Agent Helm chart values file to connect the Datadog Agent to the Observability Pipelines Worker. - -{{< tabs >}} -{{% tab "Agent configuration file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent %}} - -{{% /tab %}} -{{% tab "Agent Helm values file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent_kubernetes %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/fluent.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/fluent.md deleted file mode 100644 index 3bcc0b371b2..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/fluent.md +++ /dev/null @@ -1,409 +0,0 @@ ---- -title: Generate Metrics for Fluent -disable_toc: false ---- - -## Overview - -Configure Fluentd or Fluent Bit to send logs to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-fluent) - -## Prerequisites - -{{% observability_pipelines/prerequisites/fluent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select **Fluentd** or **Fluent Bit** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/fluent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Fluent socket address and port. The Observability Pipelines Worker listens on this address for incoming log messages. - -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Fluent - -{{% observability_pipelines/log_source_configuration/fluent %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub.md deleted file mode 100644 index e29e806537f..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub.md +++ /dev/null @@ -1,402 +0,0 @@ ---- -title: Generate Metrics for Google Pub/Sub -disable_toc: false ---- - -## Overview - -Send Google Pub/Sub logs to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/google_pubsub %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select the **Google Pub/Sub** source. - -### Set up the source - -{{% observability_pipelines/source_settings/google_pubsub %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_client.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_client.md deleted file mode 100644 index 3a1238326ed..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_client.md +++ /dev/null @@ -1,404 +0,0 @@ ---- -title: Generate Metrics for HTTP Client -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to generate metrics from your HTTP server logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_client %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select **HTTP Client** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_client %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the full path of the HTTP/S endpoint URL, such as `https://127.0.0.8/logs`. The Observability Pipelines Worker collects logs events from this endpoint. - -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_server.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_server.md deleted file mode 100644 index 83b9bfb40c9..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_server.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Generate Metrics for HTTP Server -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to generate metrics from your HTTP client logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_server %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select the **HTTP Server** source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_server %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the HTTP/S server address, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens to this socket address for your HTTP client logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/kafka.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/kafka.md deleted file mode 100644 index 2cd16b50217..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/kafka.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -title: Generate Metrics for Kafka -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to generate metrics for your logs from Kafka topics. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-kafka) - -## Prerequisites - -{{% observability_pipelines/prerequisites/kafka %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select the **Kafka** source. - -### Set up the source - -{{% observability_pipelines/source_settings/kafka %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the host and port of the Kafka bootstrap servers, which clients should use to connect to the Kafka cluster and discover all the other hosts in the cluster. Must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. - - If you enabled SASL, enter the Kafka SASL username and Kafka SASL password. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/logstash.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/logstash.md deleted file mode 100644 index 1e6a7ff22e4..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/logstash.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Generate Metrics for Logstash -disable_toc: false ---- - -## Overview - -Configure Logstash to send logs to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-logstash) - -## Prerequisites - -{{% observability_pipelines/prerequisites/logstash %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select the **Logstash** source. - -### Set up the source - -{{% observability_pipelines/source_settings/logstash %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Logstash address and port, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens on this address for incoming log messages. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Logstash - -{{% observability_pipelines/log_source_configuration/logstash %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/socket.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/socket.md deleted file mode 100644 index 60bae7020f0..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/socket.md +++ /dev/null @@ -1,384 +0,0 @@ ---- -title: Generate Metrics for the Socket Source (TCP or UDP) -disable_toc: false ---- - -## Overview - -Send logs over a socket connection to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/socket %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select the **Socket** source. - -### Set up the source - -{{% observability_pipelines/source_settings/socket %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the socket address and port, such as `0.0.0.0:9000`. This is the address and port the Observability Pipelines Worker listens on for incoming logs. The socket address must include a port. -1. If you enabled TLS, enter the TLS passphrase. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec.md deleted file mode 100644 index 49c9116818f..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec.md +++ /dev/null @@ -1,414 +0,0 @@ ---- -title: Generate Metrics for the Splunk HTTP Event Collector (HEC) -disable_toc: false ---- - -## Overview - -Configure your Splunk HTTP Event Collectors (HEC) to send logs to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following steps to set up dual shipping: - -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Worker over Splunk HEC](#send-logs-to-the-observability-pipelines-worker-over-splunk-hec) - -## Prerequisites - -### Splunk HEC -{{% observability_pipelines/prerequisites/splunk_hec %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][2]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select **Splunk HEC** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk HEC address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_hec %}} - -[1]: /observability_pipelines/dual_ship_logs/splunk_tcp/ -[2]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp.md deleted file mode 100644 index 4965682e50e..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp.md +++ /dev/null @@ -1,412 +0,0 @@ ---- -title: Generate Metrics for Splunk Heavy or Universal Forwarders (TCP) -disable_toc: false ---- - -## Overview - -Configure your Splunk Heavy or Universal Forwarders to send logs to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting Splunk Forwarder to the Observability Pipelines Worker](#connect-splunk-forwarder-to-the-observability-pipelines-worker) - -## Prerequisites - -### Splunk Heavy or Universal Forwarders (TCP) - -{{% observability_pipelines/prerequisites/splunk_tcp %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select **Splunk TCP** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_tcp %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk TCP address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_tcp %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector.md deleted file mode 100644 index 383f0066fcc..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector.md +++ /dev/null @@ -1,412 +0,0 @@ ---- -title: Generate Metrics for the Sumo Logic Hosted Collector HTTP Logs Source -disable_toc: false ---- - -## Overview - -Configure your Sumo Logic Hosted Collector HTTP Logs source to send logs to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker over Sumo Logic HTTP Source](#send-logs-to-the-observability-pipelines-worker-over-sumo-logic-http-source) - -## Prerequisites - -### Sumo Logic Hosted Collectors - -{{% observability_pipelines/prerequisites/sumo_logic %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select **Sumo Logic** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Sumo Logic address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/sumo_logic %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/syslog.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/syslog.md deleted file mode 100644 index c3dce62064a..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/syslog.md +++ /dev/null @@ -1,409 +0,0 @@ ---- -title: Generate Metrics for Syslog -disable_toc: false ---- - -## Overview - -Configure rsyslog or syslog-ng to send logs to the Observability Pipelines Worker so that you can generate metrics from those logs. - -{{% observability_pipelines/use_case_images/generate_metrics %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-syslog) - -## Prerequisites - -{{% observability_pipelines/prerequisites/syslog %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Generate Metrics** template to create a new pipeline. -1. Select **rsyslog** or **syslog-ng** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/syslog %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Syslog address. This is a Syslog-compatible endpoint, exposed by the Worker, that your applications send logs to. The Observability Pipelines Worker listens on this address for incoming logs. - -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Syslog - -{{% observability_pipelines/log_source_configuration/syslog %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/_index.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/_index.md deleted file mode 100644 index a3a184ae566..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/_index.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Log Enrichment -disable_toc: false -aliases: - - /observability_pipelines/log_enrichment/ -private: true -cascade: - private: true ---- - -## Overview - - As your organization grows, the logs from your services, systems, and applications grow in volume and complexity. To manage these logs, you might need to standardize their format and add information to make it easier to search and analyze them. For example, each log source has its own unique format. This can make it difficult to search and analyze during investigations if they have not been reformatted and standardized. You could also have additional information, such as customer IDs or IP addresses, that you want to add to your logs. Use the Log Enrichment Template and these Observability Pipelines processors to enrich and transform your logs: - -- **Enrichment Table**: Enrich your logs with information from a reference table, which could be a local file or a GeoIP database. -- **Grok Parser**: Parse your logs using grok parsing rules that are available for a set of sources. -- **Add hostname**: Add the name of the host that sent the log so you can use it to find the root cause of an issue. -- **Parse JSON**: Convert fields into JSON objects. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -Select a source to get started: - -- [Amazon Data Firehose][12] -- [Amazon S3][11] -- [Datadog Agent][1] -- [Fluentd or Fluent Bit][2] -- [Google Pub/Sub][3] -- [HTTP Client][4] -- [HTTP Server][5] -- [Kafka][13] -- [Logstash][6] -- [Splunk HTTP Event Collector (HEC)][7] -- [Splunk Heavy or Universal Forwarders (TCP)][8] -- [Socket (TCP or UDP)][14] -- [Sumo Logic Hosted Collector][9] -- [rsyslog or syslog-ng][10] - -[1]: /observability_pipelines/log_enrichment/datadog_agent -[2]: /observability_pipelines/log_enrichment/fluent -[3]: /observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub -[4]: /observability_pipelines/log_enrichment/http_client -[5]: /observability_pipelines/set_up_pipelines/log_enrichment/http_server -[6]: /observability_pipelines/set_up_pipelines/log_enrichment/logstash -[7]: /observability_pipelines/log_enrichment/splunk_hec -[8]: /observability_pipelines/log_enrichment/splunk_tcp -[9]: /observability_pipelines/log_enrichment/sumo_logic_hosted_collector -[10]: /observability_pipelines/log_enrichment/syslog -[11]: /observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3 -[12]: /observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose -[13]: /observability_pipelines/set_up_pipelines/log_enrichment/kafka -[14]: /observability_pipelines/set_up_pipelines/log_enrichment/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose.md deleted file mode 100644 index 4d5a0aaecba..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Log Enrichment for Amazon Data Firehose -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to enrich and transform your Amazon Data Firehose logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-amazon_data_firehose) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_data_firehose %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Enrichment** template to create a new pipeline. -1. Select the **Amazon Data Firehose** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_data_firehose %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Amazon Data Firehose address. The Observability Pipelines Worker listens to this address and port for incoming logs from Amazon Data Firehose. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Amazon Data Firehose - -{{% observability_pipelines/log_source_configuration/amazon_data_firehose %}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3.md deleted file mode 100644 index 1b315e44678..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Log Enrichment for Amazon S3 -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to enrich and transform your Amazon S3 logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_s3 %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Enrichment** template to create a new pipeline. -1. Select the **Amazon S3** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_s3 %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. In the **AWS S3 SQS URL** field, enter the URL of the SQS queue to which the S3 bucket sends notification events. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/datadog_agent.md deleted file mode 100644 index b5df526684e..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/datadog_agent.md +++ /dev/null @@ -1,423 +0,0 @@ ---- -title: Log Enrichment for the Datadog Agent -disable_toc: false -aliases: - - /observability_pipelines/log_enrichment/datadog_agent/ ---- - -## Overview - -Configure your Datadog Agent to send logs to the Observability Pipelines Worker and enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting the Datadog Agent to the Observability Pipelines Worker](#connect-the-datadog-agent-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/datadog_agent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select **Datadog Agent** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/datadog_agent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the listener address, which is the address and port the Observability Pipelines Worker listens on for incoming logs from the Datadog Agent. For example, `0.0.0.0:`. -1. Provide the environment variables for each of your selected destinations. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Connect the Datadog Agent to the Observability Pipelines Worker - -Use the Agent configuration file or the Agent Helm chart values file to connect the Datadog Agent to the Observability Pipelines Worker. - -{{< tabs >}} -{{% tab "Agent configuration file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent %}} - -{{% /tab %}} -{{% tab "Agent Helm values file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent_kubernetes %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/fluent.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/fluent.md deleted file mode 100644 index a607aa7f3ed..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/fluent.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Log Enrichment for Fluent -disable_toc: false -aliases: - - /observability_pipelines/log_enrichment/fluent/ ---- - -## Overview - -Configure Fluentd or Fluent Bit to send logs to the Observability Pipelines Worker and enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-fluent) - -## Prerequisites - -{{% observability_pipelines/prerequisites/fluent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **Fluentd** or **Fluent Bit** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/fluent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Fluent socket address and port. The Observability Pipelines Worker listens on this address for incoming log messages. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Fluent - -{{% observability_pipelines/log_source_configuration/fluent %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub.md deleted file mode 100644 index ecd1d1b58de..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub.md +++ /dev/null @@ -1,402 +0,0 @@ ---- -title: Log Enrichment for Google Pub/Sub -disable_toc: false ---- - -## Overview - -Configure your Google Pub/Sub to send logs to the Observability Pipelines Worker and enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/google_pubsub %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Enrichment** template to create a new pipeline. -1. Select the **Google Pub/Sub** source. - -### Set up the source - -{{% observability_pipelines/source_settings/google_pubsub %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_client.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_client.md deleted file mode 100644 index 881d7d62a81..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_client.md +++ /dev/null @@ -1,406 +0,0 @@ ---- -title: Log Enrichment for HTTP Client -disable_toc: false -aliases: - - /observability_pipelines/log_enrichment/http_client/ ---- - -## Overview - -Use the Observability Pipelines Worker to enrich and transform your HTTP server logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_client %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **HTTP Client** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_client %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the full path of the HTTP/S endpoint URL. For example, `https://127.0.0.8/logs`. The Observability Pipelines Worker collects logs events from this endpoint. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_server.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_server.md deleted file mode 100644 index f19fd94da08..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_server.md +++ /dev/null @@ -1,404 +0,0 @@ ---- -title: Log Enrichment for HTTP Server -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to enrich and transform your HTTP client logs before routing them to their destination. - - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_server %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Enrichment** template to create a new pipeline. -1. Select the **HTTP Server** source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_server %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the HTTP/S server address, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens to this socket address for your HTTP client logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/kafka.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/kafka.md deleted file mode 100644 index 9638b8519fd..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/kafka.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -title: Log Enrichment for Kafka -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to enrich and transform logs from your Kafka topics before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-kafka) - -## Prerequisites - -{{% observability_pipelines/prerequisites/kafka %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Enrichment** template to create a new pipeline. -1. Select the **Kafka** source. - -### Set up the source - -{{% observability_pipelines/source_settings/kafka %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the host and port of the Kafka bootstrap servers, which clients should use to connect to the Kafka cluster and discover all the other hosts in the cluster. Must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. - - If you enabled SASL, enter the Kafka SASL username and Kafka SASL password. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/logstash.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/logstash.md deleted file mode 100644 index bfdaf855dab..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/logstash.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Log Enrichment for Logstash -disable_toc: false ---- - -## Overview - -Configure your Logstash to send logs to the Observability Pipelines Worker and enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-logstash) - -## Prerequisites - -{{% observability_pipelines/prerequisites/logstash %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Enrichment** template to create a new pipeline. -1. Select the **Logstash** source. - -### Set up the source - -{{% observability_pipelines/source_settings/logstash %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Logstash address and port, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens on this address for incoming log messages. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Logstash - -{{% observability_pipelines/log_source_configuration/logstash %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/socket.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/socket.md deleted file mode 100644 index 1d2757b4129..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/socket.md +++ /dev/null @@ -1,384 +0,0 @@ ---- -title: Log Enrichment for the Socket Source (TCP or UDP) -disable_toc: false ---- - -## Overview - -Send logs over a socket connection to the Observability Pipelines Worker so you can enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/socket %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Enrichment** template to create a new pipeline. -1. Select the **Socket** source. - -### Set up the source - -{{% observability_pipelines/source_settings/socket %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the socket address and port, such as `0.0.0.0:9000`. This is the address and port the Observability Pipelines Worker listens on for incoming logs. The socket address must include a port. -1. If you enabled TLS, enter the TLS passphrase. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec.md deleted file mode 100644 index 38f0af3bd44..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec.md +++ /dev/null @@ -1,409 +0,0 @@ ---- -title: Log Enrichment for the Splunk HTTP Event Collector (HEC) -disable_toc: false -aliases: - - /observability_pipelines/log_enrichment/splunk_hec/ ---- - -## Overview - -Configure your Splunk HTTP Event Collectors (HEC) to send logs to the Observability Pipelines Worker and enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Worker over Splunk HEC](#send-logs-to-the-observability-pipelines-worker-over-splunk-hec) - -## Prerequisites - -{{% observability_pipelines/prerequisites/splunk_hec %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **Splunk HEC** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk HEC address. This is the address and port where your applications are sending their logging data to. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_hec %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp.md deleted file mode 100644 index ff34c1bf3ef..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Log Enrichment for Splunk Heavy or Universal Forwarders (TCP) -disable_toc: false -aliases: - - /observability_pipelines/log_enrichment/splunk_tcp/ ---- - -## Overview - -Configure your Splunk Heavy or Universal Forwarders to send logs to the Observability Pipelines Worker and enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting Splunk Forwarder to the Observability Pipelines Worker](#connect-splunk-forwarder-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/splunk_tcp %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume** template to create a new pipeline. -1. Select **Splunk TCP** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_tcp %}} - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk TCP address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_tcp %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector.md deleted file mode 100644 index 184082ba045..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector.md +++ /dev/null @@ -1,404 +0,0 @@ ---- -title: Log Enrichment for the Sumo Logic Hosted Collector HTTP Logs Source -disable_toc: false -aliases: - - /observability_pipelines/log_enrichment/sumo_logic_hosted_collector/ ---- - -## Overview - -Configure your Sumo Logic Hosted Collector the HTTP Logs source to send logs to the Observability Pipelines Worker and enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker over Sumo Logic HTTP Source](#send-logs-to-the-observability-pipelines-worker-over-sumo-logic-http-source) - -## Prerequisites - -{{% observability_pipelines/prerequisites/sumo_logic %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **Sumo Logic** as the source. - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Sumo Logic address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/sumo_logic %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/syslog.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/syslog.md deleted file mode 100644 index 0d3fb60a508..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/syslog.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Log Enrichment for Syslog -disable_toc: false -aliases: - - /observability_pipelines/log_enrichment/syslog/ ---- - -## Overview - -Configure rsyslog or syslog-ng to send logs to the Observability Pipelines Worker and enrich and transform your logs before routing them to their destination. - -{{% observability_pipelines/use_case_images/log_enrichment %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-syslog) - -## Prerequisites - -{{% observability_pipelines/prerequisites/syslog %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **rsyslog** or **syslog-ng** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/syslog %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Syslog address. This is a Syslog-compatible endpoint, exposed by the Worker, that your applications send logs to. The Observability Pipelines Worker listens on this address for incoming logs. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Syslog - -{{% observability_pipelines/log_source_configuration/syslog %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/_index.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/_index.md deleted file mode 100644 index dbb594611b9..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/_index.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: Log Volume Control -disable_toc: false -aliases: - - /observability_pipelines/log_volume_control/ -further_reading: -- link: "/observability_pipelines/guide/strategies_for_reducing_log_volume/" - tag: "documentation" - text: "Strategies for Reducing Log Volume" -private: true -cascade: - private: true ---- - -## Overview - -As your infrastructure and applications grow, so does your log volume and the complexity of the data. A large volume of logs can introduce a lot of noise and make it difficult to analyze and troubleshoot logs. Use Observability Pipelines' processors to decide which logs are valuable and which ones are noisy and uninteresting, before sending your logs to their destinations. You can use the following processors in the Observability Pipeline Worker to manage your logs: - -- **Filter**: Add a query to send only a subset of logs based on your conditions. -- **Sample**: Define a sampling rate to send only a subset of your logs. -- **Quota**: Enforce daily limits on either the volume of log data or the number of log events. -- **Dedupe**: Drop duplicate copies of your logs, for example, due to retries because of network issues. -- **Remap**: Add, drop, or rename a field in your logs. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -Select a log source to get started: - -- [Amazon Data Firehose][12] -- [Amazon S3][11] -- [Datadog Agent][1] -- [Fluentd or Fluent Bit][2] -- [Google Pub/Sub][3] -- [HTTP Client][4] -- [HTTP Server][5] -- [Kafka][13] -- [Logstash][6] -- [Splunk HTTP Event Collector (HEC)][7] -- [Splunk Heavy or Universal Forwarders (TCP)][8] -- [Socket (TCP or UDP)][14] -- [Sumo Logic Hosted Collector][9] -- [rsyslog or syslog-ng][10] - -## Further reading - -{{< partial name="whats-next/whats-next.html" >}} - -[1]: /observability_pipelines/log_volume_control/datadog_agent -[2]: /observability_pipelines/log_volume_control/fluent -[3]: /observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub -[4]: /observability_pipelines/log_volume_control/http_client -[5]: /observability_pipelines/set_up_pipelines/log_volume_control/http_server -[6]: /observability_pipelines/set_up_pipelines/log_volume_control/logstash -[7]: /observability_pipelines/log_volume_control/splunk_hec -[8]: /observability_pipelines/log_volume_control/splunk_tcp -[9]: /observability_pipelines/log_volume_control/sumo_logic_hosted_collector -[10]: /observability_pipelines/log_volume_control/syslog -[11]: /observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3 -[12]: /observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose -[13]: /observability_pipelines/set_up_pipelines/log_volume_control/kafka -[14]: /observability_pipelines/set_up_pipelines/log_volume_control/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose.md deleted file mode 100644 index d8fe0baec31..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Log Volume Control for Amazon Data Firehose -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to only route useful Amazon Data Firehose logs to their destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-amazon_data_firehose) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_data_firehose %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select the **Amazon Data Firehose** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_data_firehose %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Amazon Data Firehose address. The Observability Pipelines Worker listens to this address and port for incoming logs from Amazon Data Firehose. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Amazon Data Firehose - -{{% observability_pipelines/log_source_configuration/amazon_data_firehose %}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3.md deleted file mode 100644 index 73caf136a74..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Log Volume Control for Amazon S3 -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to only route useful Amazon S3 logs to their destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_s3 %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select the **Amazon S3** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_s3 %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. In the **AWS S3 SQS URL** field, enter the URL of the SQS queue to which the S3 bucket sends notification events. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/datadog_agent.md deleted file mode 100644 index 41321a1b2ba..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/datadog_agent.md +++ /dev/null @@ -1,423 +0,0 @@ ---- -title: Log Volume Control for the Datadog Agent -disable_toc: false -aliases: - - /observability_pipelines/log_volume_control/datadog_agent/ ---- - -## Overview - -Set up the Observability Pipelines Worker with the Datadog Agent source so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting the Datadog Agent to the Observability Pipelines Worker](#connect-the-datadog-agent-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/datadog_agent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **Datadog Agent** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/datadog_agent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the listener address, which is the address and port the Observability Pipelines Worker listens on for incoming logs from the Datadog Agent. For example, `0.0.0.0:`. -1. Provide the environment variables for each of your selected destinations. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Connect the Datadog Agent to the Observability Pipelines Worker - -Use the Agent configuration file or the Agent Helm chart values file to connect the Datadog Agent to the Observability Pipelines Worker. - -{{< tabs >}} -{{% tab "Agent configuration file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent %}} - -{{% /tab %}} -{{% tab "Agent Helm values file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent_kubernetes %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/fluent.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/fluent.md deleted file mode 100644 index 2f860280d4a..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/fluent.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Log Volume Control for Fluent -disable_toc: false -aliases: - - /observability_pipelines/log_volume_control/fluent/ ---- - -## Overview - -Set up the Observability Pipelines Worker with the Fluentd or Fluent Bit source so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-fluent) - -## Prerequisites - -{{% observability_pipelines/prerequisites/fluent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **Fluentd** or **Fluent Bit** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/fluent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Fluent socket address and port. The Observability Pipelines Worker listens on this address for incoming log messages. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Fluent - -{{% observability_pipelines/log_source_configuration/fluent %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub.md deleted file mode 100644 index 8f7885afc50..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub.md +++ /dev/null @@ -1,402 +0,0 @@ ---- -title: Log Volume Control for Google Pub/Sub -disable_toc: false ---- - -## Overview - -Set up the Observability Pipelines Worker with the Google Pub/Sub source so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/google_pubsub %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select the **Google Pub/Sub** source. - -### Set up the source - -{{% observability_pipelines/source_settings/google_pubsub %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_client.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_client.md deleted file mode 100644 index d6347642a6c..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_client.md +++ /dev/null @@ -1,406 +0,0 @@ ---- -title: Log Volume Control for HTTP Client -disable_toc: false -aliases: - - /observability_pipelines/log_volume_control/http_client/ ---- - -## Overview - -Use the Observability Pipelines Worker to only route useful HTTP server logs to their destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_client %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **HTTP Client** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_client %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the full path of the HTTP/S endpoint URL. For example, `https://127.0.0.8/logs`. The Observability Pipelines Worker collects logs events from this endpoint. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_server.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_server.md deleted file mode 100644 index 43317842e57..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_server.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Log Volume Control for HTTP Server -disable_toc: false ---- - -## Overview - -Set up the Observability Pipelines Worker to only route useful HTTP Client logs to their destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_server %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select the **HTTP Server** source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_server%}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the HTTP/S server address, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens to this socket address for your HTTP client logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/kafka.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/kafka.md deleted file mode 100644 index b20e4d283bf..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/kafka.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -title: Log Volume Control for Kafka -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to only route useful logs from your Kafka topics to their destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-kafka) - -## Prerequisites - -{{% observability_pipelines/prerequisites/kafka %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select the **Kafka** source. - -### Set up the source - -{{% observability_pipelines/source_settings/kafka %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the host and port of the Kafka bootstrap servers, which clients should use to connect to the Kafka cluster and discover all the other hosts in the cluster. Must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. - - If you enabled SASL, enter the Kafka SASL username and Kafka SASL password. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/logstash.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/logstash.md deleted file mode 100644 index cc0495670f0..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/logstash.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Log Volume Control for Logstash -disable_toc: false ---- - -## Overview - -Set up the Observability Pipelines Worker with the Logstash source so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-logstash) - -## Prerequisites - -{{% observability_pipelines/prerequisites/logstash %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select the **Logstash** source. - -### Set up the source - -{{% observability_pipelines/source_settings/logstash %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Logstash address and port, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens on this address for incoming log messages. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Logstash - -{{% observability_pipelines/log_source_configuration/logstash %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/socket.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/socket.md deleted file mode 100644 index 5cc75ad934a..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/socket.md +++ /dev/null @@ -1,384 +0,0 @@ ---- -title: Log Volume Control for the Socket Source (TCP or UDP) -disable_toc: false ---- - -## Overview - -Send your logs over a socket connection to the Observability Pipelines Worker so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/socket %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select the **Socket** source. - -### Set up the source - -{{% observability_pipelines/source_settings/socket %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the socket address and port, such as `0.0.0.0:9000`. This is the address and port the Observability Pipelines Worker listens on for incoming logs. The socket address must include a port. -1. If you enabled TLS, enter the TLS passphrase. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec.md deleted file mode 100644 index d0cf8d7704d..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec.md +++ /dev/null @@ -1,409 +0,0 @@ ---- -title: Log Volume Control for the Splunk HTTP Event Collector (HEC) -disable_toc: false -aliases: - - /observability_pipelines/log_volume_control/splunk_hec/ ---- - -## Overview - -Set up the Observability Pipelines Worker with the Splunk HTTP Event Collector (HEC) source so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Worker over Splunk HEC](#send-logs-to-the-observability-pipelines-worker-over-splunk-hec) - -## Prerequisites - -{{% observability_pipelines/prerequisites/splunk_hec %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **Splunk HEC** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk HEC address. This is the address and port where your applications are sending their logging data to. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_hec %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp.md deleted file mode 100644 index 9f412cbb2c6..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Log Volume Control for Splunk Heavy or Universal Forwarders (TCP) -disable_toc: false -aliases: - - /observability_pipelines/log_volume_control/splunk_tcp/ ---- - -## Overview - -This document walks you through the following steps to set up the Observability Pipelines Worker with Splunk Heavy or Universal Forwarders so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting Splunk Forwarder to the Observability Pipelines Worker](#connect-splunk-forwarder-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/splunk_tcp %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume** template to create a new pipeline. -1. Select **Splunk TCP** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_tcp %}} - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk TCP address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_tcp %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector.md deleted file mode 100644 index 5e331974c88..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector.md +++ /dev/null @@ -1,404 +0,0 @@ ---- -title: Log Volume Control for the Sumo Logic Hosted Collector HTTP Logs Source -disable_toc: false -aliases: - - /observability_pipelines/log_volume_control/sumo_logic_hosted_collector/ ---- - -## Overview - -This document walks you through the following steps to set up the Observability Pipelines Worker with the Sumo Logic Hosted Collector HTTP Logs source so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker over Sumo Logic HTTP Source](#send-logs-to-the-observability-pipelines-worker-over-sumo-logic-http-source) - -## Prerequisites - -{{% observability_pipelines/prerequisites/sumo_logic %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **Sumo Logic** as the source. - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Sumo Logic address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/sumo_logic %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/syslog.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/syslog.md deleted file mode 100644 index ca24ef12d0a..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/syslog.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Log Volume Control for Syslog -disable_toc: false -aliases: - - /observability_pipelines/log_volume_control/syslog/ ---- - -## Overview - -Set up the Observability Pipelines Worker with the rsyslog or syslog-ng source so that you only route useful logs to your destinations. - -{{% observability_pipelines/use_case_images/log_volume_control %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-syslog) - -## Prerequisites - -{{% observability_pipelines/prerequisites/syslog %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Log Volume Control** template to create a new pipeline. -1. Select **rsyslog** or **syslog-ng** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/syslog %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Syslog address. This is a Syslog-compatible endpoint, exposed by the Worker, that your applications send logs to. The Observability Pipelines Worker listens on this address for incoming logs. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Syslog - -{{% observability_pipelines/log_source_configuration/syslog %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/_index.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/_index.md deleted file mode 100644 index c41f0a66fbd..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/_index.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Sensitive Data Redaction -disable_toc: false -aliases: - - /observability_pipelines/sensitive_data_redaction/ -private: true -cascade: - private: true ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use the Observability Pipelines Worker to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. Or, create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -Select a log source to get started: - -- [Amazon Data Firehose][12] -- [Amazon S3][11] -- [Datadog Agent][1] -- [Fluentd or Fluent Bit][2] -- [Google Pub/Sub][3] -- [HTTP Client][4] -- [HTTP Server][5] -- [Kafka][13] -- [Logstash][6] -- [Splunk HTTP Event Collector (HEC)][7] -- [Splunk Heavy or Universal Forwarders (TCP)][8] -- [Socket (TCP or UDP)][14] -- [Sumo Logic Hosted Collector][9] -- [rsyslog or syslog-ng][10] - -[1]: /observability_pipelines/sensitive_data_redaction/datadog_agent -[2]: /observability_pipelines/sensitive_data_redaction/fluent -[3]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub -[4]: /observability_pipelines/sensitive_data_redaction/http_client -[5]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server -[6]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash -[7]: /observability_pipelines/sensitive_data_redaction/splunk_hec -[8]: /observability_pipelines/sensitive_data_redaction/splunk_tcp -[9]: /observability_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector -[10]: /observability_pipelines/sensitive_data_redaction/syslog -[11]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3 -[12]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose -[13]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka -[14]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose.md deleted file mode 100644 index d4163945d5b..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose.md +++ /dev/null @@ -1,420 +0,0 @@ ---- -title: Sensitive Data Redaction for Amazon Data Firehose -disable_toc: false ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-amazon_data_firehose) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_data_firehose %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select the **Amazon Data Firehose** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_data_firehose %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Amazon Data Firehose address. The Observability Pipelines Worker listens to this address and port for incoming logs from Amazon Data Firehose. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Amazon EKS" %}} - -{{% observability_pipelines/install_worker/amazon_eks %}} - -{{% /tab %}} -{{% tab "Azure AKS" %}} - -{{% observability_pipelines/install_worker/azure_aks %}} - -{{% /tab %}} -{{% tab "Google GKE" %}} - -{{% observability_pipelines/install_worker/google_gke %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Amazon Data Firehose - -{{% observability_pipelines/log_source_configuration/amazon_data_firehose %}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3.md deleted file mode 100644 index cbb3866846c..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3.md +++ /dev/null @@ -1,405 +0,0 @@ ---- -title: Sensitive Data Redaction for Amazon S3 -disable_toc: false ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_s3 %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select the **Amazon S3** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_s3 %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. In the **AWS S3 SQS URL** field, enter the URL of the SQS queue to which the S3 bucket sends notification events. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/datadog_agent.md deleted file mode 100644 index 9f04f4ec51d..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/datadog_agent.md +++ /dev/null @@ -1,425 +0,0 @@ ---- -title: Sensitive Data Redaction for the Datadog Agent -disable_toc: false -aliases: - - /observability_pipelines/sensitive_data_redaction/datadog_agent/ ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. Or, create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting the Datadog Agent to the Observability Pipelines Worker](#connect-the-datadog-agent-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/datadog_agent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redaction** template to create a new pipeline. -1. Select **Datadog Agent** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/datadog_agent %}} - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the listener address, which is the address and port the Observability Pipelines Worker listens on for incoming logs from the Datadog Agent. For example, `0.0.0.0:`. -1. Provide the environment variables for each of your selected destinations. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Connect the Datadog Agent to the Observability Pipelines Worker - -Use the Agent configuration file or the Agent Helm chart values file to connect the Datadog Agent to the Observability Pipelines Worker. - -{{< tabs >}} -{{% tab "Agent configuration file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent %}} - -{{% /tab %}} -{{% tab "Agent Helm values file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent_kubernetes %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/fluent.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/fluent.md deleted file mode 100644 index 164ceaddc19..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/fluent.md +++ /dev/null @@ -1,413 +0,0 @@ ---- -title: Sensitive Data Redaction for Fluent -disable_toc: false -aliases: - - /observability_pipelines/sensitive_data_redaction/fluent/ ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-fluent) - -## Prerequisites - -{{% observability_pipelines/prerequisites/fluent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select **Fluentd** or **Fluent Bit** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/fluent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Fluent socket address and port. The Observability Pipelines Worker listens on this address for incoming log messages. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Fluent - -{{% observability_pipelines/log_source_configuration/fluent %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub.md deleted file mode 100644 index 617f619c9c7..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub.md +++ /dev/null @@ -1,404 +0,0 @@ ---- -title: Sensitive Data Redaction for Google Pub/Sub -disable_toc: false ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/google_pubsub %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select the **Google Pub/Sub** source. - -### Set up the source - -{{% observability_pipelines/source_settings/google_pubsub %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_client.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_client.md deleted file mode 100644 index e9ed3dd708d..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_client.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Sensitive Data Redaction for HTTP Client -disable_toc: false -aliases: - - /observability_pipelines/sensitive_data_redaction/http_client/ ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_client %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select **HTTP Client** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_client %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the full path of the HTTP/S endpoint URL. For example, `https://127.0.0.8/logs`. The Observability Pipelines Worker collects logs events from this endpoint. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server.md deleted file mode 100644 index a9edb03868e..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server.md +++ /dev/null @@ -1,405 +0,0 @@ ---- -title: Sensitive Data Redaction for HTTP Server -disable_toc: false ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. Or, create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_server %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select the **HTTP Server** source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_server %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the HTTP/S server address, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens to this socket address for your HTTP client logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka.md deleted file mode 100644 index b5ccc393818..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka.md +++ /dev/null @@ -1,409 +0,0 @@ ---- -title: Sensitive Data Redaction for Kafka -disable_toc: false ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-kafka) - -## Prerequisites - -{{% observability_pipelines/prerequisites/kafka %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select the **Kafka** source. - -### Set up the source - -{{% observability_pipelines/source_settings/kafka %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the host and port of the Kafka bootstrap servers, which clients should use to connect to the Kafka cluster and discover all the other hosts in the cluster. Must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. - - If you enabled SASL, enter the Kafka SASL username and Kafka SASL password. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash.md deleted file mode 100644 index a2849f383c2..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash.md +++ /dev/null @@ -1,410 +0,0 @@ ---- -title: Sensitive Data Redaction for Logstash -disable_toc: false ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. Or, create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-logstash) - -## Prerequisites - -{{% observability_pipelines/prerequisites/logstash %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select the **Logstash** source. - -### Set up the source - -{{% observability_pipelines/source_settings/logstash %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Logstash address and port, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens on this address for incoming log messages. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Logstash - -{{% observability_pipelines/log_source_configuration/logstash %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket.md deleted file mode 100644 index 2f301eded4c..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket.md +++ /dev/null @@ -1,386 +0,0 @@ ---- -title: Sensitive Data Redaction for the Socket Source (TCP or UDP) -disable_toc: false ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/socket %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select the **Socket** source. - -### Set up the source - -{{% observability_pipelines/source_settings/socket %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the socket address and port, such as `0.0.0.0:9000`. This is the address and port the Observability Pipelines Worker listens on for incoming logs. The socket address must include a port. -1. If you enabled TLS, enter the TLS passphrase. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec.md deleted file mode 100644 index fca1b3fcbe6..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Sensitive Data Redaction for the Splunk HTTP Event Collector (HEC) -disable_toc: false -aliases: - - /observability_pipelines/sensitive_data_redaction/splunk_hec/ ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. Or, create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks through the following steps: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Worker over Splunk HEC](#send-logs-to-the-observability-pipelines-worker-over-splunk-hec) - -## Prerequisites - -{{% observability_pipelines/prerequisites/splunk_hec %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redaction** template to create a new pipeline. -1. Select **Splunk HEC** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk HEC address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_hec %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp.md deleted file mode 100644 index f357af4c06b..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp.md +++ /dev/null @@ -1,410 +0,0 @@ ---- -title: Sensitive Data Redaction for Splunk Heavy or Universal Forwarders (TCP) -disable_toc: false -aliases: - - /observability_pipelines/sensitive_data_redaction/splunk_tcp/ ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. Or, create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting Splunk Forwarder to the Observability Pipelines Worker](#connect-splunk-forwarder-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/splunk_tcp %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redaction** template to create a new pipeline. -1. Select **Splunk TCP** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_tcp %}} - -### Set up the destination - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk TCP address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_tcp %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector.md deleted file mode 100644 index 14cc9beb954..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector.md +++ /dev/null @@ -1,412 +0,0 @@ ---- -title: Sensitive Data Redaction for the Sumo Logic Hosted Collector HTTP Logs Source -disable_toc: false -aliases: - - /observability_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector/ ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. Or, create custom scanning rules using regex patterns to match sensitive information. - -Observability Pipelines supports Sumo Logic Collector HTTP Logs source. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker over Sumo Logic HTTP Source](#send-logs-to-the-observability-pipelines-worker-over-sumo-logic-http-source) - -## Prerequisites - -{{% observability_pipelines/prerequisites/sumo_logic %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redaction** template to create a new pipeline. -1. Select **Sumo Logic** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/sumo_logic %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Sumo Logic address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/sumo_logic %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog.md deleted file mode 100644 index dcb58cc2504..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog.md +++ /dev/null @@ -1,413 +0,0 @@ ---- -title: Sensitive Data Redaction for Syslog -disable_toc: false -aliases: - - /observability_pipelines/sensitive_data_redaction/syslog/ ---- - -## Overview - -Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. - -Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. - -{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-syslog) - -## Prerequisites - -{{% observability_pipelines/prerequisites/syslog %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Sensitive Data Redactions** template to create a new pipeline. -1. Select **rsyslog** or **syslog-ng** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/syslog %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors_sds %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Syslog address. This is a Syslog-compatible endpoint, exposed by the Worker, that your applications send logs to. The Observability Pipelines Worker listens on this address for incoming logs. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Syslog - -{{% observability_pipelines/log_source_configuration/syslog %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/_index.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/_index.md deleted file mode 100644 index fdccd703cb1..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/_index.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Split Logs -disable_toc: false -aliases: - - /observability_pipelines/split_logs/ -private: true -cascade: - private: true ---- - -## Overview - -Often, organizations need to send their logs to multiple products for different use cases. For example, you might send your security logs to your SIEM application and your DevOps logs to Datadog. Use Observability Pipelines to send your logs to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -Select your log source to get started: - -- [Amazon Data Firehose][12] -- [Amazon S3][11] -- [Datadog Agent][1] -- [Fluentd or Fluent Bit][2] -- [Google Pub/Sub][3] -- [HTTP Client][4] -- [HTTP Server][5] -- [Kafka][13] -- [Logstash][6] -- [Splunk HTTP Event Collector (HEC)][7] -- [Splunk Heavy or Universal Forwarders (TCP)][8] -- [Socket (TCP or UDP)][14] -- [Sumo Logic Hosted Collector][9] -- [rsyslog or syslog-ng][10] - -[1]: /observability_pipelines/split_logs/datadog_agent -[2]: /observability_pipelines/split_logs/fluent -[3]: /observability_pipelines/set_up_pipelines/split_logs/google_pubsub -[4]: /observability_pipelines/split_logs/http_client -[5]: /observability_pipelines/set_up_pipelines/split_logs/http_server -[6]: /observability_pipelines/set_up_pipelines/split_logs/logstash -[7]: /observability_pipelines/split_logs/splunk_hec -[8]: /observability_pipelines/split_logs/splunk_tcp -[9]: /observability_pipelines/split_logs/sumo_logic_hosted_collector -[10]: /observability_pipelines/split_logs/syslog -[11]: /observability_pipelines/set_up_pipelines/split_logs/amazon_s3 -[12]: /observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose -[13]: /observability_pipelines/set_up_pipelines/split_logs/kafka -[14]: /observability_pipelines/set_up_pipelines/split_logs/socket \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose.md deleted file mode 100644 index 555b74e5c00..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Split Logs for Amazon Data Firehose -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to process and route your Amazon Data Firehose logs to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-amazon_data_firehose) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_data_firehose %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select the **Amazon Data Firehose** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_data_firehose %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Amazon Data Firehose address. The Observability Pipelines Worker listens to this address and port for incoming logs from Amazon Data Firehose. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Amazon Data Firehose - -{{% observability_pipelines/log_source_configuration/amazon_data_firehose %}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_s3.md deleted file mode 100644 index 4c9149500ae..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_s3.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Split Logs for Amazon S3 -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to process and route your Amazon S3 logs to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/amazon_s3 %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select the **Amazon S3** source. - -### Set up the source - -{{% observability_pipelines/source_settings/amazon_s3 %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. In the **AWS S3 SQS URL** field, enter the URL of the SQS queue to which the S3 bucket sends notification events. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/datadog_agent.md deleted file mode 100644 index d922970932b..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/datadog_agent.md +++ /dev/null @@ -1,436 +0,0 @@ ---- -title: Split Logs for the Datadog Agent -disable_toc: false -aliases: - - /observability_pipelines/split_logs/datadog_agent/ ---- - -## Overview - -Configure your Datadog Agent to send logs to the Observability Pipelines Worker and then process and route them to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting the Datadog Agent to the Observability Pipelines Worker](#connect-the-datadog-agent-to-the-observability-pipelines-worker) - -## Prerequisites - -{{% observability_pipelines/prerequisites/datadog_agent %}} - -{{< tabs >}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/prerequisites/splunk_hec_destination_only %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/prerequisites/sumo_logic_destination_only %}} - -{{% /tab %}} -{{< /tabs >}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select **Datadog Agent** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/datadog_agent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the listener address, which is the address and port the Observability Pipelines Worker listens on for incoming logs from the Datadog Agent. For example, `0.0.0.0:`. -1. Provide the environment variables for each of your selected destinations. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Connect the Datadog Agent to the Observability Pipelines Worker - -Use the Agent configuration file or the Agent Helm chart values file to connect the Datadog Agent to the Observability Pipelines Worker. - -{{< tabs >}} -{{% tab "Agent configuration file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent %}} - -{{% /tab %}} -{{% tab "Agent Helm values file" %}} - -{{% observability_pipelines/log_source_configuration/datadog_agent_kubernetes %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/fluent.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/fluent.md deleted file mode 100644 index fca27e5b903..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/fluent.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Split Logs for Fluent -disable_toc: false -aliases: - - /observability_pipelines/split_logs/fluent/ ---- - -## Overview - -Configure Fluentd and Fluent Bit to send logs to the Observability Pipelines Worker and then process and route them to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-fluent) - -## Prerequisites - -{{% observability_pipelines/prerequisites/fluent %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select **Fluentd** or **Fluent Bit** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/fluent %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Fluent socket address and port. The Observability Pipelines Worker listens on this address for incoming log messages. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Fluent - -{{% observability_pipelines/log_source_configuration/fluent %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/google_pubsub.md deleted file mode 100644 index 473acf76589..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/google_pubsub.md +++ /dev/null @@ -1,402 +0,0 @@ ---- -title: Split Logs for Google Pub/Sub -disable_toc: false ---- - -## Overview - -Configure Google Pub/Sub to send logs to the Observability Pipelines Worker and then process and route them to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/google_pubsub %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select the **Google Pub/Sub** source. - -### Set up the source - -{{% observability_pipelines/source_settings/google_pubsub %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/http_client.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/http_client.md deleted file mode 100644 index 9407a6c6897..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/http_client.md +++ /dev/null @@ -1,406 +0,0 @@ ---- -title: Split Logs for HTTP Client -disable_toc: false -aliases: - - /observability_pipelines/split_logs/http_client/ ---- - -## Overview - -Use the Observability Pipelines Worker to process and route your HTTP server logs to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_client %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select **HTTP Client** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_client %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the full path of the HTTP/S endpoint URL. For example, `https://127.0.0.8/logs`. The Observability Pipelines Worker collects logs events from this endpoint. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/http_server.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/http_server.md deleted file mode 100644 index b8f206743b5..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/http_server.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Split Logs for HTTP Server -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to process and route your HTTP client logs to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/http_server %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select the **HTTP Server** source. - -### Set up the source - -{{% observability_pipelines/source_settings/http_server %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the HTTP/S server address, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens to this socket address for your HTTP client logs. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/kafka.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/kafka.md deleted file mode 100644 index 0684b17de5b..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/kafka.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -title: Split Logs for Kafka -disable_toc: false ---- - -## Overview - -Use the Observability Pipelines Worker to process and route logs from your Kafka topics to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-kafka) - -## Prerequisites - -{{% observability_pipelines/prerequisites/kafka %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select the **Kafka** source. - -### Set up the source - -{{% observability_pipelines/source_settings/kafka %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the host and port of the Kafka bootstrap servers, which clients should use to connect to the Kafka cluster and discover all the other hosts in the cluster. Must be entered in the format of `host:port`, such as `10.14.22.123:9092`. If there is more than one server, use commas to separate them. - - If you enabled SASL, enter the Kafka SASL username and Kafka SASL password. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/logstash.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/logstash.md deleted file mode 100644 index abf23f85e24..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/logstash.md +++ /dev/null @@ -1,408 +0,0 @@ ---- -title: Split Logs for Logstash -disable_toc: false ---- - -## Overview - -Configure Logstash to send logs to the Observability Pipelines Worker and then process and route them to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-logstash) - -## Prerequisites - -{{% observability_pipelines/prerequisites/logstash %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select the **Logstash** source. - -### Set up the source - -{{% observability_pipelines/source_settings/logstash %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Logstash address and port, such as `0.0.0.0:9997`. The Observability Pipelines Worker listens on this address for incoming log messages. -1. Provide the environment variables for each of your selected destinations. See the [prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Logstash - -{{% observability_pipelines/log_source_configuration/logstash %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/socket.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/socket.md deleted file mode 100644 index cb178b48068..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/socket.md +++ /dev/null @@ -1,384 +0,0 @@ ---- -title: Split Logs for the Socket Source (TCP or UDP) -disable_toc: false ---- - -## Overview - -Send logs over a socket connection to the Observability Pipelines Worker, then process and route them to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document covers: -1. [Prerequisites](#prerequisites) for setting up Observability Pipelines -1. [How to set up Observability Pipelines](#set-up-observability-pipelines) - -## Prerequisites - -{{% observability_pipelines/prerequisites/socket %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select the **Socket** source. - -### Set up the source - -{{% observability_pipelines/source_settings/socket %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -Follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the socket address and port, such as `0.0.0.0:9000`. This is the address and port the Observability Pipelines Worker listens on for incoming logs. The socket address must include a port. -1. If you enabled TLS, enter the TLS passphrase. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_hec.md deleted file mode 100644 index 8181fa9580c..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_hec.md +++ /dev/null @@ -1,415 +0,0 @@ ---- -title: Split Logs for the Splunk HTTP Event Collector (HEC) -disable_toc: false -aliases: - - /observability_pipelines/split_logs/splunk_hec/ ---- - -## Overview - -Configure your Splunk HTTP Event Collectors (HEC) to send logs to the Observability Pipelines Worker and then to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following setup steps: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Worker over Splunk HEC](#send-logs-to-the-observability-pipelines-worker-over-splunk-hec) - -## Prerequisites - -### Splunk HEC - -{{% observability_pipelines/prerequisites/splunk_hec %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select **Splunk HEC** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_hec %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk HEC address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_hec %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_tcp.md deleted file mode 100644 index 2fed755849b..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_tcp.md +++ /dev/null @@ -1,414 +0,0 @@ ---- -title: Split Logs for Splunk Heavy or Universal Forwarders (TCP) -disable_toc: false -aliases: - - /observability_pipelines/split_logs/splunk_tcp/ ---- - -## Overview - -Configure your Splunk Heavy or Universal Forwarders to send logs to the Observability Pipelines Worker and then process and route them to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Connecting Splunk Forwarder to the Observability Pipelines Worker](#connect-splunk-forwarder-to-the-observability-pipelines-worker) - -## Prerequisites - -### Splunk TCP - -{{% observability_pipelines/prerequisites/splunk_tcp %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select **Splunk TCP** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/splunk_tcp %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Splunk TCP address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/splunk_tcp %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector.md deleted file mode 100644 index 8a8defe9387..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector.md +++ /dev/null @@ -1,414 +0,0 @@ ---- -title: Split Logs for the Sumo Logic Hosted Collector HTTP Logs Source -disable_toc: false -aliases: - - /observability_pipelines/split_logs/sumo_logic_hosted_collector/ ---- - -## Overview - -Configure you Sumo Logic Hosted Collector HTTP Logs source to send logs to the Observability Pipelines Worker and then to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker over Sumo Logic HTTP Source](#send-logs-to-the-observability-pipelines-worker-over-sumo-logic-http-source) - -## Prerequisites - -### Sumo Logic Hosted Collector with HTTP Logs source - -{{% observability_pipelines/prerequisites/sumo_logic %}} - -### Datadog Log Management - -{{% observability_pipelines/prerequisites/datadog_agent_destination_only %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select **Sumo Logic** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/sumo_logic %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Sumo Logic address. This is the address and port where your applications are sending their logging data. The Observability Pipelines Worker listens to this address for incoming logs. -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -{{% observability_pipelines/log_source_configuration/sumo_logic %}} - -[1]: https://app.datadoghq.com/observability-pipelines diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/syslog.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/syslog.md deleted file mode 100644 index f433eeab629..00000000000 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/syslog.md +++ /dev/null @@ -1,411 +0,0 @@ ---- -title: Split Logs for Syslog -disable_toc: false -aliases: - - /observability_pipelines/split_logs/syslog/ ---- - -## Overview - -Configure rsyslog or syslog-ng to send logs to the Observability Pipelines Worker and then process and route them to different destinations based on your use case. - -{{% observability_pipelines/use_case_images/split_logs %}} - -This document walks you through the following: -1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines -1. [Setting up Observability Pipelines](#set-up-observability-pipelines) -1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-syslog) - -## Prerequisites - -{{% observability_pipelines/prerequisites/syslog %}} - -## Set up Observability Pipelines - -1. Navigate to [Observability Pipelines][1]. -1. Select the **Split Logs** template to create a new pipeline. -1. Select **rsyslog** or **syslog-ng** as the source. - -### Set up the source - -{{% observability_pipelines/source_settings/syslog %}} - -### Set up the destinations - -Enter the following information based on your selected logs destinations. - -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_settings/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -##### Prerequisites - -{{% observability_pipelines/prerequisites/amazon_security_lake %}} - -##### Set up the destination - -{{% observability_pipelines/destination_settings/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_settings/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_settings/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_note %}} - -{{% observability_pipelines/destination_settings/datadog_archives_prerequisites %}} - -To set up the destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_settings/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_settings/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_settings/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_settings/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_settings/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_settings/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_settings/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_settings/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_settings/syslog %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add additional destinations - -{{% observability_pipelines/multiple_destinations %}} - -### Set up processors - -{{% observability_pipelines/processors/intro %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% observability_pipelines/processors/add_processors %}} - -{{< tabs >}} -{{% tab "Add env vars" %}} - -{{% observability_pipelines/processors/add_env_vars %}} - -{{% /tab %}} -{{% tab "Add hostname" %}} - -{{% observability_pipelines/processors/add_hostname %}} - -{{% /tab %}} -{{% tab "Custom Processor" %}} - -{{% observability_pipelines/processors/custom_processor %}} - -{{% /tab %}} -{{% tab "Dedupe" %}} - -{{% observability_pipelines/processors/dedupe %}} - -{{% /tab %}} -{{% tab "Edit fields" %}} - -{{% observability_pipelines/processors/remap %}} - -{{% /tab %}} -{{% tab "Enrichment table" %}} - -{{% observability_pipelines/processors/enrichment_table %}} - -{{% /tab %}} -{{% tab "Filter" %}} - -{{% observability_pipelines/processors/filter %}} - -{{% /tab %}} -{{% tab "Generate metrics" %}} - -{{% observability_pipelines/processors/generate_metrics %}} - -{{% /tab %}} -{{% tab "Grok Parser" %}} - -{{% observability_pipelines/processors/grok_parser %}} - -{{% /tab %}} -{{% tab "Parse JSON" %}} - -{{% observability_pipelines/processors/parse_json %}} - -{{% /tab %}} -{{% tab "Parse XML" %}} - -{{% observability_pipelines/processors/parse_xml %}} - -{{% /tab %}} -{{% tab "Quota" %}} - -{{% observability_pipelines/processors/quota %}} - -{{% /tab %}} -{{% tab "Reduce" %}} - -{{% observability_pipelines/processors/reduce %}} - -{{% /tab %}} -{{% tab "Remap to OCSF" %}} - -{{% observability_pipelines/processors/remap_ocsf %}} - -{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} - -{{% /collapse-content %}} - -{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} - -{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} - -{{% /collapse-content %}} - -{{% observability_pipelines/processors/filter_syntax %}} - -{{% /tab %}} -{{% tab "Sample" %}} - -{{% observability_pipelines/processors/sample %}} - -{{% /tab %}} -{{% tab "Sensitive Data Scanner" %}} - -{{% observability_pipelines/processors/sensitive_data_scanner %}} - -{{% collapse-content title="Add rules from the library" level="h5" %}} - -{{% observability_pipelines/processors/sds_library_rules %}} - -{{% /collapse-content %}} -{{% collapse-content title="Add a custom rule" level="h5" %}} - -{{% observability_pipelines/processors/sds_custom_rules %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Split array" %}} - -{{% observability_pipelines/processors/split_array %}} - -{{% /tab %}} -{{% tab "Tags Processor" %}} - -{{% observability_pipelines/processors/tags_processor %}} - -{{% /tab %}} -{{% tab "Throttle" %}} - -{{% observability_pipelines/processors/throttle %}} - -{{% /tab %}} -{{< /tabs >}} - -#### Add another set of processors and destinations - -{{% observability_pipelines/multiple_processors %}} - -### Install the Observability Pipelines Worker -1. Select your platform in the **Choose your installation platform** dropdown menu. -1. Enter the Syslog address. This is a Syslog-compatible endpoint, exposed by the Worker, that your applications send logs to. The Observability Pipelines Worker listens on this address for incoming logs. - -1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. -{{< tabs >}} -{{% tab "Amazon OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} - -{{% /tab %}} -{{% tab "Amazon Security Lake" %}} - -{{% observability_pipelines/destination_env_vars/amazon_security_lake %}} - -{{% /tab %}} -{{% tab "Chronicle" %}} - -{{% observability_pipelines/destination_env_vars/chronicle %}} - -{{% /tab %}} -{{% tab "CrowdStrike NG-SIEM" %}} - -{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} - -{{% /tab %}} -{{% tab "Datadog" %}} - -{{% observability_pipelines/destination_env_vars/datadog %}} - -{{% /tab %}} -{{% tab "Datadog Archives" %}} - -For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. - -{{% collapse-content title="Amazon S3" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} - -{{% /collapse-content %}} -{{% collapse-content title="Google Cloud Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} - -{{% /collapse-content %}} -{{% collapse-content title="Azure Storage" level="h5" %}} - -{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} - -{{% /collapse-content %}} - -{{% /tab %}} -{{% tab "Elasticsearch" %}} - -{{% observability_pipelines/destination_env_vars/elasticsearch %}} - -{{% /tab %}} -{{% tab "Microsoft Sentinel" %}} - -{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} - -{{% /tab %}} -{{% tab "New Relic" %}} - -{{% observability_pipelines/destination_env_vars/new_relic %}} - -{{% /tab %}} -{{% tab "OpenSearch" %}} - -{{% observability_pipelines/destination_env_vars/opensearch %}} - -{{% /tab %}} -{{% tab "SentinelOne" %}} - -{{% observability_pipelines/destination_env_vars/sentinelone %}} - -{{% /tab %}} -{{% tab "Socket" %}} - -{{% observability_pipelines/destination_env_vars/socket %}} - -{{% /tab %}} -{{% tab "Splunk HEC" %}} - -{{% observability_pipelines/destination_env_vars/splunk_hec %}} - -{{% /tab %}} -{{% tab "Sumo Logic" %}} - -{{% observability_pipelines/destination_env_vars/sumo_logic %}} - -{{% /tab %}} -{{% tab "Syslog" %}} - -{{% observability_pipelines/destination_env_vars/syslog %}} - -{{% /tab %}} -{{< /tabs >}} -1. Follow the instructions for your environment to install the Worker. -{{< tabs >}} -{{% tab "Docker" %}} - -{{% observability_pipelines/install_worker/docker %}} - -{{% /tab %}} -{{% tab "Kubernetes" %}} - -{{% observability_pipelines/install_worker/kubernetes %}} - -{{% /tab %}} -{{% tab "Linux (APT)" %}} - -{{% observability_pipelines/install_worker/linux_apt %}} - -{{% /tab %}} -{{% tab "Linux (RPM)" %}} - -{{% observability_pipelines/install_worker/linux_rpm %}} - -{{% /tab %}} -{{% tab "CloudFormation" %}} - -{{% observability_pipelines/install_worker/cloudformation %}} - -{{% /tab %}} -{{< /tabs >}} - -## Send logs to the Observability Pipelines Worker over Syslog - -{{% observability_pipelines/log_source_configuration/syslog %}} - -[1]: https://app.datadoghq.com/observability-pipelines