kafka_flink_elasticsearch.d2

#!/usr/bin/env d2
#
#  vim:ts=2:sts=2:sw=2:et:filetype=d2
#
#  Author: Hari Sekhon
#  Date: 2024-08-22 22:51:44 +0200 (Thu, 22 Aug 2024)
#
#  https///github.com/HariSekhon/Diagrams-as-Code
#
#  License: see accompanying Hari Sekhon LICENSE file
#
#  If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
#  https://www.linkedin.com/in/HariSekhon
#

# ============================================================================ #
#                 Kafka Flink to HDFS and Elasticsearch + Kibana
# ============================================================================ #

direction: right

title: {
  label: Kafka Flink Elasticsearch
  near: top-center
  shape: text
  style.font-size: 40
  style.underline: true
}

classes: {
  users: {
    label: Users
    icon: https://icons.terrastruct.com/essentials%2F359-users.svg
    # icon: https://diagrams.mingrammer.com/img/resources/onprem/client/users.png
    shape: image
  }
  kafka: {
    label: Kafka
    icon: https://diagrams.mingrammer.com/img/resources/onprem/queue/kafka.png
    shape: image
  }
  flink: {
    label: Flink
    icon: https://diagrams.mingrammer.com/img/resources/onprem/analytics/flink.png
    shape: image
  }
  elasticsearch: {
    label: Elasticsearch
    icon: https://diagrams.mingrammer.com/img/resources/elastic/saas/elastic.png
    shape: image
  }
  hdfs: {
    label: HDFS
    icon: https://diagrams.mingrammer.com/img/resources/onprem/analytics/hadoop.png
    shape: image
  }
  kibana: {
    label: Kibana
    icon: https://diagrams.mingrammer.com/img/resources/elastic/elasticsearch/kibana.png
    shape: image
  }
}

kafka.class: kafka
flink.class: flink
elasticsearch.class: elasticsearch
kibana.class: kibana
users.class: users
hdfs.class: hdfs

kafka -> flink: streaming {
  style.animated: true
}

flink -> hdfs: batch storage\n(for elasticsearch\nshard recovery\nreindexing) {
  style.animated: true
}

flink -> elasticsearch: streaming {
  style.animated: true
}

elasticsearch -> kibana: Elasticsearch\nAPI access

kibana -> users: UI\nSearch