chore: add kafka to docker compose yaml (#837)

traas-stack · Apr 11, 2024 · 3571398 · 3571398
1 parent 8627e19
commit 3571398
Show file tree

Hide file tree

Showing 7 changed files with 95 additions and 7 deletions.
diff --git a/server/agg/agg-executor-bootstrap/src/main/resources/config/application.yaml b/server/agg/agg-executor-bootstrap/src/main/resources/config/application.yaml
@@ -11,7 +11,7 @@ holoinsight:
   roles:
     active: agg-executor
   agg:
-    kafkaBootstrapServers: 127.0.0.1:9092
+    kafkaBootstrapServers: kafka:9092
     topic: aggv1
     consumerGroupId: "x1"
 

diff --git a/server/all-in-one/all-in-one-bootstrap/pom.xml b/server/all-in-one/all-in-one-bootstrap/pom.xml
@@ -49,6 +49,11 @@
 			<artifactId>home-boot</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>io.holoinsight.server</groupId>
+			<artifactId>agg-boot</artifactId>
+			<version>${project.version}</version>
+		</dependency>
 		<dependency>
 			<groupId>io.holoinsight.server</groupId>
 			<artifactId>otlp-server</artifactId>

diff --git a/server/all-in-one/all-in-one-bootstrap/src/main/resources/log4j2.xml b/server/all-in-one/all-in-one-bootstrap/src/main/resources/log4j2.xml
@@ -265,6 +265,30 @@
 			</Policies>
 		</RollingFile>
 
+
+		<RollingFile name="AGG" fileName="${LOG_PATH}/agg/agg.log"
+		             filePattern="${LOG_PATH}/agg/agg.log.%i" immediateFlush="true">
+			<DefaultRolloverStrategy max="1"/>
+			<PatternLayout charset="utf-8">
+				<pattern>%d %-5p [%t] %c{1.} - %m%n</pattern>
+			</PatternLayout>
+			<LevelRangeFilter maxLevel="INFO" minLevel="WARN" onMatch="ACCEPT" onMismatch="DENY"/>
+			<Policies>
+				<SizeBasedTriggeringPolicy size="2GB"/>
+			</Policies>
+		</RollingFile>
+
+		<RollingFile name="AGG-ERROR" fileName="${LOG_PATH}/agg/agg-error.log"
+		             filePattern="${LOG_PATH}/agg/agg-error.log.%i" immediateFlush="true">
+			<DefaultRolloverStrategy max="1"/>
+			<PatternLayout charset="utf-8">
+				<pattern>%d %-5p [%t] %c{1.} - %m%n</pattern>
+			</PatternLayout>
+			<Policies>
+				<SizeBasedTriggeringPolicy size="2GB"/>
+			</Policies>
+		</RollingFile>
+
 	</Appenders>
 
 	<Loggers>
@@ -310,6 +334,10 @@
 		<Logger name="AGENT_EVENT_UP" additivity="false" level="INFO">
 			<Appender-Ref ref="AGENT_EVENT_UP"/>
 		</Logger>
+		<Logger name="io.holoinsight.server.agg" additivity="false" level="INFO">
+			<Appender-Ref ref="AGG"/>
+			<Appender-Ref ref="AGG-ERROR" level="ERROR"/>
+		</Logger>
 		<Root level="INFO">
 			<Appender-Ref level="INFO" ref="ROOT-APPENDER"/>
 			<Appender-Ref level="ERROR" ref="ERROR-APPENDER"/>

diff --git a/test/scenes/common/base.yaml b/test/scenes/common/base.yaml
@@ -77,8 +77,8 @@ services:
       retries: 300
       timeout: 10s
     ports:
-      - 11800
-      - 5001
+    - 11800
+    - 5001
     volumes:
     - ./collector-config.yml:/config/config.yml
   agent-image:
@@ -111,9 +111,9 @@ services:
   tatris:
     image: ${tatris_image:-tatrisio/tatris:latest}
     volumes:
-      - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
+    - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
     ports:
-      - 6060
+    - 6060
     healthcheck:
       test: [ "CMD", "timeout", "1", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6060" ]
       interval: 1s
@@ -136,4 +136,30 @@ services:
   # web terminal
   ttyd:
     image: tsl0922/ttyd
-    entrypoint: ["true"]
+    entrypoint: [ "true" ]
+  kafka:
+    image: bitnami/kafka:3.5.0
+    environment:
+      KAFKA_CFG_NODE_ID: "0"
+      KAFKA_CFG_PROCESS_ROLES: controller,broker
+      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true"
+      ALLOW_PLAINTEXT_LISTENER: "yes"
+      #KAFKA_HEAP_OPTS: "-Xmx1024m -Xms1024m"
+    ports:
+    - 9092
+    volumes:
+    - ./kafka_init.sh:/docker-entrypoint-initdb.d/kafka_init.sh
+  kafka-ui:
+    image: provectuslabs/kafka-ui
+    environment:
+      KAFKA_CLUSTERS_0_NAME: test
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
+    ports:
+    - 8080
+    profiles:
+    - debug
diff --git a/test/scenes/common/status.sh b/test/scenes/common/status.sh
@@ -37,6 +37,16 @@ if echo "$ps" | grep grafana >/dev/null; then
   Grafana_Web_UI\t  http://$ip:`get_port grafana 3000`"
 fi
 
+if echo "$ps" | grep kafka >/dev/null; then
+  source="$source
+  Kafka\t  $ip:`get_port kafka 9092`"
+fi
+
+if echo "$ps" | grep kafka-ui >/dev/null; then
+  source="$source
+  Kafka_Web_UI\t  http://$ip:`get_port kafka-ui 8080`"
+fi
+
 #echo "$source" | column -t | sed '1{p;s/./-/g}'
 #echo
 echo "$source" | ../common/utils/prettytable/prettytable.sh 2

diff --git a/test/scenes/scene-default/application.yaml b/test/scenes/scene-default/application.yaml
@@ -7,7 +7,7 @@ spring:
 
 holoinsight:
   roles:
-    active: query,registry,gateway,home,meta,apm
+    active: query,registry,gateway,home,meta,apm,agg-dispatcher,agg-executor
   metric:
     storage:
       type: ceresdbx
@@ -32,3 +32,7 @@ holoinsight:
       port: 9200
   security:
     whiteHosts: server
+  agg:
+    kafkaBootstrapServers: kafka:9092
+    topic: aggv1
+    consumerGroupId: "x1"
diff --git a/test/scenes/scene-default/docker-compose.yaml b/test/scenes/scene-default/docker-compose.yaml
@@ -142,3 +142,18 @@ services:
     extends:
       file: ../demo/docker-compose.yaml
       service: demo-redis
+  kafka:
+    extends:
+      file: ../common/base.yaml
+      service: kafka
+    healthcheck:
+      test: [ "CMD", "timeout", "1", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9092" ]
+      interval: 3s
+      retries: 300
+      timeout: 10s
+  kafka-ui:
+    extends:
+      file: ../common/base.yaml
+      service: kafka-ui
+    depends_on:
+    - kafka