From d73f93f34a6f8c0aa3c7c980d5223409bd38de8b Mon Sep 17 00:00:00 2001 From: Gabriele Losiczko Date: Wed, 26 Apr 2023 17:35:12 +0200 Subject: [PATCH] docker compose files --- data/.gitkeep | 0 docker-compose-svil-all-except-frontend.yml | 109 ++++++++ docker-compose-svil-basic.yml | 291 ++++++++++++++++++++ docker-compose-svil-microservices-only.yml | 24 ++ hadoop.env | 8 + hbase-distributed-local.env | 14 + 6 files changed, 446 insertions(+) create mode 100644 data/.gitkeep create mode 100644 docker-compose-svil-all-except-frontend.yml create mode 100644 docker-compose-svil-basic.yml create mode 100644 docker-compose-svil-microservices-only.yml create mode 100644 hadoop.env create mode 100644 hbase-distributed-local.env diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker-compose-svil-all-except-frontend.yml b/docker-compose-svil-all-except-frontend.yml new file mode 100644 index 00000000..cec6cf12 --- /dev/null +++ b/docker-compose-svil-all-except-frontend.yml @@ -0,0 +1,109 @@ +version: '2.1' + +networks: + hyperiot: + name: hyperiot + driver: bridge + +services: + zookeeper: + extends: + file: docker-compose-svil-basic.yml + service: zookeeper + + kafka: + extends: + file: docker-compose-svil-basic.yml + service: kafka + depends_on: + - zookeeper + + nimbus: + extends: + file: docker-compose-svil-basic.yml + service: nimbus + links: + - zookeeper + depends_on: + - zookeeper + + supervisor: + extends: + file: docker-compose-svil-basic.yml + service: supervisor + links: + - zookeeper + - nimbus + + storm-ui: + extends: + file: docker-compose-svil-basic.yml + service: storm-ui + links: + - nimbus + depends_on: + - supervisor + - nimbus + + postgresql: + extends: + file: docker-compose-svil-basic.yml + service: postgresql + + karaf-activemq-mqtt: + extends: + file: docker-compose-svil-basic.yml + service: karaf-activemq-mqtt + depends_on: + - postgresql + + karaf-microservices: + extends: + file: docker-compose-svil-basic.yml + service: karaf-microservices + depends_on: + - postgresql + + namenode: + extends: + file: docker-compose-svil-basic.yml + service: namenode + + datanode1: + extends: + file: docker-compose-svil-basic.yml + service: datanode1 + depends_on: + - "namenode" + links: + - "namenode" + + hbase-master: + extends: + file: docker-compose-svil-basic.yml + service: hbase-master + + hbase-region: + extends: + file: docker-compose-svil-basic.yml + service: hbase-region + depends_on: + - hbase-master + + spark-master: + extends: + file: docker-compose-svil-basic.yml + service: spark-master + depends_on: + - "namenode" + links: + - "namenode" + + spark-worker-1: + extends: + file: docker-compose-svil-basic.yml + service: spark-worker-1 + depends_on: + - "spark-master" + links: + - "spark-master" diff --git a/docker-compose-svil-basic.yml b/docker-compose-svil-basic.yml new file mode 100644 index 00000000..141efaed --- /dev/null +++ b/docker-compose-svil-basic.yml @@ -0,0 +1,291 @@ +version: '2.1' + +networks: + hyperiot: + name: hyperiot + driver: bridge + +services: + + zookeeper: + image: zookeeper:3.7.0 + container_name: zookeper + hostname: zookeeper-1.hyperiot.com + ports: + - "2181:2181" + environment: + - ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true +# volumes: +# - ./data/zookeeper:/data/version-2/ + networks: + hyperiot: + aliases: + - zookeeper-1.hyperiot.com + kafka: + image: wurstmeister/kafka:2.13-2.8.1 + container_name: kafka + ports: + - "9092:9092" + environment: + - KAFKA_BOOTSTRAP_SERVERS=kafka-1.hyperiot.com:9092 + - KAFKA_BROKER_ID=1 + - KAFKA_ADVERTISED_HOST_NAME=kafka-1.hyperiot.com + - KAFKA_ADVERTISED_PORT=9092 + - KAFKA_DELETE_TOPIC_ENABLE=true + - KAFKA_LOG_DIRS=/kafka/kafka-logs + - KAFKA_LOG_RETENTION_HOURS=1 + - KAFKA_MESSAGE_MAX_BYTES=10000000 + - KAFKA_REPLICA_FETCH_MAX_BYTES=10000000 + - KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS=60000 + - KAFKA_NUM_PARTITIONS=2 + - KAFKA_DELETE_RETENTION_MS=1000 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper-1.hyperiot.com:2181 + networks: + hyperiot: + aliases: + - kafka-1.hyperiot.com + kafka-manager: + image: hlebalbau/kafka-manager:stable + container_name: kafka-manager + ports: + - "9000:9000" + environment: + ZK_HOSTS: zookeeper-1.hyperiot.com:2181 + APPLICATION_SECRET: letmein + command: -Dpidfile.path=/dev/null + networks: + - hyperiot + + nimbus: + image: nexus.acsoftware.it:18079/hyperiot/storm:2.2.4-1 + command: storm nimbus + container_name: nimbus + hostname: storm.hyperiot.com + networks: + hyperiot: + aliases: + - storm.hyperiot.com + restart: always + ports: + - 6627:6627 + volumes: + - ../hyperiot-storm/storm_extlib:/apache-storm-2.4.0/extlib/ + - ./data/storm/logs/workers-artifacts/:/logs/workers-artifacts + environment: + ZOOKEEPER_SERVERS: "'zookeeper-1.hyperiot.com'" + NIMBUS_SEEDS: "'storm.hyperiot.com'" + SUPERVISOR_SLOTS_PORTS: "6700,6701,6702,6703" + supervisor: + image: nexus.acsoftware.it:18079/hyperiot/storm:2.2.4-1 + command: storm supervisor + container_name: supervisor + restart: always + networks: + - hyperiot + volumes: + - ../hyperiot-storm/storm_extlib:/apache-storm-2.4.0/extlib/ + - ./data/storm/logs/workers-artifacts/:/logs/workers-artifacts + environment: + ZOOKEEPER_SERVERS: "'zookeeper-1.hyperiot.com'" + NIMBUS_SEEDS: "'storm.hyperiot.com'" + SUPERVISOR_SLOTS_PORTS: "6700,6701,6702,6703" + storm-ui: + image: nexus.acsoftware.it:18079/hyperiot/storm:2.2.4-1 + command: storm ui + container_name: storm_ui + ports: + - "8080:8080" + restart: always + networks: + - hyperiot + volumes: + - ../hyperiot-storm/storm_extlib:/apache-storm-2.4.0/extlib/ + - ./data/storm/logs/workers-artifacts/:/logs/workers-artifacts + environment: + ZOOKEEPER_SERVERS: "'zookeeper-1.hyperiot.com'" + NIMBUS_SEEDS: "'storm.hyperiot.com'" + SUPERVISOR_SLOTS_PORTS: "6700,6701,6702,6703" + + postgresql: + image: postgres:9.6.8 + container_name: postgresql + networks: + hyperiot: + aliases: + - postgres.hyperiot.com + ports: + - 5432:5432 + environment: + - POSTGRES_USER=hyperiot + - POSTGRES_PASSWORD=hyperiot + - POSTGRES_DB=hyperiot + - PGDATA=/var/lib/postgresql/data/pgdata + - SCHED_DB_BACKUP="false" + volumes: + - ./data/postgres:/var/lib/postgresql/data + + karaf-activemq-mqtt: + image: nexus.acsoftware.it:18079/hyperiot/karaf-activemq-mqtt:5.16.3-2.2.4-1 + container_name: karaf_active_mqtt + ports: + - "1883:1883" + - "8102:8101" + - "5006:5005" + environment: + - KAFKA_BOOTSTRAP_SERVERS=kafka-1.hyperiot.com:9092 + - JAVA_MIN_MEM=1G + - JAVA_MAX_MEM=1G + - DB_SERVER_NAME=postgres.hyperiot.com + - DB_SERVER_PORT=5432 + - DB_SERVER_USER=hyperiot + - DB_SERVER_PASSWORD=hyperiot + - DB_SERVER_JDBC_DRIVER=org.postgresql.Driver + - HYPERIOT_LOG_LEVEL=TRACE + - HYPERIOT_NODE_ID=1 + - HYPERIOT_ZOOKEEPER_URL=zookeeper-1.hyperiot.com:2181 + networks: + hyperiot: + aliases: + - mqtt.hyperiot.com + karaf-microservices: + image: nexus.acsoftware.it:18079/hyperiot/karaf-microservices:2.2.4-3 + container_name: karaf_microservices + ports: + - "8101:8101" + - "8181:8181" + - "5005:5005" + environment: + - KAFKA_BOOTSTRAP_SERVERS=kafka-1.hyperiot.com:9092 + - HYPERIOT_STORM_NIMBUS_SEEDS=storm.hyperiot.com + - JAVA_MIN_MEM=1G + - JAVA_MAX_MEM=1G +# - EXTRA_JAVA_OPTS=-XX:+PrintCompilation -XX:+PrintInlining XX:-UseCodeCacheFlushing + - DB_SERVER_NAME=postgres.hyperiot.com + - DB_SERVER_PORT=5432 + - DB_SERVER_USER=hyperiot + - DB_SERVER_PASSWORD=hyperiot + - DB_SERVER_JDBC_DRIVER=org.postgresql.Driver + - HYPERIOT_LOG_LEVEL=TRACE + - HYPERIOT_NODE_ID=1 + - HYPERIOT_ZOOKEEPER_URL=zookeeper-1.hyperiot.com:2181 + networks: + hyperiot: + aliases: + - karaf-microservices.hyperiot.com + - karaf-microservices-test.hyperiot.cloud + + nginx-frontend: + image: nexus.acsoftware.it:18079/hyperiot/nginx-hyperiot-frontend:1.3.28-1 + container_name: nginx-frontend + networks: + - hyperiot + ports: + - "80:80" + - "443:443" + environment: + - API_URL=http://karaf-microservices.hyperiot.com:8181 + + namenode: + image: nexus.acsoftware.it:18079/hyperiot/hadoop-namenode:1.0.5 + container_name: namenode + hostname: namenode + networks: + hyperiot: + aliases: + - namenode + volumes: + - ./data/hadoop/namenode:/hadoop/dfs/name + environment: + - CLUSTER_NAME=test + env_file: + - ./hadoop.env + ports: + - "50100:50100" + - "50105:50105" + - "9870:9870" + - "9871:9871" + - "8020:8020" + + datanode1: + image: nexus.acsoftware.it:18079/hyperiot/hadoop-datanode:1.0.5 + container_name: datanode1 + hostname: datanode1 + networks: + hyperiot: + aliases: + - datanode1 + volumes: + - ./data/hadoop/datanode1:/hadoop/dfs/data + env_file: + - ./hadoop.env + ports: + - "9864:9864" # HTTP port + - "9865:9865" # HTTPS port + - "9866:9866" # port for data transfer + - "9867:9867" # ipc port + + hbase-master: + image: nexus.acsoftware.it:18079/hyperiot/hbase-hmaster:1.0.7 + networks: + hyperiot: + aliases: + - hbase-master.hyperiot.com + - hbase-test.hyperiot.cloud + container_name: hbase-master + hostname: hbase-test.hyperiot.cloud + env_file: + - ./hbase-distributed-local.env + environment: + SERVICE_PRECONDITION: "namenode:9870 datanode1:9864 zookeeper-1.hyperiot.com:2181" + ports: + - "16000:16000" + - "16010:16010" + - "9090:9090" + + hbase-region: + image: nexus.acsoftware.it:18079/hyperiot/hbase-regionserver:1.0.7 + networks: + hyperiot: + aliases: + - hbase-regionserver.hyperiot.com + - hbase-regionserver-test.hyperiot.cloud + container_name: hbase-regionserver + hostname: hbase-regionserver.hyperiot.com + env_file: + - ./hbase-distributed-local.env + environment: + HBASE_CONF_hbase_regionserver_hostname: hbase-regionserver.hyperiot.com + SERVICE_PRECONDITION: "namenode:9870 datanode1:9864 zookeeper-1.hyperiot.com:2181 hbase-test.hyperiot.cloud:16010" + ports: + - "16020:16020" + - "16030:16030" + + spark-master: + image: nexus.acsoftware.it:18079/hyperiot/spark-master:1.0.7 + container_name: spark-master + networks: + hyperiot: + aliases: + - spark-master + ports: + - "8082:8080" + - "7077:7077" + - "6066:6066" + environment: + - INIT_DAEMON_STEP=setup_spark + - SPARK_MASTER_HOST=spark-master + - SPARK_CONF_DIR=/spark/conf + - SPARK_EVENT_LOG_DIR=hdfs://namenode:8020/shared/spark-logs + spark-worker-1: + image: nexus.acsoftware.it:18079/hyperiot/spark-worker:1.0.7 + container_name: spark-worker-1 + networks: + hyperiot: + aliases: + - spark-worker-1 + ports: + - "8081:8081" + environment: + - SPARK_MASTER_HOST=spark-master + - SPARK_MASTER=spark://spark-master:7077 + - SPARK_EVENT_LOG_DIR=hdfs://namenode:8020/shared/spark-logs \ No newline at end of file diff --git a/docker-compose-svil-microservices-only.yml b/docker-compose-svil-microservices-only.yml new file mode 100644 index 00000000..62c6583f --- /dev/null +++ b/docker-compose-svil-microservices-only.yml @@ -0,0 +1,24 @@ +version: '2.1' + +networks: + hyperiot: + name: hyperiot + driver: bridge + +services: + zookeeper: + extends: + file: docker-compose-svil-basic.yml + service: zookeeper + + postgresql: + extends: + file: docker-compose-svil-basic.yml + service: postgresql + + karaf-microservices: + extends: + file: docker-compose-svil-basic.yml + service: karaf-microservices + depends_on: + - postgresql diff --git a/hadoop.env b/hadoop.env new file mode 100644 index 00000000..ae75bf19 --- /dev/null +++ b/hadoop.env @@ -0,0 +1,8 @@ +CORE_CONF_fs_defaultFS=hdfs://namenode:8020 +CORE_CONF_hadoop_http_staticuser_user=root +CORE_CONF_hadoop_proxyuser_hue_hosts=* +CORE_CONF_hadoop_proxyuser_hue_groups=* + +HDFS_CONF_dfs_webhdfs_enabled=true +HDFS_CONF_dfs_permissions_enabled=false + diff --git a/hbase-distributed-local.env b/hbase-distributed-local.env new file mode 100644 index 00000000..357515a0 --- /dev/null +++ b/hbase-distributed-local.env @@ -0,0 +1,14 @@ +HBASE_CONF_hbase_rootdir=hdfs://namenode:8020/hbase +HBASE_CONF_hbase_cluster_distributed=true +HBASE_CONF_hbase_zookeeper_quorum=zookeeper-1.hyperiot.com +#HBASE_CONF_hbase_zookeeper_property_dataDir=/usr/local/hbase/zookeeper +#HBASE_CONF_zookeeper_znode_parent=/zookeeper-1.hyperiot.com + +HBASE_CONF_hbase_master=hbase-test.hyperiot.cloud:16000 +HBASE_CONF_hbase_master_hostname=hbase-test.hyperiot.cloud +HBASE_CONF_hbase_master_port=16000 +HBASE_CONF_hbase_master_info_port=16010 +HBASE_CONF_hbase_regionserver_port=16020 +HBASE_CONF_hbase_regionserver_info_port=16030 + +HBASE_MANAGES_ZK=false