-
Notifications
You must be signed in to change notification settings - Fork 31
/
Copy pathdocker-compose.yml
133 lines (122 loc) · 2.92 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
version: "3.9"
services:
####### HADOOP
hadoop-namenode:
container_name: hadoop-namenode
restart: always
build: ./hadoop/namenode
ports:
- 9870:9870
- 9000:9000
networks:
- bds-network
volumes:
- hadoop_namenode:/hadoop/dfs/name
environment:
- CLUSTER_NAME=test
env_file:
- ./hadoop/hadoop.env
hadoop-datanode:
container_name: hadoop-datanode
restart: always
build: ./hadoop/datanode
volumes:
- hadoop_datanode:/hadoop/dfs/data
networks:
- bds-network
environment:
SERVICE_PRECONDITION: "hadoop-namenode:9870"
env_file:
- ./hadoop/hadoop.env
hadoop-resourcemanager:
container_name: hadoop-resourcemanager
restart: always
build: ./hadoop/resourcemanager
networks:
- bds-network
environment:
SERVICE_PRECONDITION: "hadoop-namenode:9000 hadoop-namenode:9870 hadoop-datanode:9864"
env_file:
- ./hadoop/hadoop.env
hadoop-nodemanager-1:
container_name: hadoop-nodemanager-1
restart: always
build: ./hadoop/nodemanager
networks:
- bds-network
environment:
SERVICE_PRECONDITION: "hadoop-namenode:9000 hadoop-namenode:9870 hadoop-datanode:9864 hadoop-resourcemanager:8088"
env_file:
- ./hadoop/hadoop.env
hadoop-historyserver:
container_name: hadoop-historyserver
restart: always
build: ./hadoop/historyserver
networks:
- bds-network
environment:
SERVICE_PRECONDITION: "hadoop-namenode:9000 hadoop-namenode:9870 hadoop-datanode:9864 hadoop-resourcemanager:8088"
volumes:
- hadoop_historyserver:/hadoop/yarn/timeline
env_file:
- ./hadoop/hadoop.env
####### SPARK
spark-master:
build: ./spark/master
container_name: spark-master
ports:
- 8080:8080
- 7077:7077
networks:
- bds-network
volumes:
- shared-workspace:/opt/workspace
spark-worker-1:
build: ./spark/worker
container_name: spark-worker-1
environment:
- SPARK_WORKER_CORES=1
- SPARK_WORKER_MEMORY=512m
ports:
- 8081:8081
networks:
- bds-network
volumes:
- shared-workspace:/opt/workspace
depends_on:
- spark-master
spark-worker-2:
build: ./spark/worker
container_name: spark-worker-2
environment:
- SPARK_WORKER_CORES=1
- SPARK_WORKER_MEMORY=512m
networks:
- bds-network
ports:
- 8082:8081
volumes:
- shared-workspace:/opt/workspace
depends_on:
- spark-master
####### JUPYTER
jupyter-notebook:
build: ./jupyter
container_name: jupyter-notebook
ports:
- 8888:8888
- 4040:4040
networks:
- bds-network
volumes:
- shared-workspace:/opt/workspace
volumes:
hadoop_namenode:
hadoop_datanode:
hadoop_historyserver:
shared-workspace:
name: "hadoop-distributed-file-system"
driver: local
networks:
bds-network:
external: true