forked from tobilg/docker-spark-jobserver
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Dockerfile
88 lines (72 loc) · 2.84 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
FROM ubuntu:15.10
MAINTAINER tobilg <fb.tools.github@gmail.com>
# packages
RUN apt-get update && apt-get install -yq --no-install-recommends --force-yes \
wget \
git \
openjdk-7-jre \
libjansi-java \
libsvn1 \
libcurl3 \
libsasl2-modules && \
rm -rf /var/lib/apt/lists/*
# Overall ENV vars
ENV SBT_VERSION 0.13.9
ENV SCALA_VERSION 2.10.5
ENV SPARK_VERSION 1.5.1
ENV MESOS_BUILD_VERSION 0.25.0-0.2.70
ENV SPARK_JOBSERVER_BRANCH v0.6.1
# SBT install
RUN wget https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb && \
dpkg -i sbt-$SBT_VERSION.deb && \
rm sbt-$SBT_VERSION.deb
# Scala install
RUN wget http://downloads.typesafe.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.deb && \
dpkg -i scala-$SCALA_VERSION.deb && \
rm scala-$SCALA_VERSION.deb
# Mesos install
RUN wget http://downloads.mesosphere.io/master/ubuntu/15.04/mesos_$MESOS_BUILD_VERSION.ubuntu1504_amd64.deb && \
dpkg -i mesos_$MESOS_BUILD_VERSION.ubuntu1504_amd64.deb && \
rm mesos_$MESOS_BUILD_VERSION.ubuntu1504_amd64.deb
# Spark ENV vars
ENV SPARK_VERSION_STRING spark-$SPARK_VERSION-bin-hadoop2.6
ENV SPARK_DOWNLOAD_URL http://d3kbcqa49mib13.cloudfront.net/$SPARK_VERSION_STRING.tgz
# Download and unzip Spark
RUN wget $SPARK_DOWNLOAD_URL && \
mkdir -p /usr/local/spark && \
tar xvf $SPARK_VERSION_STRING.tgz -C /tmp && \
cp -rf /tmp/$SPARK_VERSION_STRING/* /usr/local/spark/ && \
rm -rf -- /tmp/$SPARK_VERSION_STRING && \
rm spark-$SPARK_VERSION-bin-hadoop2.6.tgz
# Set SPARK_HOME
ENV SPARK_HOME /usr/local/spark
# Set native Mesos library path
ENV MESOS_NATIVE_JAVA_LIBRARY /usr/local/lib/libmesos.so
# H2 Database folder for Spark JobServer
RUN mkdir -p /database
# Clone Spark-Jobserver repository
ENV SPARK_JOBSERVER_BUILD_HOME /spark-jobserver
ENV SPARK_JOBSERVER_APP_HOME /app
RUN git clone --branch $SPARK_JOBSERVER_BRANCH https://github.com/spark-jobserver/spark-jobserver.git
RUN mkdir -p $SPARK_JOBSERVER_APP_HOME
# Add custom files, set permissions
ADD docker.conf $SPARK_JOBSERVER_BUILD_HOME/config/docker.conf
ADD docker.sh $SPARK_JOBSERVER_BUILD_HOME/config/docker.sh
ADD log4j-docker.properties $SPARK_JOBSERVER_BUILD_HOME/config/log4j-server.properties
ADD server_deploy.sh $SPARK_JOBSERVER_BUILD_HOME/bin/server_deploy.sh
RUN chmod +x $SPARK_JOBSERVER_BUILD_HOME/bin/server_deploy.sh
RUN chmod +x $SPARK_JOBSERVER_BUILD_HOME/bin/server_start.sh
RUN chmod +x $SPARK_JOBSERVER_BUILD_HOME/bin/server_stop.sh
# Build Spark-Jobserver
WORKDIR $SPARK_JOBSERVER_BUILD_HOME
RUN bin/server_deploy.sh docker && \
cd / && \
rm -rf -- $SPARK_JOBSERVER_BUILD_HOME
# Cleanup files, folders and variables
RUN unset SPARK_VERSION_STRING && \
unset SPARK_DOWNLOAD_URL && \
unset SPARK_JOBSERVER_BRANCH && \
unset SPARK_JOBSERVER_BUILD_HOME && \
unset MESOS_BUILD_VERSION
EXPOSE 8090 9999
ENTRYPOINT ["/app/server_start.sh"]