Skip to content

Commit

Permalink
test to bump spark version
Browse files Browse the repository at this point in the history
  • Loading branch information
andremann committed Nov 5, 2024
1 parent 9a36d8c commit 0cc3b75
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,19 @@ RUN update-alternatives --install "/usr/bin/python" "python" "$(which python3)"

# Fix the value of PYTHONHASHSEED
# Note: this is needed when you use Python 3.3 or greater
ENV SPARK_VERSION=3.5.1 \
ENV SPARK_VERSION=3.5.3 \
HADOOP_VERSION=3 \
SPARK_HOME=/opt/spark \
PYTHONHASHSEED=1


# Download and uncompress spark from the apache archive
RUN wget https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz
RUN wget https://dlcdn.apache.org/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz

RUN mkdir -p /opt/
RUN tar -xf spark-3.5.1-bin-hadoop3.tgz -C /opt/
RUN rm spark-3.5.1-bin-hadoop3.tgz
RUN mv /opt/spark-3.5.1-bin-hadoop3 /opt/spark
RUN tar -xf spark-3.5.3-bin-hadoop3.tgz -C /opt/
RUN rm spark-3.5.3-bin-hadoop3.tgz
RUN mv /opt/spark-3.5.3-bin-hadoop3 /opt/spark

COPY log4j2.properties /opt/spark/conf

Expand Down

0 comments on commit 0cc3b75

Please sign in to comment.