Merge pull request #40 from Chia-Network/python-updates

.github/workflows/build-rocky8.yml

@@ -0,0 +1,29 @@
+name: Build Rocky 8 Image
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'rocky8/*'
+      - '.github/workflows/build-rocky8.yml'
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '30 12 * * 5'
+
+concurrency:
+  # SHA is added to the end if on `main` to let all main workflows run
+  group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref == 'refs/heads/main' && github.sha || '' }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    uses: Chia-Network/actions/.github/workflows/docker-build.yaml@main
+    with:
+      docker-context: "./rocky8"
+      dockerfile: "./rocky8/Dockerfile"
+      dockerhub_imagename: "chianetwork/rocky8-builder"
+    secrets:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_PAT: ${{ secrets.DOCKERHUB_PAT }}

centos7/Dockerfile

@@ -1,43 +1,42 @@
 FROM centos:7
 
+ENV PYENV_ROOT=/root/.pyenv
+ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH"
+# Python needs to be able to find the proper sqlite3 install
+ENV LD_LIBRARY_PATH="/usr/lib"
+
 # General Deps
 RUN yum -y groupinstall "Development Tools" && \
     yum -y install ca-certificates createrepo dnf epel-release rpm-build squashfs-tools vim wget && \
-    # relies on epel
+    # jq relies on epel
     yum -y install jq && \
+    yum clean all
 
-    # Compile newer version of sqlite3 \
-    cd ~ && \
-    wget -q https://www.sqlite.org/2021/sqlite-autoconf-3370000.tar.gz && \
-    tar -xzf sqlite-autoconf-3370000.tar.gz && \
-    cd sqlite-autoconf-3370000 && \
-    ./configure --prefix=/usr && \
+# Compile newer version of sqlite3
+RUN cd ~ && \
+    wget -q https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz && \
+    tar -xzf sqlite-autoconf-3400100.tar.gz && \
+    cd sqlite-autoconf-3400100 && \
+    CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=500000" ./configure --prefix=/usr && \
     make && \
     make install && \
     cd ~ && \
-    rm -rf ~/sqlite-autoconf* && \
-    yum clean all
+    rm -rf ~/sqlite-autoconf*
 
 # Compile newer version of python3
 # CVE-2022-0778 fixed in openssl11-1.1.1k-3.el7 https://bugzilla.redhat.com/show_bug.cgi?id=2064913
-RUN yum -y install openssl11 openssl11-devel bzip2-devel libffi-devel zlib-devel && \
+RUN yum -y install openssl-devel openssl11 openssl11-devel zlib-devel bzip2 bzip2-devel readline-devel tk-devel libffi-devel xz-devel && \
     cd ~ && \
-    wget -q https://www.python.org/ftp/python/3.9.9/Python-3.9.9.tgz && \
-    tar xvf Python-3.9.9.tgz && \
-    cd Python-3.9*/ && \
-    # A bit hacky, but gets the proper/newer openssl 1.1.1 version into configure, instead of the default old openssl version \
-    # https://stackoverflow.com/questions/69371800/how-to-link-python3-to-use-openssl11-or-latest-version-of-openssl-1-1-1-on-c
-    sed -i 's/PKG_CONFIG openssl /PKG_CONFIG openssl11 /g' configure && \
-    ./configure --enable-optimizations --prefix=/usr --enable-shared --enable-loadable-sqlite-extensions LDFLAGS="-Wl,-rpath /usr/lib" && \
-    make install && \
-    cd ~ && \
-    rm -rf ~/Python-3.9* && \
-    ln -s /usr/bin/python3 /usr/local/bin/python && \
+    # Set up pyenv \
+    git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
+    CPPFLAGS=$(pkg-config --cflags openssl11) LDFLAGS=$(pkg-config --libs openssl11) pyenv install 3.10 && \
+    pyenv global 3.10 && \
+    pip install --upgrade pip && \
+    pip install --no-cache-dir py3createtorrent awscli && \
+    python3 -c 'import sys; import sqlite3; sys.exit(sqlite3.sqlite_version != "3.40.1")' && \
     yum clean all
 
-RUN pip3 install --upgrade pip && \
-    pip3 install awscli
-
+# Add newer git version
 RUN yum -y install curl-devel expat-devel gettext-devel perl-CPAN perl-devel zlib-devel && \
     cd ~ && \
     wget -q https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.34.1.tar.gz && \
@@ -55,8 +54,7 @@ RUN yum -y install curl-devel expat-devel gettext-devel perl-CPAN perl-devel zli
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN curl -fsSL https://rpm.nodesource.com/setup_16.x | bash - && \
     yum install -y nodejs && \
-    yum clean all && \
-    pip3 install --no-cache-dir py3createtorrent
+    yum clean all
 
 # Add FPM
 RUN gpg2 --keyserver keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && \

rocky8/Dockerfile

@@ -0,0 +1,58 @@
+FROM rockylinux:8
+
+ENV PYENV_ROOT=/root/.pyenv
+ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH"
+# Python needs to be able to find the proper sqlite3 install
+ENV LD_LIBRARY_PATH="/usr/lib64"
+ENV NVM_DIR="/root/.nvm"
+
+# General Deps
+RUN yum -y groupinstall "Development Tools" && \
+    yum -y install ca-certificates createrepo dnf epel-release git rpm-build squashfs-tools vim wget && \
+    # jq relies on epel
+    yum -y install jq && \
+    yum clean all
+
+# Compile newer version of sqlite3
+RUN cd ~ && \
+    wget -q https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz && \
+    tar -xzf sqlite-autoconf-3400100.tar.gz && \
+    cd sqlite-autoconf-3400100 && \
+    CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=500000" ./configure --prefix=/usr && \
+    make && \
+    make install && \
+    cd ~ && \
+    rm -rf ~/sqlite-autoconf*
+
+# Compile newer version of python3
+RUN yum -y install openssl openssl-devel zlib-devel bzip2 bzip2-devel readline-devel tk-devel libffi-devel xz-devel && \
+    cd ~ && \
+    # Set up pyenv \
+    git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
+    pyenv install 3.10 && \
+    pyenv global 3.10 && \
+    pip install --upgrade pip && \
+    pip install --no-cache-dir py3createtorrent awscli && \
+    python3 -c 'import sys; import sqlite3; sys.exit(sqlite3.sqlite_version != "3.40.1")' && \
+    yum clean all
+
+# Add nodejs
+RUN git clone https://github.com/nvm-sh/nvm.git "$NVM_DIR" && \
+    cd "$NVM_DIR" && \
+    git checkout `git describe --abbrev=0 --tags --match "v[0-9]*" $(git rev-list --tags --max-count=1)` && \
+    \. "$NVM_DIR/nvm.sh" && \
+    nvm install 18 && \
+    nvm alias default 18 && \
+    for nodebin in $NVM_DIR/versions/node/$(nvm current)/bin/*; do ln -s "$nodebin" /usr/bin/$(basename $nodebin); done
+
+# Add FPM
+RUN gpg2 --keyserver keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && \
+    curl -sSL https://get.rvm.io | bash -s stable && \
+    /bin/bash -l -c ". /etc/profile.d/rvm.sh && rvm install ruby-3 && gem install fpm" && \
+    yum clean all
+
+# Add GitHub CLI
+RUN dnf -y update && \
+    dnf -y install 'dnf-command(config-manager)' && \
+    dnf -y config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \
+    dnf -y install gh

ubuntu-18.04/Dockerfile

@@ -36,8 +36,8 @@ RUN apt-get update && \
         zlib1g-dev && \
     # Set up pyenv \
     git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
-    pyenv install 3.8 && \
-    pyenv global 3.8 && \
+    pyenv install 3.10 && \
+    pyenv global 3.10 && \
     pip install --upgrade pip && \
     pip install --no-cache-dir py3createtorrent && \
     rm -rf /var/lib/apt/lists/*

ubuntu-20.04/Dockerfile

@@ -2,16 +2,20 @@ FROM ubuntu:20.04
 
 ENV PYENV_ROOT=/root/.pyenv
 ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH"
+ENV LD_LIBRARY_PATH="/usr/lib"
 
 RUN apt-get update && \
     DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
         apt-utils \
+        gpg \
+        gpg-agent \
         software-properties-common  && \
     add-apt-repository ppa:git-core/ppa -y && \
     add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
     DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
         awscli \
         build-essential \
+        ca-certificates \
         curl \
         dialog \
         fakeroot \
@@ -25,7 +29,6 @@ RUN apt-get update && \
         libncursesw5-dev \
         libreadline-dev \
         libssl-dev \
-        libsqlite3-dev \
         libxml2-dev \
         libxmlsec1-dev \
         sudo \
@@ -34,18 +37,35 @@ RUN apt-get update && \
         wget \
         xz-utils \
         zlib1g-dev && \
-    # Set up pyenv \
-    git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
-    pyenv install 3.9 && \
-    pyenv global 3.9 && \
+    rm -rf /var/lib/apt/lists/*
+
+# Compile newer version of sqlite3 \
+RUN cd ~ && \
+    wget -q https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz && \
+    tar -xzf sqlite-autoconf-3400100.tar.gz && \
+    cd sqlite-autoconf-3400100 && \
+    CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=500000" ./configure --prefix=/usr && \
+    make && \
+    make install && \
+    cd ~ && \
+    rm -rf ~/sqlite-autoconf*
+
+# Set up pyenv \
+RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
+    pyenv install 3.10 && \
+    pyenv global 3.10 && \
     pip install --upgrade pip && \
     pip install --no-cache-dir py3createtorrent && \
-    rm -rf /var/lib/apt/lists/*
+    python3 -c 'import sys; import sqlite3; sys.exit(sqlite3.sqlite_version != "3.40.1")'
 
 # Add nodejs
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
-    apt-get install -y nodejs
+ENV NODE_MAJOR=18
+RUN mkdir -p /etc/apt/keyrings/ && \
+    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list && \
+    apt-get update && \
+    apt-get install -y nodejs && \
+    rm -rf /var/lib/apt/lists/*
 
 # Newer version of CMAKE
 RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null && \

ubuntu-22.04/Dockerfile

@@ -2,6 +2,7 @@ FROM ubuntu:22.04
 
 ENV PYENV_ROOT=/root/.pyenv
 ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH"
+ENV LD_LIBRARY_PATH="/usr/lib"
 
 RUN apt-get update && \
     DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
@@ -14,6 +15,7 @@ RUN apt-get update && \
     DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
         awscli \
         build-essential \
+        ca-certificates \
         curl \
         dialog \
         fakeroot \
@@ -27,7 +29,6 @@ RUN apt-get update && \
         libncursesw5-dev \
         libreadline-dev \
         libssl-dev \
-        libsqlite3-dev \
         libxml2-dev \
         libxmlsec1-dev \
         sudo \
@@ -36,18 +37,34 @@ RUN apt-get update && \
         wget \
         xz-utils \
         zlib1g-dev && \
-    # Set up pyenv \
-    git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
+    rm -rf /var/lib/apt/lists/*
+
+# Compile newer version of sqlite3 \
+RUN cd ~ && \
+    wget -q https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz && \
+    tar -xzf sqlite-autoconf-3400100.tar.gz && \
+    cd sqlite-autoconf-3400100 && \
+    CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=500000" ./configure --prefix=/usr && \
+    make && \
+    make install && \
+    cd ~ && \
+    rm -rf ~/sqlite-autoconf*
+
+# Set up pyenv
+RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
     pyenv install 3.10 && \
     pyenv global 3.10 && \
     pip install --upgrade pip && \
     pip install --no-cache-dir py3createtorrent && \
-    rm -rf /var/lib/apt/lists/*
+    python3 -c 'import sys; import sqlite3; sys.exit(sqlite3.sqlite_version != "3.40.1")'
 
 # Add nodejs
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
-    apt-get install -y nodejs
+ENV NODE_MAJOR=18
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list && \
+    apt-get update && \
+    apt-get install -y nodejs && \
+    rm -rf /var/lib/apt/lists/*
 
 # Newer version of CMAKE
 RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null && \