chore: switch to Github Actions (#1933)

* add github actions

* fix attempt 1

* fix attempt 2

* fix attempt 3

* fix attempt 4

* fix attempt 5

* fix attempt 6

* fix attempt 7

* fix attempt 8

* fix attempt 9

* fix attempt 10

* fix attempt 11

* fix attempt 12

* fix attempt 13

* fix attempt 14

* fix attempt 15

* fix attempt 16

* fix attempt 17

* fix lint errors

* make deps for lint

* fix fmt

* test self-hosted

* self-hosted runner 1

* self-hosted runner 2

* self-hosted runner 3

* self-hosted runner 4

* self-hosted runner 5

* add CI linter, yugabyte cleanup

* add setup files

* disable shellcheck

* fix systemd file

* update setup instructions

* test without go cache

.github/actionlint.yaml

@@ -0,0 +1,4 @@
+self-hosted-runner:
+  # Labels of self-hosted runner in array of string
+  labels:
+    - docker

.github/actions/install-deps/action.yml

@@ -0,0 +1,24 @@
+name: 'Install Dependencies'
+description: 'Install common dependencies'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y openssh-client git ocl-icd-opencl-dev libhwloc-dev
+      shell: bash
+
+    - name: Fetch all tags
+      run: git fetch --all
+      shell: bash
+
+    - name: Sync submodules
+      run: git submodule sync
+      shell: bash
+
+    - name: Update submodules
+      run: git submodule update --init
+      shell: bash
+

.github/actions/setup-go/action.yml

@@ -0,0 +1,16 @@
+name: 'Setup Go'
+description: 'Setup Go environment'
+
+inputs:
+  go-version:
+    description: 'Go version to use'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ inputs.go-version }}
+        cache: false

.github/image/Dockerfile

@@ -0,0 +1,74 @@
+## This image is used to github action runner for this repo
+# Use the official Golang image as the base image
+FROM golang:1.22-bullseye AS builder
+
+# Set the working directory inside the container
+WORKDIR /app
+
+# Install Git
+RUN apt-get update && apt-get install -y git
+
+# Clone the repositories
+RUN git clone https://github.com/filecoin-project/boost.git
+RUN git clone https://github.com/filecoin-project/curio.git
+
+# Download Go dependencies for Boost
+WORKDIR /app/boost
+RUN git submodule update --init
+RUN go mod download
+
+# Download Go dependencies for Curio
+WORKDIR /app/boost
+RUN git submodule update --init
+RUN go mod download
+
+# Stage 2: Install Lotus binary
+FROM ghcr.io/filecoin-shipyard/lotus-containers:lotus-v1.28.1-devnet AS lotus-test
+
+# Stage 3: Build the final image
+FROM myoung34/github-runner AS curio-github-runner
+
+# Copy Go dependencies from the builder stage
+COPY --from=builder /go/pkg /go/pkg
+COPY --from=builder /go/bin /go/bin
+COPY --from=lotus-test /usr/local/bin/lotus /usr/local/bin/
+
+RUN apt update && apt install -y \
+      build-essential \
+      bzr pkg-config \
+      clang \
+      curl \
+      gcc git \
+      hwloc \
+      jq \
+      libhwloc-dev wget \
+      mesa-opencl-icd \
+      ocl-icd-opencl-dev
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.76.0
+
+RUN set -eux; \
+    dpkgArch="$(dpkg --print-architecture)"; \
+    case "${dpkgArch##*-}" in \
+        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db' ;; \
+        armhf) rustArch='armv7-unknown-linux-gnueabihf'; rustupSha256='f21c44b01678c645d8fbba1e55e4180a01ac5af2d38bcbd14aa665e0d96ed69a' ;; \
+        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='673e336c81c65e6b16dcdede33f4cc9ed0f08bde1dbe7a935f113605292dc800' ;; \
+        i386) rustArch='i686-unknown-linux-gnu'; rustupSha256='e7b0f47557c1afcd86939b118cbcf7fb95a5d1d917bdd355157b63ca00fc4333' ;; \
+        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
+    esac; \
+    url="https://static.rust-lang.org/rustup/archive/1.26.0/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+# Allow attaching a volume for the specified path
+VOLUME /var/tmp/filecoin-proof-parameters

.github/utils/boost-github-actions-runner.env

@@ -0,0 +1,7 @@
+RUNNER_SCOPE=repo
+REPO_URL=https://github.com/filecoin-project/boost
+LABELS=docker
+ACCESS_TOKEN=<ADD TOKEN HERE>
+RUNNER_WORKDIR=/tmp/runner/work
+DISABLE_AUTO_UPDATE=1
+EPHEMERAL=1

.github/utils/create-runner-image.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Directory where the GitHub Action runner will be set up
+RUNNER_DIR="/tmp/github-runner"
+
+# Clone the necessary repositories and set up the runner
+setup_github_runner() {
+    # Remove any existing directory
+    rm -rf $RUNNER_DIR
+
+    # Create the directory
+    mkdir -p $RUNNER_DIR
+
+    # Navigate to the directory
+    # shellcheck disable=SC2164
+    cd $RUNNER_DIR
+
+    # Clone the required repositories
+    git clone https://github.com/filecoin-project/curio.git
+    git clone https://github.com/filecoin-project/boost.git
+
+    # Copy necessary files
+    cp -r boost/.github/utils/* .
+
+    # Copy the Dockerfile
+    cp boost/.github/image/Dockerfile .
+
+    # Build the Docker image
+    docker buildx build -t curio/github-runner:latest .
+}
+
+# Execute the function
+setup_github_runner

.github/utils/curio-github-actions-runner.env

@@ -0,0 +1,7 @@
+RUNNER_SCOPE=repo
+REPO_URL=https://github.com/filecoin-project/curio
+LABELS=docker
+ACCESS_TOKEN=<TOKEN>
+RUNNER_WORKDIR=/tmp/runner/work
+DISABLE_AUTO_UPDATE=1
+EPHEMERAL=1

.github/utils/github-actions-runner.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Ephemeral GitHub Actions Runner Container
+After=docker.service
+Requires=docker.service
+[Service]
+TimeoutStartSec=0
+Restart=always
+ExecStartPre=-/usr/bin/docker stop github-runnerNUM
+ExecStartPre=-/usr/bin/docker rm github-runnerNUM
+#ExecStartPre=-/usr/bin/docker pull curio/github-runner:latest
+ExecStart=/usr/bin/docker run --rm \
+                              --env-file /etc/curio-github-actions-runner.env \
+                              -e RUNNER_NAME=docker-runnerNUM \
+                              -v /var/run/docker.sock:/var/run/docker.sock \
+                              -v /opt/filecoin-proof-parameters:/var/tmp/filecoin-proof-parameters \
+                              --name github-runnerNUM \
+                              curio/github-runner:latest
+[Install]
+WantedBy=multi-user.target

.github/utils/setup.md

@@ -0,0 +1,77 @@
+## How to set up Github Action runner
+1. Clone this repo and copy out the files in this directory to one level above
+```shell
+mkdir github-runner
+
+cd github-runner
+
+git clone https://github.com/filecoin-project/curio.git
+git clone https://github.com/filecoin-project/boost.git
+
+cp -r boost/.github/utils/* .
+```
+2. Copy the Dockerfile
+
+```shell
+copy boost/.github/image/Dockerfile .
+```
+
+3. Create new image
+
+```shell
+docker buildx build -t curio/github-runner:latest .
+```
+
+4. Create systemd file. Please ensure to equal number files for Boost and Curio. If server can host 10 runner then 5 should be for Boost and 5 for Curio.
+
+```shell
+for i in {1..5}; do cat github-actions-runner.service | sed "s/NUM/$i/g" > github-actions-runner$i.service; done
+for i in {6..10}; do cat github-actions-runner.service | sed 's/curio-/boost-/g' | sed "s/NUM/$i/g" > github-actions-runner$i.service; done
+for i in {1..10}; do install -m 644 github-actions-runner$i.service /etc/systemd/system/ ; done
+systemctl daemon-reload
+```
+5. Add the token to ENV files
+6. Copy the ENV files to /etc
+
+```shell
+cp boost-github-actions-runner.env
+cp curio-github-actions-runner.env
+```
+
+7. Start and Enable the services
+```shell
+for i in {1..10}; do systemctl start github-actions-runner$i.service; done
+for i in {1..10}; do systemctl status github-actions-runner$i.service; done
+for i in {1..10}; do systemctl enable github-actions-runner$i.service; done
+```
+
+8. Verify that new runners are visible in the repo.
+
+## Set up docker image creator
+1. Make the script executable
+```shell
+cd github-runner
+chmod +x create-runner-image.sh
+```
+2. Create a cron job to update the image every day
+```shell
+crontab -e
+```
+
+```shell
+0 0 * * * /root/github-runner/create-runner-image.sh
+```
+
+## Github Token
+Creating GitHub personal access token (PAT) for using by self-hosted runner make sure the following scopes are selected:
+
+```text
+repo (all)
+admin:public_key - read:public_key
+admin:repo_hook - read:repo_hook
+admin:org_hook
+notifications
+workflow
+```
+
+## This setup is based in the https://github.com/myoung34/docker-github-actions-runner