From 8a61e2adf0f9b69759522e389b3c9ad5637fb6c8 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Fri, 10 Nov 2023 14:17:47 +0000 Subject: [PATCH 1/7] Update LibAFL and move to LLVM 16 --- fuzzers/libafl/builder.Dockerfile | 14 ++++---------- fuzzers/libafl/fuzzer.py | 2 +- requirements.txt | 2 +- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/fuzzers/libafl/builder.Dockerfile b/fuzzers/libafl/builder.Dockerfile index cef4659cc..34b0103ee 100644 --- a/fuzzers/libafl/builder.Dockerfile +++ b/fuzzers/libafl/builder.Dockerfile @@ -26,19 +26,18 @@ RUN apt-get update && \ apt-get remove -y llvm-10 && \ apt-get install -y \ build-essential \ - llvm-11 \ - clang-12 \ - cargo && \ + lsb-release wget software-properties-common gnupg && \ apt-get install -y wget libstdc++5 libtool-bin automake flex bison \ libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ apt-utils apt-transport-https ca-certificates joe curl && \ + wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 \ PATH="/root/.cargo/bin/:$PATH" cargo install cargo-make # Download libafl. RUN git clone https://github.com/AFLplusplus/LibAFL /libafl # Checkout a current commit -RUN cd /libafl && git pull && git checkout b20fda2a4ada2a6462718dc661e139e6c7a29807 || true +RUN cd /libafl && git pull && git checkout 1138e6a341d14a4751aa7889cfde0344cb59da05 || true # Note that due a nightly bug it is currently fixed to a known version on top! # Compile libafl. @@ -46,9 +45,4 @@ RUN cd /libafl && \ unset CFLAGS CXXFLAGS && \ export LIBAFL_EDGES_MAP_SIZE=2621440 && \ cd ./fuzzers/fuzzbench && \ - PATH="/root/.cargo/bin/:$PATH" cargo build --profile release-fuzzbench --features no_link_main - -# Auxiliary weak references. -RUN cd /libafl/fuzzers/fuzzbench && \ - clang -c stub_rt.c && \ - ar r /stub_rt.a stub_rt.o + PATH="/root/.cargo/bin/:$PATH" cargo build --profile release-fuzzbench \ No newline at end of file diff --git a/fuzzers/libafl/fuzzer.py b/fuzzers/libafl/fuzzer.py index cfd2a64d7..760cdb1aa 100755 --- a/fuzzers/libafl/fuzzer.py +++ b/fuzzers/libafl/fuzzer.py @@ -53,7 +53,7 @@ def build(): # pylint: disable=too-many-branches,too-many-statements utils.append_flags('CXXFLAGS', cflags) utils.append_flags('LDFLAGS', cflags) - os.environ['FUZZER_LIB'] = '/stub_rt.a' + os.environ['FUZZER_LIB'] = '' utils.build_benchmark() diff --git a/requirements.txt b/requirements.txt index 56b835357..58e0529df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,5 +27,5 @@ protobuf==3.20.3 # Needed for development. pylint==2.15.4 -pytype==2022.10.13 +pytype==2023.10.31 yapf==0.32.0 From 5573ee1c216981128684d55e57b7fe80395d4ba8 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Fri, 10 Nov 2023 16:44:17 +0000 Subject: [PATCH 2/7] Remove outdated libafl branch --- fuzzers/libafl_27042023/builder.Dockerfile | 54 ----------------- fuzzers/libafl_27042023/description.md | 11 ---- fuzzers/libafl_27042023/fuzzer.py | 67 ---------------------- fuzzers/libafl_27042023/runner.Dockerfile | 23 -------- 4 files changed, 155 deletions(-) delete mode 100644 fuzzers/libafl_27042023/builder.Dockerfile delete mode 100644 fuzzers/libafl_27042023/description.md delete mode 100755 fuzzers/libafl_27042023/fuzzer.py delete mode 100644 fuzzers/libafl_27042023/runner.Dockerfile diff --git a/fuzzers/libafl_27042023/builder.Dockerfile b/fuzzers/libafl_27042023/builder.Dockerfile deleted file mode 100644 index 01141c74e..000000000 --- a/fuzzers/libafl_27042023/builder.Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG parent_image -FROM $parent_image - -# Uninstall old Rust & Install the latest one. -RUN if which rustup; then rustup self uninstall -y; fi && \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /rustup.sh && \ - sh /rustup.sh --default-toolchain nightly-2023-03-29 -y && \ - rm /rustup.sh - -# Install dependencies. -RUN apt-get update && \ - apt-get remove -y llvm-10 && \ - apt-get install -y \ - build-essential \ - llvm-11 \ - clang-12 \ - cargo && \ - apt-get install -y wget libstdc++5 libtool-bin automake flex bison \ - libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ - apt-utils apt-transport-https ca-certificates joe curl && \ - PATH="/root/.cargo/bin/:$PATH" cargo install cargo-make - -# Download libafl. -RUN git clone https://github.com/AFLplusplus/LibAFL /libafl - -# Checkout a current commit -RUN cd /libafl && git checkout 3ce0c102affeace754072368013422e97b9dce9c || true -# Note that due a nightly bug it is currently fixed to a known version on top! - -# Compile libafl. -RUN cd /libafl && \ - unset CFLAGS CXXFLAGS && \ - export LIBAFL_EDGES_MAP_SIZE=2621440 && \ - cd ./fuzzers/fuzzbench && \ - PATH="/root/.cargo/bin/:$PATH" cargo build --release --features no_link_main - -# Auxiliary weak references. -RUN cd /libafl/fuzzers/fuzzbench && \ - clang -c stub_rt.c && \ - ar r /stub_rt.a stub_rt.o diff --git a/fuzzers/libafl_27042023/description.md b/fuzzers/libafl_27042023/description.md deleted file mode 100644 index ea9b947d6..000000000 --- a/fuzzers/libafl_27042023/description.md +++ /dev/null @@ -1,11 +0,0 @@ -# libafl - -libafl fuzzer instance - - cmplog feature - - persistent mode - -Repository: [https://github.com/AFLplusplus/libafl/](https://github.com/AFLplusplus/libafl/) - -[builder.Dockerfile](builder.Dockerfile) -[fuzzer.py](fuzzer.py) -[runner.Dockerfile](runner.Dockerfile) diff --git a/fuzzers/libafl_27042023/fuzzer.py b/fuzzers/libafl_27042023/fuzzer.py deleted file mode 100755 index d00bb1dd5..000000000 --- a/fuzzers/libafl_27042023/fuzzer.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Integration code for a LibAFL-based fuzzer.""" - -import os -import subprocess - -from fuzzers import utils - - -def prepare_fuzz_environment(input_corpus): - """Prepare to fuzz with a LibAFL-based fuzzer.""" - os.environ['ASAN_OPTIONS'] = 'abort_on_error=1:detect_leaks=0:'\ - 'malloc_context_size=0:symbolize=0:'\ - 'allocator_may_return_null=1:'\ - 'detect_odr_violation=0:handle_segv=0:'\ - 'handle_sigbus=0:handle_abort=0:'\ - 'handle_sigfpe=0:handle_sigill=0' - os.environ['UBSAN_OPTIONS'] = 'abort_on_error=1:'\ - 'allocator_release_to_os_interval_ms=500:'\ - 'handle_abort=0:handle_segv=0:'\ - 'handle_sigbus=0:handle_sigfpe=0:'\ - 'handle_sigill=0:print_stacktrace=0:'\ - 'symbolize=0:symbolize_inline_frames=0' - # Create at least one non-empty seed to start. - utils.create_seed_file_for_empty_corpus(input_corpus) - - -def build(): # pylint: disable=too-many-branches,too-many-statements - """Build benchmark.""" - os.environ['CC'] = '/libafl/fuzzers/fuzzbench/target/release/libafl_cc' - os.environ['CXX'] = '/libafl/fuzzers/fuzzbench/target/release/libafl_cxx' - - os.environ['ASAN_OPTIONS'] = 'abort_on_error=0:allocator_may_return_null=1' - os.environ['UBSAN_OPTIONS'] = 'abort_on_error=0' - - cflags = ['--libafl'] - utils.append_flags('CFLAGS', cflags) - utils.append_flags('CXXFLAGS', cflags) - utils.append_flags('LDFLAGS', cflags) - - os.environ['FUZZER_LIB'] = '/stub_rt.a' - utils.build_benchmark() - - -def fuzz(input_corpus, output_corpus, target_binary): - """Run fuzzer.""" - prepare_fuzz_environment(input_corpus) - dictionary_path = utils.get_dictionary_path(target_binary) - command = [target_binary] - if dictionary_path: - command += (['-x', dictionary_path]) - command += (['-o', output_corpus, '-i', input_corpus]) - print(command) - subprocess.check_call(command, cwd=os.environ['OUT']) diff --git a/fuzzers/libafl_27042023/runner.Dockerfile b/fuzzers/libafl_27042023/runner.Dockerfile deleted file mode 100644 index 7aa1da8e4..000000000 --- a/fuzzers/libafl_27042023/runner.Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM gcr.io/fuzzbench/base-image - -# This makes interactive docker runs painless: -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out" -#ENV AFL_MAP_SIZE=2621440 -ENV PATH="$PATH:/out" -ENV AFL_SKIP_CPUFREQ=1 -ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 -ENV AFL_TESTCACHE_SIZE=2 From 418fac5573fff131e94d83d26bbfc421521d7d35 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Sat, 11 Nov 2023 01:33:03 +0000 Subject: [PATCH 3/7] Address toka's feedback --- fuzzers/libafl/builder.Dockerfile | 13 +++++++------ fuzzers/libafl/fuzzer.py | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fuzzers/libafl/builder.Dockerfile b/fuzzers/libafl/builder.Dockerfile index 34b0103ee..d9631e589 100644 --- a/fuzzers/libafl/builder.Dockerfile +++ b/fuzzers/libafl/builder.Dockerfile @@ -15,12 +15,6 @@ ARG parent_image FROM $parent_image -# Uninstall old Rust & Install the latest one. -RUN if which rustup; then rustup self uninstall -y; fi && \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /rustup.sh && \ - sh /rustup.sh --default-toolchain nightly-2023-09-21 -y && \ - rm /rustup.sh - # Install dependencies. RUN apt-get update && \ apt-get remove -y llvm-10 && \ @@ -31,6 +25,13 @@ RUN apt-get update && \ libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ apt-utils apt-transport-https ca-certificates joe curl && \ wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 \ + wget https://gist.githubusercontent.com/tokatoka/26f4ba95991c6e33139999976332aa8e/raw/20996c7348bcbab115b79c315cee964f1a19e5f3/createAliases.sh && chmod u+x createAliases.sh && ./createAliases.sh + +# Uninstall old Rust & Install the latest one. +RUN if which rustup; then rustup self uninstall -y; fi && \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /rustup.sh && \ + sh /rustup.sh --default-toolchain nightly-2023-09-21 -y && \ + rm /rustup.sh && \ PATH="/root/.cargo/bin/:$PATH" cargo install cargo-make # Download libafl. diff --git a/fuzzers/libafl/fuzzer.py b/fuzzers/libafl/fuzzer.py index 760cdb1aa..3af30d45c 100755 --- a/fuzzers/libafl/fuzzer.py +++ b/fuzzers/libafl/fuzzer.py @@ -49,8 +49,9 @@ def build(): # pylint: disable=too-many-branches,too-many-statements os.environ['UBSAN_OPTIONS'] = 'abort_on_error=0' cflags = ['--libafl'] + cxxflags = ['--libafl', '--std=c++14'] utils.append_flags('CFLAGS', cflags) - utils.append_flags('CXXFLAGS', cflags) + utils.append_flags('CXXFLAGS', cxxflags) utils.append_flags('LDFLAGS', cflags) os.environ['FUZZER_LIB'] = '' From 17cefce0434e309dc3aad84b87ec009ba67e4346 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Mon, 13 Nov 2023 21:32:31 +0100 Subject: [PATCH 4/7] Fix? Co-authored-by: toka --- fuzzers/libafl/builder.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzzers/libafl/builder.Dockerfile b/fuzzers/libafl/builder.Dockerfile index d9631e589..591050f93 100644 --- a/fuzzers/libafl/builder.Dockerfile +++ b/fuzzers/libafl/builder.Dockerfile @@ -24,8 +24,8 @@ RUN apt-get update && \ apt-get install -y wget libstdc++5 libtool-bin automake flex bison \ libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ apt-utils apt-transport-https ca-certificates joe curl && \ - wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 \ - wget https://gist.githubusercontent.com/tokatoka/26f4ba95991c6e33139999976332aa8e/raw/20996c7348bcbab115b79c315cee964f1a19e5f3/createAliases.sh && chmod u+x createAliases.sh && ./createAliases.sh + wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 && \ + wget wget https://gist.githubusercontent.com/tokatoka/26f4ba95991c6e33139999976332aa8e/raw/698ac2087d58ce5c7a6ad59adce58dbfdc32bd46/createAliases.sh && chmod u+x ./createAliases.sh && ./createAliases.sh # Uninstall old Rust & Install the latest one. RUN if which rustup; then rustup self uninstall -y; fi && \ From a17b4480a25a8e99fc618316d855544ed65be963 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Mon, 13 Nov 2023 20:48:36 +0000 Subject: [PATCH 5/7] format --- fuzzers/libafl/fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzers/libafl/fuzzer.py b/fuzzers/libafl/fuzzer.py index 3af30d45c..90ee4fc90 100755 --- a/fuzzers/libafl/fuzzer.py +++ b/fuzzers/libafl/fuzzer.py @@ -49,7 +49,7 @@ def build(): # pylint: disable=too-many-branches,too-many-statements os.environ['UBSAN_OPTIONS'] = 'abort_on_error=0' cflags = ['--libafl'] - cxxflags = ['--libafl', '--std=c++14'] + cxxflags = ['--libafl', '--std=c++14'] utils.append_flags('CFLAGS', cflags) utils.append_flags('CXXFLAGS', cxxflags) utils.append_flags('LDFLAGS', cflags) From 6ffee337339ad206bbbf7ace343e423494474adf Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Thu, 16 Nov 2023 18:07:04 +0000 Subject: [PATCH 6/7] More toka comments --- fuzzers/libafl/builder.Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fuzzers/libafl/builder.Dockerfile b/fuzzers/libafl/builder.Dockerfile index 591050f93..8b7293897 100644 --- a/fuzzers/libafl/builder.Dockerfile +++ b/fuzzers/libafl/builder.Dockerfile @@ -24,8 +24,9 @@ RUN apt-get update && \ apt-get install -y wget libstdc++5 libtool-bin automake flex bison \ libglib2.0-dev libpixman-1-dev python3-setuptools unzip \ apt-utils apt-transport-https ca-certificates joe curl && \ - wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 && \ - wget wget https://gist.githubusercontent.com/tokatoka/26f4ba95991c6e33139999976332aa8e/raw/698ac2087d58ce5c7a6ad59adce58dbfdc32bd46/createAliases.sh && chmod u+x ./createAliases.sh && ./createAliases.sh + wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 16 + +RUN wget https://gist.githubusercontent.com/tokatoka/26f4ba95991c6e33139999976332aa8e/raw/698ac2087d58ce5c7a6ad59adce58dbfdc32bd46/createAliases.sh && chmod u+x ./createAliases.sh && ./createAliases.sh # Uninstall old Rust & Install the latest one. RUN if which rustup; then rustup self uninstall -y; fi && \ From 05e3ce86ed9ab9f675f29e9effedcc8b4ed7306a Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Thu, 16 Nov 2023 19:18:11 +0000 Subject: [PATCH 7/7] bring back the mystery stub --- fuzzers/libafl/builder.Dockerfile | 7 ++++++- fuzzers/libafl/fuzzer.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fuzzers/libafl/builder.Dockerfile b/fuzzers/libafl/builder.Dockerfile index 8b7293897..3f6f70a0d 100644 --- a/fuzzers/libafl/builder.Dockerfile +++ b/fuzzers/libafl/builder.Dockerfile @@ -47,4 +47,9 @@ RUN cd /libafl && \ unset CFLAGS CXXFLAGS && \ export LIBAFL_EDGES_MAP_SIZE=2621440 && \ cd ./fuzzers/fuzzbench && \ - PATH="/root/.cargo/bin/:$PATH" cargo build --profile release-fuzzbench \ No newline at end of file + PATH="/root/.cargo/bin/:$PATH" cargo build --profile release-fuzzbench --features no_link_main + +# Auxiliary weak references. +RUN cd /libafl/fuzzers/fuzzbench && \ + clang -c stub_rt.c && \ + ar r /stub_rt.a stub_rt.o diff --git a/fuzzers/libafl/fuzzer.py b/fuzzers/libafl/fuzzer.py index 90ee4fc90..0f191a744 100755 --- a/fuzzers/libafl/fuzzer.py +++ b/fuzzers/libafl/fuzzer.py @@ -54,7 +54,7 @@ def build(): # pylint: disable=too-many-branches,too-many-statements utils.append_flags('CXXFLAGS', cxxflags) utils.append_flags('LDFLAGS', cflags) - os.environ['FUZZER_LIB'] = '' + os.environ['FUZZER_LIB'] = '/stub_rt.a' utils.build_benchmark()