diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d76999d099c68..5f756dfc2e2084 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -390,7 +390,7 @@ jobs: - jobname: linux-meson image: ubuntu:rolling cc: gcc - - jobname: linux-musl + - jobname: linux-musl-meson image: alpine:latest # Supported until 2025-04-02. - jobname: linux32 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index be00a1572c4bda..629492694c8949 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,7 +60,7 @@ test:linux: CC: clang - jobname: pedantic image: fedora:latest - - jobname: linux-musl + - jobname: linux-musl-meson image: alpine:latest - jobname: linux32 image: i386/ubuntu:20.04 diff --git a/Makefile b/Makefile index 3d43046c0813c9..f97529e618deb3 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,8 @@ include shared.mak # byte-order mark (BOM) when writing UTF-16 or UTF-32 and always writes in # big-endian format. # -# Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. +# Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. Define +# ZLIB_NG if you want to use zlib-ng instead of zlib. # # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) @@ -986,7 +987,6 @@ LIB_OBJS += commit.o LIB_OBJS += compat/nonblock.o LIB_OBJS += compat/obstack.o LIB_OBJS += compat/terminal.o -LIB_OBJS += compat/zlib-uncompress2.o LIB_OBJS += config.o LIB_OBJS += connect.o LIB_OBJS += connected.o @@ -1695,11 +1695,20 @@ else endif IMAP_SEND_LDFLAGS += $(OPENSSL_LINK) $(OPENSSL_LIBSSL) $(LIB_4_CRYPTO) -ifdef ZLIB_PATH - BASIC_CFLAGS += -I$(ZLIB_PATH)/include - EXTLIBS += $(call libpath_template,$(ZLIB_PATH)/$(lib)) +ifdef ZLIB_NG + BASIC_CFLAGS += -DHAVE_ZLIB_NG + ifdef ZLIB_NG_PATH + BASIC_CFLAGS += -I$(ZLIB_NG_PATH)/include + EXTLIBS += $(call libpath_template,$(ZLIB_NG_PATH)/$(lib)) + endif + EXTLIBS += -lz-ng +else + ifdef ZLIB_PATH + BASIC_CFLAGS += -I$(ZLIB_PATH)/include + EXTLIBS += $(call libpath_template,$(ZLIB_PATH)/$(lib)) + endif + EXTLIBS += -lz endif -EXTLIBS += -lz ifndef NO_OPENSSL OPENSSL_LIBSSL = -lssl diff --git a/archive-tar.c b/archive-tar.c index e7b3489e1e6c82..0edf13fba7568b 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -473,9 +473,7 @@ static const char internal_gzip_command[] = "git archive gzip"; static int write_tar_filter_archive(const struct archiver *ar, struct archiver_args *args) { -#if ZLIB_VERNUM >= 0x1221 struct gz_header_s gzhead = { .os = 3 }; /* Unix, for reproducibility */ -#endif struct strbuf cmd = STRBUF_INIT; struct child_process filter = CHILD_PROCESS_INIT; int r; @@ -486,10 +484,8 @@ static int write_tar_filter_archive(const struct archiver *ar, if (!strcmp(ar->filter_command, internal_gzip_command)) { write_block = tgz_write_block; git_deflate_init_gzip(&gzstream, args->compression_level); -#if ZLIB_VERNUM >= 0x1221 if (deflateSetHeader(&gzstream.z, &gzhead) != Z_OK) BUG("deflateSetHeader() called too late"); -#endif gzstream.next_out = outbuf; gzstream.avail_out = sizeof(outbuf); diff --git a/archive.c b/archive.c index b9c200cba618eb..8be4e7ac8db50c 100644 --- a/archive.c +++ b/archive.c @@ -7,6 +7,7 @@ #include "convert.h" #include "environment.h" #include "gettext.h" +#include "git-zlib.h" #include "hex.h" #include "object-name.h" #include "path.h" diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index d5a959e25ff323..332ba960039157 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -24,8 +24,8 @@ fi case "$distro" in alpine-*) - apk add --update shadow sudo build-base curl-dev openssl-dev expat-dev gettext \ - pcre2-dev python3 musl-libintl perl-utils ncurses \ + apk add --update shadow sudo meson ninja-build gcc libc-dev curl-dev openssl-dev expat-dev gettext \ + zlib-ng-dev pcre2-dev python3 musl-libintl perl-utils ncurses \ apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ bash cvs gnupg perl-cgi perl-dbd-sqlite perl-io-tty >/dev/null ;; diff --git a/ci/lib.sh b/ci/lib.sh index c2574edff29a69..028fea0e7b52d8 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -349,10 +349,7 @@ linux32) CC=gcc ;; linux-musl) - CC=gcc - MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3 USE_LIBPCRE2=Yes" - MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" - MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8" + MESONFLAGS="$MESONFLAGS -DGIT_TEST_UTF8_LOCALE=C.UTF-8" ;; linux-leaks|linux-reftable-leaks) export SANITIZE=leak diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 03d10c61d5621c..1c698467235943 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -56,7 +56,8 @@ case "$jobname" in --fatal-meson-warnings \ --warnlevel 2 --werror \ --wrap-mode nofallback \ - -Dfuzzers=true + -Dfuzzers=true \ + $MESONFLAGS group "Build" meson compile -C build -- if test -n "$run_tests" then diff --git a/compat/zlib-compat.h b/compat/zlib-compat.h new file mode 100644 index 00000000000000..58e53927b227f8 --- /dev/null +++ b/compat/zlib-compat.h @@ -0,0 +1,47 @@ +#ifndef COMPAT_ZLIB_H +#define COMPAT_ZLIB_H + +#ifdef HAVE_ZLIB_NG +# include + +# define z_stream zng_stream +#define gz_header_s zng_gz_header_s + +# define crc32(crc, buf, len) zng_crc32(crc, buf, len) + +# define inflate(strm, bits) zng_inflate(strm, bits) +# define inflateEnd(strm) zng_inflateEnd(strm) +# define inflateInit(strm) zng_inflateInit(strm) +# define inflateInit2(strm, bits) zng_inflateInit2(strm, bits) +# define inflateReset(strm) zng_inflateReset(strm) + +# define deflate(strm, flush) zng_deflate(strm, flush) +# define deflateBound(strm, source_len) zng_deflateBound(strm, source_len) +# define deflateEnd(strm) zng_deflateEnd(strm) +# define deflateInit(strm, level) zng_deflateInit(strm, level) +# define deflateInit2(stream, level, method, window_bits, mem_level, strategy) zng_deflateInit2(stream, level, method, window_bits, mem_level, strategy) +# define deflateReset(strm) zng_deflateReset(strm) +# define deflateSetHeader(strm, head) zng_deflateSetHeader(strm, head) + +#else +# include + +# if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 +# define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) +# endif + +# if ZLIB_VERNUM < 0x1221 +struct gz_header_s { + int os; +}; + +static int deflateSetHeader(z_streamp strm, struct gz_header_s *head) +{ + (void)(strm); + (void)(head); + return Z_OK; +} +# endif +#endif /* HAVE_ZLIB_NG */ + +#endif /* COMPAT_ZLIB_H */ diff --git a/compat/zlib-uncompress2.c b/compat/zlib-uncompress2.c deleted file mode 100644 index 77a1b08048463d..00000000000000 --- a/compat/zlib-uncompress2.c +++ /dev/null @@ -1,96 +0,0 @@ -#include "git-compat-util.h" - -#if ZLIB_VERNUM < 0x1290 -/* taken from zlib's uncompr.c - - commit cacf7f1d4e3d44d871b605da3b647f07d718623f - Author: Mark Adler - Date: Sun Jan 15 09:18:46 2017 -0800 - - zlib 1.2.11 - -*/ - -/* - * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* clang-format off */ - -/* =========================================================================== - Decompresses the source buffer into the destination buffer. *sourceLen is - the byte length of the source buffer. Upon entry, *destLen is the total size - of the destination buffer, which must be large enough to hold the entire - uncompressed data. (The size of the uncompressed data must have been saved - previously by the compressor and transmitted to the decompressor by some - mechanism outside the scope of this compression library.) Upon exit, - *destLen is the size of the decompressed data and *sourceLen is the number - of source bytes consumed. Upon return, source + *sourceLen points to the - first unused input byte. - - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough - memory, Z_BUF_ERROR if there was not enough room in the output buffer, or - Z_DATA_ERROR if the input data was corrupted, including if the input data is - an incomplete zlib stream. -*/ -int ZEXPORT uncompress2 ( - Bytef *dest, - uLongf *destLen, - const Bytef *source, - uLong *sourceLen) { - z_stream stream; - int err; - const uInt max = (uInt)-1; - uLong len, left; - Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ - - len = *sourceLen; - if (*destLen) { - left = *destLen; - *destLen = 0; - } - else { - left = 1; - dest = buf; - } - - stream.next_in = (z_const Bytef *)source; - stream.avail_in = 0; - stream.zalloc = (alloc_func)0; - stream.zfree = (free_func)0; - stream.opaque = (voidpf)0; - - err = inflateInit(&stream); - if (err != Z_OK) return err; - - stream.next_out = dest; - stream.avail_out = 0; - - do { - if (stream.avail_out == 0) { - stream.avail_out = left > (uLong)max ? max : (uInt)left; - left -= stream.avail_out; - } - if (stream.avail_in == 0) { - stream.avail_in = len > (uLong)max ? max : (uInt)len; - len -= stream.avail_in; - } - err = inflate(&stream, Z_NO_FLUSH); - } while (err == Z_OK); - - *sourceLen -= len + stream.avail_in; - if (dest != buf) - *destLen = stream.total_out; - else if (stream.total_out && err == Z_BUF_ERROR) - left = 1; - - inflateEnd(&stream); - return err == Z_STREAM_END ? Z_OK : - err == Z_NEED_DICT ? Z_DATA_ERROR : - err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : - err; -} -#else -static void *dummy_variable = &dummy_variable; -#endif diff --git a/config.c b/config.c index 50f2d17b399446..36f76fafe53ee7 100644 --- a/config.c +++ b/config.c @@ -19,6 +19,7 @@ #include "convert.h" #include "environment.h" #include "gettext.h" +#include "git-zlib.h" #include "ident.h" #include "repository.h" #include "lockfile.h" diff --git a/csum-file.c b/csum-file.c index 232121f415fa50..c81645ba08b810 100644 --- a/csum-file.c +++ b/csum-file.c @@ -11,9 +11,10 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" -#include "progress.h" #include "csum-file.h" +#include "git-zlib.h" #include "hash.h" +#include "progress.h" static void verify_buffer_or_die(struct hashfile *f, const void *buf, diff --git a/environment.c b/environment.c index 8389a272700eac..e5b361bb5de4c6 100644 --- a/environment.c +++ b/environment.c @@ -16,6 +16,7 @@ #include "convert.h" #include "environment.h" #include "gettext.h" +#include "git-zlib.h" #include "repository.h" #include "config.h" #include "refs.h" diff --git a/git-compat-util.h b/git-compat-util.h index f2935750bf2904..f8c6aedc2ee2ca 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1557,18 +1557,6 @@ int cmd_main(int, const char **); int common_exit(const char *file, int line, int code); #define exit(code) exit(common_exit(__FILE__, __LINE__, (code))) -#define z_const -#include - -#if ZLIB_VERNUM < 0x1290 -/* - * This is uncompress2, which is only available in zlib >= 1.2.9 - * (released as of early 2017). See compat/zlib-uncompress2.c. - */ -int uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, - uLong *sourceLen); -#endif - /* * This include must come after system headers, since it introduces macros that * replace system names. diff --git a/git-zlib.c b/git-zlib.c index d43bbeb6daa4c1..519b3647ac8c55 100644 --- a/git-zlib.c +++ b/git-zlib.c @@ -59,7 +59,7 @@ static void zlib_post_call(git_zstream *s) s->total_out = s->z.total_out; s->total_in = s->z.total_in; - s->next_in = s->z.next_in; + s->next_in = (unsigned char *) s->z.next_in; s->next_out = s->z.next_out; s->avail_in -= bytes_consumed; s->avail_out -= bytes_produced; @@ -147,10 +147,6 @@ int git_inflate(git_zstream *strm, int flush) return status; } -#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 -#define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) -#endif - unsigned long git_deflate_bound(git_zstream *strm, unsigned long size) { return deflateBound(&strm->z, size); diff --git a/git-zlib.h b/git-zlib.h index d8a670aff9fbc4..1e8d9aabcb4cb2 100644 --- a/git-zlib.h +++ b/git-zlib.h @@ -1,6 +1,8 @@ #ifndef GIT_ZLIB_H #define GIT_ZLIB_H +#include "compat/zlib-compat.h" + typedef struct git_zstream { z_stream z; unsigned long avail_in; diff --git a/meson.build b/meson.build index b8db1e83eb72cc..d375231f3d5a4f 100644 --- a/meson.build +++ b/meson.build @@ -263,7 +263,6 @@ libgit_sources = [ 'compat/nonblock.c', 'compat/obstack.c', 'compat/terminal.c', - 'compat/zlib-uncompress2.c', 'config.c', 'connect.c', 'connected.c', @@ -668,7 +667,7 @@ build_options_config.set('GIT_TEST_CMP_USE_COPIED_CONTEXT', '') build_options_config.set('GIT_TEST_INDEX_VERSION', '') build_options_config.set('GIT_TEST_OPTS', '') build_options_config.set('GIT_TEST_PERL_FATAL_WARNINGS', '') -build_options_config.set('GIT_TEST_UTF8_LOCALE', '') +build_options_config.set_quoted('GIT_TEST_UTF8_LOCALE', get_option('test_utf8_locale')) build_options_config.set_quoted('LOCALEDIR', fs.as_posix(get_option('prefix') / get_option('localedir'))) build_options_config.set('GITWEBDIR', fs.as_posix(get_option('prefix') / get_option('datadir') / 'gitweb')) @@ -802,11 +801,23 @@ else build_options_config.set('NO_PERL_CPAN_FALLBACKS', '') endif -zlib = dependency('zlib', default_options: ['default_library=static', 'tests=disabled']) -if zlib.version().version_compare('<1.2.0') - libgit_c_args += '-DNO_DEFLATE_BOUND' +zlib_backend = get_option('zlib_backend') +if zlib_backend in ['auto', 'zlib-ng'] + zlib_ng = dependency('zlib-ng', required: zlib_backend == 'zlib-ng') + if zlib_ng.found() + zlib_backend = 'zlib-ng' + libgit_c_args += '-DHAVE_ZLIB_NG' + libgit_dependencies += zlib_ng + endif +endif +if zlib_backend in ['auto', 'zlib'] + zlib = dependency('zlib', default_options: ['default_library=static', 'tests=disabled']) + if zlib.version().version_compare('<1.2.0') + libgit_c_args += '-DNO_DEFLATE_BOUND' + endif + zlib_backend = 'zlib' + libgit_dependencies += zlib endif -libgit_dependencies += zlib threads = dependency('threads', required: false) if threads.found() @@ -2013,4 +2024,5 @@ summary({ 'sha1': sha1_backend, 'sha1_unsafe': sha1_unsafe_backend, 'sha256': sha256_backend, + 'zlib': zlib_backend, }, section: 'Backends') diff --git a/meson_options.txt b/meson_options.txt index f624f97d423999..ce9649783cd5b2 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -57,6 +57,8 @@ option('sha1_unsafe_backend', type: 'combo', choices: ['openssl', 'block', 'Comm description: 'The backend used for hashing data with the SHA1 object format in case no cryptographic security is needed.') option('sha256_backend', type: 'combo', choices: ['openssl', 'nettle', 'gcrypt', 'block'], value: 'block', description: 'The backend used for hashing objects with the SHA256 object format.') +option('zlib_backend', type: 'combo', choices: ['auto', 'zlib', 'zlib-ng'], value: 'auto', + description: 'The backend used for compressing objects and other data.') # Build tweaks. option('breaking_changes', type: 'boolean', value: false, @@ -99,5 +101,7 @@ option('tests', type: 'boolean', value: true, description: 'Enable building tests. This requires Perl, but is separate from the "perl" option such that you can build tests without Perl features enabled.') option('test_output_directory', type: 'string', description: 'Path to the directory used to store test outputs') +option('test_utf8_locale', type: 'string', + description: 'Name of a UTF-8 locale used for testing.') option('fuzzers', type: 'boolean', value: false, description: 'Enable building fuzzers.') diff --git a/reftable/block.c b/reftable/block.c index 9858bbc7c5f7aa..38e3081c614ab2 100644 --- a/reftable/block.c +++ b/reftable/block.c @@ -13,7 +13,6 @@ license that can be found in the LICENSE file or at #include "record.h" #include "reftable-error.h" #include "system.h" -#include int header_size(int version) { diff --git a/reftable/system.h b/reftable/system.h index 5274eca1d05b18..f194a3868634ce 100644 --- a/reftable/system.h +++ b/reftable/system.h @@ -14,6 +14,7 @@ license that can be found in the LICENSE file or at #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "compat/zlib-compat.h" /* * An implementation-specific temporary file. By making this specific to the