diff --git a/.github/workflows/ci-master.yml b/.github/workflows/ci-master.yml index 39b8c8046f..5e30a91874 100644 --- a/.github/workflows/ci-master.yml +++ b/.github/workflows/ci-master.yml @@ -109,7 +109,7 @@ jobs: build-windows: name: Build for Windows needs: create-source-distribution - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 env: ARTIFACT_DIR: windows-binaries steps: diff --git a/.gitignore b/.gitignore index b1c3253502..1011a72e52 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ src/tor/src/or *.log *.trs *.dmg +*.iso *.json.h *.raw.h @@ -142,6 +143,8 @@ firo.creator.user firo.files firo.includes +/guix-build-* + # VS .vs /.vs diff --git a/Makefile.am b/Makefile.am index aa14a23a65..bb10f090fb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,8 +5,8 @@ ACLOCAL_AMFLAGS = -I build-aux/m4 SUBDIRS = src .PHONY: deploy FORCE +.INTERMEDIATE: $(OSX_TEMP_ISO) -GZIP_ENV="-9n" export PYTHONPATH if BUILD_BITCOIN_LIBS @@ -25,15 +25,13 @@ space := $(empty) $(empty) OSX_APP=Firo-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg +OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso OSX_BACKGROUND_SVG=background.svg OSX_BACKGROUND_IMAGE=background.tiff OSX_BACKGROUND_IMAGE_DPIS=36 72 -OSX_DSSTORE_GEN=$(top_srcdir)/contrib/macdeploy/custom_dsstore.py OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus -OSX_FANCY_PLIST=$(top_srcdir)/contrib/macdeploy/fancy.plist OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed -OSX_QT_TRANSLATIONS = da,de,es,hu,ru,uk,zh_CN,zh_TW DIST_DOCS = $(wildcard doc/*.md) $(wildcard doc/release-notes/*.md) DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ @@ -50,9 +48,8 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/share/pixmaps/nsis-wizard.bmp \ $(top_srcdir)/doc/README_windows.txt -OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \ +OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ $(top_srcdir)/contrib/macdeploy/$(OSX_BACKGROUND_SVG) \ - $(OSX_DSSTORE_GEN) \ $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh @@ -69,17 +66,18 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIND_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release - @test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \ + @test -f $(MAKENSIS) && echo 'OutFile "$@"' | cat $(top_builddir)/share/setup.nsi - | $(MAKENSIS) -V2 - || \ echo error: could not build $@ @echo built $@ +if TARGET_DARWIN $(OSX_APP)/Contents/PkgInfo: $(MKDIR_P) $(@D) @echo "APPL????" > $@ $(OSX_APP)/Contents/Resources/empty.lproj: $(MKDIR_P) $(@D) - @touch $@ + @touch $@ $(OSX_APP)/Contents/Info.plist: $(OSX_PLIST) $(MKDIR_P) $(@D) @@ -89,9 +87,9 @@ $(OSX_APP)/Contents/Resources/bitcoin.icns: $(OSX_INSTALLER_ICONS) $(MKDIR_P) $(@D) $(INSTALL_DATA) $< $@ -$(OSX_APP)/Contents/MacOS/Firo-Qt: $(BITCOIN_QT_BIN) +$(OSX_APP)/Contents/MacOS/Firo-Qt: all-recursive $(MKDIR_P) $(@D) - STRIPPROG="$(STRIP)" ./libtool --mode=install $(INSTALL_STRIP_PROGRAM) "$<" "$(abspath $@)" + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $@ $(OSX_APP)/Contents/Resources/Base.lproj/InfoPlist.strings: $(MKDIR_P) $(@D) @@ -106,7 +104,7 @@ osx_volname: if BUILD_DARWIN $(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(OSX_BACKGROUND_IMAGE) - $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -add-qt-tr $(OSX_QT_TRANSLATIONS) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -fancy $(OSX_FANCY_PLIST) -verbose 2 -volname $(OSX_VOLNAME) + $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg $(OSX_BACKGROUND_IMAGE).png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 36 -p 36 -o $@ @@ -116,7 +114,7 @@ $(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE).png $(OSX_BACKGROUND_IMAGE)@2x. tiffutil -cathidpicheck $^ -out $@ deploydir: $(OSX_DMG) -else +else !BUILD_DARWIN APP_DIST_DIR=$(top_builddir)/dist APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE) $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications @@ -126,8 +124,11 @@ $(APP_DIST_DIR)/Applications: $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Firo-Qt -$(OSX_DMG): $(APP_DIST_EXTRAS) - $(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -apple -o $@ dist +$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) + $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH)) + +$(OSX_DMG): $(OSX_TEMP_ISO) + $(DMG) dmg "$<" "$@" dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG) sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@ @@ -136,19 +137,16 @@ $(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE_DPIF $(MKDIR_P) $(@D) $(TIFFCP) -c none $(OSX_BACKGROUND_IMAGE_DPIFILES) $@ -$(APP_DIST_DIR)/.DS_Store: $(OSX_DSSTORE_GEN) - $(PYTHON) $< "$@" "$(OSX_VOLNAME)" - $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Firo-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -translations-dir=$(QT_TRANSLATION_DIR) -add-qt-tr $(OSX_QT_TRANSLATIONS) -verbose 2 + INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) deploydir: $(APP_DIST_EXTRAS) -endif +endif !BUILD_DARWIN -if TARGET_DARWIN appbundle: $(OSX_APP_BUILT) deploy: $(OSX_DMG) endif + if TARGET_WINDOWS deploy: $(BITCOIN_WIN_INSTALLER) endif @@ -224,8 +222,6 @@ CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) # This file is problematic for out-of-tree builds if it exists. DISTCLEANFILES = qa/pull-tester/tests_config.pyc -.INTERMEDIATE: $(COVERAGE_INFO) - DISTCHECK_CONFIGURE_FLAGS = --enable-man clean-local: diff --git a/build-aux/m4/ax_check_link_flag.m4 b/build-aux/m4/ax_check_link_flag.m4 index eb01a6ce13..03a30ce4c7 100644 --- a/build-aux/m4/ax_check_link_flag.m4 +++ b/build-aux/m4/ax_check_link_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim # Copyright (c) 2011 Maarten Bosmans # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_LINK_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 83d054af5f..5b5a8ed16e 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -64,6 +64,13 @@ AC_DEFUN([BITCOIN_QT_INIT],[ ], [bitcoin_qt_want_version=auto]) + AS_IF([test "x$with_gui" = xqt5_debug], + [AS_CASE([$host], + [*darwin*], [qt_lib_suffix=_debug], + [*mingw*], [qt_lib_suffix=d], + [qt_lib_suffix= ]); bitcoin_qt_want_version=qt5], + [qt_lib_suffix= ]) + AC_ARG_WITH([qt-incdir],[AS_HELP_STRING([--with-qt-incdir=INC_DIR],[specify qt include path (overridden by pkgconfig)])], [qt_include_path=$withval], []) AC_ARG_WITH([qt-libdir],[AS_HELP_STRING([--with-qt-libdir=LIB_DIR],[specify qt lib path (overridden by pkgconfig)])], [qt_lib_path=$withval], []) AC_ARG_WITH([qt-plugindir],[AS_HELP_STRING([--with-qt-plugindir=PLUGIN_DIR],[specify qt plugin path (overridden by pkgconfig)])], [qt_plugin_path=$withval], []) @@ -72,41 +79,39 @@ AC_DEFUN([BITCOIN_QT_INIT],[ AC_ARG_WITH([qtdbus], [AS_HELP_STRING([--with-qtdbus], - [enable DBus support (default is yes if qt is enabled and QtDBus is found)])], + [enable DBus support (default is yes if qt is enabled and QtDBus is found, except on Android)])], [use_dbus=$withval], [use_dbus=auto]) + dnl Android doesn't support D-Bus and certainly doesn't use it for notifications + case $host in + *android*) + if test "x$use_dbus" != xyes; then + use_dbus=no + fi + ;; + esac + AC_SUBST(QT_TRANSLATION_DIR,$qt_translation_path) ]) -dnl Find the appropriate version of Qt libraries and includes. -dnl Inputs: $1: Whether or not pkg-config should be used. yes|no. Default: yes. -dnl Inputs: $2: If $1 is "yes" and --with-gui=auto, which qt version should be -dnl tried first. -dnl Outputs: See _BITCOIN_QT_FIND_LIBS_* +dnl Find Qt libraries and includes. +dnl +dnl BITCOIN_QT_CONFIGURE([MINIMUM-VERSION]) +dnl +dnl Outputs: See _BITCOIN_QT_FIND_LIBS dnl Outputs: Sets variables for all qt-related tools. dnl Outputs: bitcoin_enable_qt, bitcoin_enable_qt_dbus, bitcoin_enable_qt_test AC_DEFUN([BITCOIN_QT_CONFIGURE],[ - use_pkgconfig=$1 - - if test "x$use_pkgconfig" = x; then - use_pkgconfig=yes - fi - - if test "x$use_pkgconfig" = xyes; then - BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG]) - else - BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG]) - fi + qt_version=">= $1" + qt_lib_prefix="Qt5" + BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS]) dnl This is ugly and complicated. Yuck. Works as follows: - dnl For Qt5, we can check a header to find out whether Qt is build - dnl statically. When Qt is built statically, some plugins must be linked into - dnl the final binary as well. - dnl With Qt5, languages moved into core and the WindowsIntegration plugin was - dnl added. - dnl _BITCOIN_QT_CHECK_STATIC_PLUGINS does a quick link-check and appends the - dnl results to QT_LIBS. + dnl We check a header to find out whether Qt is built statically. + dnl When Qt is built statically, some plugins must be linked into + dnl the final binary as well. _BITCOIN_QT_CHECK_STATIC_PLUGIN does + dnl a quick link-check and appends the results to QT_LIBS. BITCOIN_QT_CHECK([ TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS @@ -114,24 +119,50 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ CXXFLAGS="$PIC_FLAGS $CXXFLAGS" _BITCOIN_QT_IS_STATIC if test "x$bitcoin_cv_static_qt" = xyes; then - _BITCOIN_QT_FIND_STATIC_PLUGINS + _BITCOIN_QT_CHECK_STATIC_LIBS + + if test "x$qt_plugin_path" != x; then + if test -d "$qt_plugin_path/platforms"; then + QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms" + fi + if test -d "$qt_plugin_path/styles"; then + QT_LIBS="$QT_LIBS -L$qt_plugin_path/styles" + fi + if test -d "$qt_plugin_path/accessible"; then + QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible" + fi + if test -d "$qt_plugin_path/platforms/android"; then + QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" + fi + fi + AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) if test "x$TARGET_OS" != xandroid; then - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) - AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMinimalIntegrationPlugin], [-lqminimal]) + AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) fi if test "x$TARGET_OS" = xwindows; then - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QWindowsIntegrationPlugin)],[-lqwindows]) + dnl Linking against wtsapi32 is required. See #17749 and + dnl https://bugreports.qt.io/browse/QTBUG-27097. + AX_CHECK_LINK_FLAG([-lwtsapi32], [QT_LIBS="$QT_LIBS -lwtsapi32"], [AC_MSG_ERROR([could not link against -lwtsapi32])]) + _BITCOIN_QT_CHECK_STATIC_PLUGIN([QWindowsIntegrationPlugin], [-lqwindows]) + _BITCOIN_QT_CHECK_STATIC_PLUGIN([QWindowsVistaStylePlugin], [-lqwindowsvistastyle]) AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows]) elif test "x$TARGET_OS" = xlinux; then - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QXcbIntegrationPlugin)],[-lqxcb -lxcb-static]) + dnl workaround for https://bugreports.qt.io/browse/QTBUG-74874 + AX_CHECK_LINK_FLAG([-lxcb-shm], [QT_LIBS="$QT_LIBS -lxcb-shm"], [AC_MSG_ERROR([could not link against -lxcb-shm])]) + _BITCOIN_QT_CHECK_STATIC_PLUGIN([QXcbIntegrationPlugin], [-lqxcb]) AC_DEFINE(QT_QPA_PLATFORM_XCB, 1, [Define this symbol if the qt platform is xcb]) elif test "x$TARGET_OS" = xdarwin; then - AX_CHECK_LINK_FLAG([[-framework IOKit]],[QT_LIBS="$QT_LIBS -framework IOKit"],[AC_MSG_ERROR(could not iokit framework)]) - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin)],[-lqcocoa]) + AX_CHECK_LINK_FLAG([[-framework Carbon]],[QT_LIBS="$QT_LIBS -framework Carbon"],[AC_MSG_ERROR(could not link against Carbon framework)]) + AX_CHECK_LINK_FLAG([[-framework IOSurface]],[QT_LIBS="$QT_LIBS -framework IOSurface"],[AC_MSG_ERROR(could not link against IOSurface framework)]) + AX_CHECK_LINK_FLAG([[-framework Metal]],[QT_LIBS="$QT_LIBS -framework Metal"],[AC_MSG_ERROR(could not link against Metal framework)]) + AX_CHECK_LINK_FLAG([[-framework QuartzCore]],[QT_LIBS="$QT_LIBS -framework QuartzCore"],[AC_MSG_ERROR(could not link against QuartzCore framework)]) + _BITCOIN_QT_CHECK_STATIC_PLUGIN([QCocoaIntegrationPlugin], [-lqcocoa]) + _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMacStylePlugin], [-lqmacstyle]) AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) elif test "x$TARGET_OS" = xandroid; then - QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype -lQt5EglSupport $QT_LIBS" + QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype $QT_LIBS" AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android]) fi fi @@ -139,8 +170,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ CXXFLAGS=$TEMP_CXXFLAGS ]) - if test "x$use_pkgconfig$qt_bin_path" = xyes; then - qt_bin_path="`$PKG_CONFIG --variable=host_bins Qt5Core 2>/dev/null`" + if test "x$qt_bin_path" = x; then + qt_bin_path="`$PKG_CONFIG --variable=host_bins ${qt_lib_prefix}Core 2>/dev/null`" fi if test "x$use_hardening" != xno; then @@ -195,13 +226,14 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_PATH_PROGS([RCC], [rcc-qt5 rcc5 rcc], $qt_bin_path) BITCOIN_QT_PATH_PROGS([LRELEASE], [lrelease-qt5 lrelease5 lrelease], $qt_bin_path) BITCOIN_QT_PATH_PROGS([LUPDATE], [lupdate-qt5 lupdate5 lupdate],$qt_bin_path, yes) + BITCOIN_QT_PATH_PROGS([LCONVERT], [lconvert-qt5 lconvert5 lconvert], $qt_bin_path, yes) MOC_DEFS='-DHAVE_CONFIG_H -I$(srcdir)' case $host in *darwin*) BITCOIN_QT_CHECK([ MOC_DEFS="${MOC_DEFS} -DQ_OS_MAC" - base_frameworks="-framework Foundation -framework ApplicationServices -framework AppKit" + base_frameworks="-framework Foundation -framework AppKit" AX_CHECK_LINK_FLAG([[$base_frameworks]],[QT_LIBS="$QT_LIBS $base_frameworks"],[AC_MSG_ERROR(could not find base frameworks)]) ]) ;; @@ -213,7 +245,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ dnl enable qt support - AC_MSG_CHECKING(whether to build ]AC_PACKAGE_NAME[ GUI) + AC_MSG_CHECKING([whether to build ]AC_PACKAGE_NAME[ GUI]) BITCOIN_QT_CHECK([ bitcoin_enable_qt=yes bitcoin_enable_qt_test=yes @@ -228,13 +260,16 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AC_MSG_ERROR([libQtDBus not found. Install libQtDBus or remove --with-qtdbus.]) fi if test "x$LUPDATE" = x; then - AC_MSG_WARN([lupdate is required to update qt translations]) + AC_MSG_WARN([lupdate tool is required to update Qt translations.]) + fi + if test "x$LCONVERT" = x; then + AC_MSG_WARN([lconvert tool is required to update Qt translations.]) fi ],[ bitcoin_enable_qt=no ]) if test x$bitcoin_enable_qt = xyes; then - AC_MSG_RESULT([$bitcoin_enable_qt ($QT_LIB_PREFIX)]) + AC_MSG_RESULT([$bitcoin_enable_qt ($qt_lib_prefix)]) else AC_MSG_RESULT([$bitcoin_enable_qt]) fi @@ -251,61 +286,20 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AC_SUBST(MOC_DEFS) ]) -dnl All macros below are internal and should _not_ be used from the main -dnl configure.ac. -dnl ---- - -dnl Internal. Check included version of Qt against minimum specified in doc/dependencies.md -dnl Requires: INCLUDES must be populated as necessary. -dnl Output: bitcoin_cv_qt5=yes|no -AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[ - AC_CACHE_CHECK(for Qt 5, bitcoin_cv_qt5,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include - #ifndef QT_VERSION - # include - #endif - ]], - [[ - #if QT_VERSION < 0x050501 - choke - #endif - ]])], - [bitcoin_cv_qt5=yes], - [bitcoin_cv_qt5=no]) -])]) +dnl All macros below are internal and should _not_ be used from configure.ac. -dnl Internal. Check if the included version of Qt is greater than Qt58. -dnl Requires: INCLUDES must be populated as necessary. -dnl Output: bitcoin_cv_qt58=yes|no -AC_DEFUN([_BITCOIN_QT_CHECK_QT58],[ - AC_CACHE_CHECK(for > Qt 5.7, bitcoin_cv_qt58,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include - #ifndef QT_VERSION - # include - #endif - ]], - [[ - #if QT_VERSION_MINOR < 8 - choke - #endif - ]])], - [bitcoin_cv_qt58=yes], - [bitcoin_cv_qt58=no]) -])]) - - -dnl Internal. Check if the linked version of Qt was built as static libs. -dnl Requires: Qt5. +dnl Internal. Check if the linked version of Qt was built statically. +dnl +dnl _BITCOIN_QT_IS_STATIC +dnl --------------------- +dnl dnl Requires: INCLUDES and LIBS must be populated as necessary. dnl Output: bitcoin_cv_static_qt=yes|no -dnl Output: Defines QT_STATICPLUGIN if plugins are static. AC_DEFUN([_BITCOIN_QT_IS_STATIC],[ AC_CACHE_CHECK(for static Qt, bitcoin_cv_static_qt,[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include - #ifndef QT_VERSION OR QT_VERSION_STR + #ifndef QT_VERSION # include #endif ]], @@ -317,203 +311,88 @@ AC_DEFUN([_BITCOIN_QT_IS_STATIC],[ [bitcoin_cv_static_qt=yes], [bitcoin_cv_static_qt=no]) ]) - if test "x$bitcoin_cv_static_qt" = xyes; then - AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol for static Qt plugins]) - fi ]) -dnl Internal. Check if the link-requirements for static plugins are met. +dnl Internal. Check if the link-requirements for a static plugin are met. +dnl +dnl _BITCOIN_QT_CHECK_STATIC_PLUGIN(PLUGIN, LIBRARIES) +dnl -------------------------------------------------- +dnl dnl Requires: INCLUDES and LIBS must be populated as necessary. -dnl Inputs: $1: A series of Q_IMPORT_PLUGIN(). +dnl Inputs: $1: A static plugin name. dnl Inputs: $2: The libraries that resolve $1. dnl Output: QT_LIBS is prepended or configure exits. -AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_PLUGINS],[ - AC_MSG_CHECKING(for static Qt plugins: $2) +AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_PLUGIN], [ + AC_MSG_CHECKING([for $1 ($2)]) CHECK_STATIC_PLUGINS_TEMP_LIBS="$LIBS" - LIBS="$2 $QT_LIBS $LIBS" + LIBS="$2${qt_lib_suffix} $QT_LIBS $LIBS" AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #define QT_STATICPLUGIN - #include - $1]], - [[return 0;]])], - [AC_MSG_RESULT(yes); QT_LIBS="$2 $QT_LIBS"], - [AC_MSG_RESULT(no); BITCOIN_QT_FAIL(Could not resolve: $2)]) + #include + Q_IMPORT_PLUGIN($1) + ]])], + [AC_MSG_RESULT([yes]); QT_LIBS="$2${qt_lib_suffix} $QT_LIBS"], + [AC_MSG_RESULT([no]); BITCOIN_QT_FAIL([$1 not found.])]) LIBS="$CHECK_STATIC_PLUGINS_TEMP_LIBS" ]) -dnl Internal. Find paths necessary for linking qt static plugins -dnl Inputs: qt_plugin_path. optional. -dnl Outputs: QT_LIBS is appended -AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ - if test "x$qt_plugin_path" != x; then - QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms" - if test -d "$qt_plugin_path/accessible"; then - QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible" - fi - if test -d "$qt_plugin_path/platforms/android"; then - QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" - fi - if test "x$use_pkgconfig" = xyes; then - : dnl - m4_ifdef([PKG_CHECK_MODULES],[ - if test x$bitcoin_cv_qt58 = xno; then - PKG_CHECK_MODULES([QTPLATFORM], [Qt5PlatformSupport], [QT_LIBS="$QTPLATFORM_LIBS $QT_LIBS"]) - else - PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) - fi - if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xdarwin; then - PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) - fi - ]) - else - if test "x$TARGET_OS" = xwindows; then - AC_CACHE_CHECK(for Qt >= 5.6, bitcoin_cv_need_platformsupport,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include - #ifndef QT_VERSION - # include - #endif - ]], - [[ - #if QT_VERSION < 0x050600 || QT_VERSION_MINOR < 6 - choke - #endif - ]])], - [bitcoin_cv_need_platformsupport=yes], - [bitcoin_cv_need_platformsupport=no]) - ]) - if test "x$bitcoin_cv_need_platformsupport" = xyes; then - if test x$bitcoin_cv_qt58 = xno; then - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}PlatformSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXPlatformSupport not found))) - else - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}FontDatabaseSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXFontDatabaseSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}EventDispatcherSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXEventDispatcherSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}ThemeSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXThemeSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}FbSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXFbSupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}DeviceDiscoverySupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXDeviceDiscoverySupport not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}AccessibilitySupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXAccessibilitySupport not found))) - QT_LIBS="$QT_LIBS -lversion -ldwmapi -luxtheme" - fi - fi - fi - fi - fi +dnl Internal. Check Qt static libs with PKG_CHECK_MODULES. +dnl +dnl _BITCOIN_QT_CHECK_STATIC_LIBS +dnl ----------------------------- +dnl +dnl Outputs: QT_LIBS is prepended. +AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_LIBS], [ + PKG_CHECK_MODULES([QT_ACCESSIBILITY], [${qt_lib_prefix}AccessibilitySupport${qt_lib_suffix}], [QT_LIBS="$QT_ACCESSIBILITY_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_DEVICEDISCOVERY], [${qt_lib_prefix}DeviceDiscoverySupport${qt_lib_suffix}], [QT_LIBS="$QT_DEVICEDISCOVERY_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_EDID], [${qt_lib_prefix}EdidSupport${qt_lib_suffix}], [QT_LIBS="$QT_EDID_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_EVENTDISPATCHER], [${qt_lib_prefix}EventDispatcherSupport${qt_lib_suffix}], [QT_LIBS="$QT_EVENTDISPATCHER_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_FB], [${qt_lib_prefix}FbSupport${qt_lib_suffix}], [QT_LIBS="$QT_FB_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_FONTDATABASE], [${qt_lib_prefix}FontDatabaseSupport${qt_lib_suffix}], [QT_LIBS="$QT_FONTDATABASE_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_THEME], [${qt_lib_prefix}ThemeSupport${qt_lib_suffix}], [QT_LIBS="$QT_THEME_LIBS $QT_LIBS"]) + if test "x$TARGET_OS" = xlinux; then + PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_XCBQPA], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_XCBQPA_LIBS $QT_LIBS"]) + elif test "x$TARGET_OS" = xdarwin; then + PKG_CHECK_MODULES([QT_CLIPBOARD], [${qt_lib_prefix}ClipboardSupport${qt_lib_suffix}], [QT_LIBS="$QT_CLIPBOARD_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_GRAPHICS], [${qt_lib_prefix}GraphicsSupport${qt_lib_suffix}], [QT_LIBS="$QT_GRAPHICS_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) + elif test "x$TARGET_OS" = xwindows; then + PKG_CHECK_MODULES([QT_WINDOWSUIAUTOMATION], [${qt_lib_prefix}WindowsUIAutomationSupport${qt_lib_suffix}], [QT_LIBS="$QT_WINDOWSUIAUTOMATION_LIBS $QT_LIBS"]) + elif test "x$TARGET_OS" = xandroid; then + PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) + fi ]) dnl Internal. Find Qt libraries using pkg-config. -dnl Inputs: bitcoin_qt_want_version (from --with-gui=). The version to check -dnl first. -dnl Inputs: $1: If bitcoin_qt_want_version is "auto", check for this version -dnl first. +dnl +dnl _BITCOIN_QT_FIND_LIBS +dnl --------------------- +dnl dnl Outputs: All necessary QT_* variables are set. dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. -AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG],[ - m4_ifdef([PKG_CHECK_MODULES],[ - QT_LIB_PREFIX=Qt5 - qt5_modules="Qt5Core Qt5Gui Qt5Network Qt5Widgets" - BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT5], [$qt5_modules], [QT_INCLUDES="$QT5_CFLAGS"; QT_LIBS="$QT5_LIBS" have_qt=yes],[have_qt=no]) - - if test "x$have_qt" != xyes; then - have_qt=no - BITCOIN_QT_FAIL([Qt dependencies not found]) - fi - ]) - BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT_TEST], [${QT_LIB_PREFIX}Test], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) - if test "x$use_dbus" != xno; then - PKG_CHECK_MODULES([QT_DBUS], [${QT_LIB_PREFIX}DBus], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) - fi - ]) +AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_CORE], [${qt_lib_prefix}Core${qt_lib_suffix} $qt_version], [QT_INCLUDES="$QT_CORE_CFLAGS $QT_INCLUDES" QT_LIBS="$QT_CORE_LIBS $QT_LIBS"], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Core${qt_lib_suffix} $qt_version not found])]) ]) - true; dnl -]) - -dnl Internal. Find Qt libraries without using pkg-config. Version is deduced -dnl from the discovered headers. -dnl Inputs: bitcoin_qt_want_version (from --with-gui=). The version to use. -dnl If "auto", the version will be discovered by _BITCOIN_QT_CHECK_QT5. -dnl Outputs: All necessary QT_* variables are set. -dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. -AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG],[ - TEMP_CPPFLAGS="$CPPFLAGS" - TEMP_CXXFLAGS="$CXXFLAGS" - CXXFLAGS="$PIC_FLAGS $CXXFLAGS" - TEMP_LIBS="$LIBS" BITCOIN_QT_CHECK([ - if test "x$qt_include_path" != x; then - QT_INCLUDES="-I$qt_include_path -I$qt_include_path/QtCore -I$qt_include_path/QtGui -I$qt_include_path/QtWidgets -I$qt_include_path/QtNetwork -I$qt_include_path/QtTest -I$qt_include_path/QtDBus" - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - fi + PKG_CHECK_MODULES([QT_GUI], [${qt_lib_prefix}Gui${qt_lib_suffix} $qt_version], [QT_INCLUDES="$QT_GUI_CFLAGS $QT_INCLUDES" QT_LIBS="$QT_GUI_LIBS $QT_LIBS"], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Gui${qt_lib_suffix} $qt_version not found])]) ]) - - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QtPlugin],,BITCOIN_QT_FAIL(QtCore headers missing))]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QApplication],, BITCOIN_QT_FAIL(QtGui headers missing))]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([QLocalSocket],, BITCOIN_QT_FAIL(QtNetwork headers missing))]) - BITCOIN_QT_CHECK([ - if test "x$bitcoin_qt_want_version" = xauto; then - _BITCOIN_QT_CHECK_QT5 - _BITCOIN_QT_CHECK_QT58 - fi - QT_LIB_PREFIX=Qt5 + PKG_CHECK_MODULES([QT_WIDGETS], [${qt_lib_prefix}Widgets${qt_lib_suffix} $qt_version], [QT_INCLUDES="$QT_WIDGETS_CFLAGS $QT_INCLUDES" QT_LIBS="$QT_WIDGETS_LIBS $QT_LIBS"], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Widgets${qt_lib_suffix} $qt_version not found])]) ]) - BITCOIN_QT_CHECK([ - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="$LIBS -L$qt_lib_path" - fi - - if test "x$TARGET_OS" = xwindows; then - AC_CHECK_LIB([imm32], [main],, BITCOIN_QT_FAIL(libimm32 not found)) - fi + PKG_CHECK_MODULES([QT_NETWORK], [${qt_lib_prefix}Network${qt_lib_suffix} $qt_version], [QT_INCLUDES="$QT_NETWORK_CFLAGS $QT_INCLUDES" QT_LIBS="$QT_NETWORK_LIBS $QT_LIBS"], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Network${qt_lib_suffix} $qt_version not found])]) ]) - BITCOIN_QT_CHECK(AC_CHECK_LIB([z] ,[main],,AC_MSG_WARN([zlib not found. Assuming qt has it built-in]))) - if test x$bitcoin_cv_qt58 = xno; then - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in]))) - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre16_exec], [qtpcre pcre16],,AC_MSG_WARN([libpcre16 not found. Assuming qt has it built-in]))) - else - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtlibpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in]))) - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre2_match_16], [qtpcre2 libqtpcre2],,AC_MSG_WARN([libqtpcre2 not found. Assuming qt has it built-in]))) - fi - BITCOIN_QT_CHECK(AC_SEARCH_LIBS([hb_ot_tags_from_script] ,[qtharfbuzzng qtharfbuzz harfbuzz],,AC_MSG_WARN([libharfbuzz not found. Assuming qt has it built-in or support is disabled]))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Core] ,[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Core not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Gui] ,[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Gui not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Network],[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Network not found))) - BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Widgets],[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Widgets not found))) - QT_LIBS="$LIBS" - LIBS="$TEMP_LIBS" - BITCOIN_QT_CHECK([ - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="-L$qt_lib_path" - fi - AC_CHECK_LIB([${QT_LIB_PREFIX}Test], [main],, have_qt_test=no) - AC_CHECK_HEADER([QTest],, have_qt_test=no) - QT_TEST_LIBS="$LIBS" + PKG_CHECK_MODULES([QT_TEST], [${qt_lib_prefix}Test${qt_lib_suffix} $qt_version], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) if test "x$use_dbus" != xno; then - LIBS= - if test "x$qt_lib_path" != x; then - LIBS="-L$qt_lib_path" - fi - AC_CHECK_LIB([${QT_LIB_PREFIX}DBus], [main],, have_qt_dbus=no) - AC_CHECK_HEADER([QtDBus],, have_qt_dbus=no) - QT_DBUS_LIBS="$LIBS" + PKG_CHECK_MODULES([QT_DBUS], [${qt_lib_prefix}DBus $qt_version], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) fi ]) - CPPFLAGS="$TEMP_CPPFLAGS" - CXXFLAGS="$TEMP_CXXFLAGS" - LIBS="$TEMP_LIBS" ]) diff --git a/configure.ac b/configure.ac index d994e38813..a3f6e817e1 100644 --- a/configure.ac +++ b/configure.ac @@ -419,7 +419,8 @@ case $host in *) AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) - AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) + AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs) + AC_PATH_PROGS([DMG], [dmg], dmg) AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) @@ -1277,7 +1278,7 @@ if test x$openssl_prefix != x; then fi AX_SUBDIRS_CONFIGURE([src/secp256k1], [[--disable-shared], [--with-pic], [--with-bignum=no], [--enable-module-recovery], [--enable-experimental], [--enable-module-ecdh], [--enable-endomorphism]]) -AX_SUBDIRS_CONFIGURE([src/tor], [[--disable-system-torrc], [--disable-systemd], [--disable-lzma], [--disable-asciidoc], [--disable-libscrypt]]) +AX_SUBDIRS_CONFIGURE([src/tor], [[--disable-system-torrc], [--disable-systemd], [--disable-lzma], [--disable-asciidoc], [--disable-libscrypt], [--disable-gcc-hardening], [--enable-pic]]) AC_CONFIG_SUBDIRS([src/bls-signatures]) diff --git a/contrib/devtools/pixie.py b/contrib/devtools/pixie.py new file mode 100644 index 0000000000..64660968ad --- /dev/null +++ b/contrib/devtools/pixie.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 Wladimir J. van der Laan +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Compact, self-contained ELF implementation for bitcoin-core security checks. +''' +import struct +import types +from typing import Dict, List, Optional, Union, Tuple + +# you can find all these values in elf.h +EI_NIDENT = 16 + +# Byte indices in e_ident +EI_CLASS = 4 # ELFCLASSxx +EI_DATA = 5 # ELFDATAxxxx + +ELFCLASS32 = 1 # 32-bit +ELFCLASS64 = 2 # 64-bit + +ELFDATA2LSB = 1 # little endian +ELFDATA2MSB = 2 # big endian + +# relevant values for e_machine +EM_386 = 3 +EM_PPC64 = 21 +EM_ARM = 40 +EM_AARCH64 = 183 +EM_X86_64 = 62 +EM_RISCV = 243 + +# relevant values for e_type +ET_DYN = 3 + +# relevant values for sh_type +SHT_PROGBITS = 1 +SHT_STRTAB = 3 +SHT_DYNAMIC = 6 +SHT_DYNSYM = 11 +SHT_GNU_verneed = 0x6ffffffe +SHT_GNU_versym = 0x6fffffff + +# relevant values for p_type +PT_LOAD = 1 +PT_GNU_STACK = 0x6474e551 +PT_GNU_RELRO = 0x6474e552 + +# relevant values for p_flags +PF_X = (1 << 0) +PF_W = (1 << 1) +PF_R = (1 << 2) + +# relevant values for d_tag +DT_NEEDED = 1 +DT_FLAGS = 30 + +# relevant values of `d_un.d_val' in the DT_FLAGS entry +DF_BIND_NOW = 0x00000008 + +# relevant d_tags with string payload +STRING_TAGS = {DT_NEEDED} + +# rrlevant values for ST_BIND subfield of st_info (symbol binding) +STB_LOCAL = 0 + +class ELFRecord(types.SimpleNamespace): + '''Unified parsing for ELF records.''' + def __init__(self, data: bytes, offset: int, eh: 'ELFHeader', total_size: Optional[int]) -> None: + hdr_struct = self.STRUCT[eh.ei_class][0][eh.ei_data] + if total_size is not None and hdr_struct.size > total_size: + raise ValueError(f'{self.__class__.__name__} header size too small ({total_size} < {hdr_struct.size})') + for field, value in zip(self.STRUCT[eh.ei_class][1], hdr_struct.unpack(data[offset:offset + hdr_struct.size])): + setattr(self, field, value) + +def BiStruct(chars: str) -> Dict[int, struct.Struct]: + '''Compile a struct parser for both endians.''' + return { + ELFDATA2LSB: struct.Struct('<' + chars), + ELFDATA2MSB: struct.Struct('>' + chars), + } + +class ELFHeader(ELFRecord): + FIELDS = ['e_type', 'e_machine', 'e_version', 'e_entry', 'e_phoff', 'e_shoff', 'e_flags', 'e_ehsize', 'e_phentsize', 'e_phnum', 'e_shentsize', 'e_shnum', 'e_shstrndx'] + STRUCT = { + ELFCLASS32: (BiStruct('HHIIIIIHHHHHH'), FIELDS), + ELFCLASS64: (BiStruct('HHIQQQIHHHHHH'), FIELDS), + } + + def __init__(self, data: bytes, offset: int) -> None: + self.e_ident = data[offset:offset + EI_NIDENT] + if self.e_ident[0:4] != b'\x7fELF': + raise ValueError('invalid ELF magic') + self.ei_class = self.e_ident[EI_CLASS] + self.ei_data = self.e_ident[EI_DATA] + + super().__init__(data, offset + EI_NIDENT, self, None) + + def __repr__(self) -> str: + return f'Header(e_ident={self.e_ident!r}, e_type={self.e_type}, e_machine={self.e_machine}, e_version={self.e_version}, e_entry={self.e_entry}, e_phoff={self.e_phoff}, e_shoff={self.e_shoff}, e_flags={self.e_flags}, e_ehsize={self.e_ehsize}, e_phentsize={self.e_phentsize}, e_phnum={self.e_phnum}, e_shentsize={self.e_shentsize}, e_shnum={self.e_shnum}, e_shstrndx={self.e_shstrndx})' + +class Section(ELFRecord): + name: Optional[bytes] = None + FIELDS = ['sh_name', 'sh_type', 'sh_flags', 'sh_addr', 'sh_offset', 'sh_size', 'sh_link', 'sh_info', 'sh_addralign', 'sh_entsize'] + STRUCT = { + ELFCLASS32: (BiStruct('IIIIIIIIII'), FIELDS), + ELFCLASS64: (BiStruct('IIQQQQIIQQ'), FIELDS), + } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: + super().__init__(data, offset, eh, eh.e_shentsize) + self._data = data + + def __repr__(self) -> str: + return f'Section(sh_name={self.sh_name}({self.name!r}), sh_type=0x{self.sh_type:x}, sh_flags={self.sh_flags}, sh_addr=0x{self.sh_addr:x}, sh_offset=0x{self.sh_offset:x}, sh_size={self.sh_size}, sh_link={self.sh_link}, sh_info={self.sh_info}, sh_addralign={self.sh_addralign}, sh_entsize={self.sh_entsize})' + + def contents(self) -> bytes: + '''Return section contents.''' + return self._data[self.sh_offset:self.sh_offset + self.sh_size] + +class ProgramHeader(ELFRecord): + STRUCT = { + # different ELF classes have the same fields, but in a different order to optimize space versus alignment + ELFCLASS32: (BiStruct('IIIIIIII'), ['p_type', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_flags', 'p_align']), + ELFCLASS64: (BiStruct('IIQQQQQQ'), ['p_type', 'p_flags', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_align']), + } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: + super().__init__(data, offset, eh, eh.e_phentsize) + + def __repr__(self) -> str: + return f'ProgramHeader(p_type={self.p_type}, p_offset={self.p_offset}, p_vaddr={self.p_vaddr}, p_paddr={self.p_paddr}, p_filesz={self.p_filesz}, p_memsz={self.p_memsz}, p_flags={self.p_flags}, p_align={self.p_align})' + +class Symbol(ELFRecord): + STRUCT = { + # different ELF classes have the same fields, but in a different order to optimize space versus alignment + ELFCLASS32: (BiStruct('IIIBBH'), ['st_name', 'st_value', 'st_size', 'st_info', 'st_other', 'st_shndx']), + ELFCLASS64: (BiStruct('IBBHQQ'), ['st_name', 'st_info', 'st_other', 'st_shndx', 'st_value', 'st_size']), + } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader, symtab: Section, strings: bytes, version: Optional[bytes]) -> None: + super().__init__(data, offset, eh, symtab.sh_entsize) + self.name = _lookup_string(strings, self.st_name) + self.version = version + + def __repr__(self) -> str: + return f'Symbol(st_name={self.st_name}({self.name!r}), st_value={self.st_value}, st_size={self.st_size}, st_info={self.st_info}, st_other={self.st_other}, st_shndx={self.st_shndx}, version={self.version!r})' + + @property + def is_import(self) -> bool: + '''Returns whether the symbol is an imported symbol.''' + return self.st_bind != STB_LOCAL and self.st_shndx == 0 + + @property + def is_export(self) -> bool: + '''Returns whether the symbol is an exported symbol.''' + return self.st_bind != STB_LOCAL and self.st_shndx != 0 + + @property + def st_bind(self) -> int: + '''Returns STB_*.''' + return self.st_info >> 4 + +class Verneed(ELFRecord): + DEF = (BiStruct('HHIII'), ['vn_version', 'vn_cnt', 'vn_file', 'vn_aux', 'vn_next']) + STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: + super().__init__(data, offset, eh, None) + + def __repr__(self) -> str: + return f'Verneed(vn_version={self.vn_version}, vn_cnt={self.vn_cnt}, vn_file={self.vn_file}, vn_aux={self.vn_aux}, vn_next={self.vn_next})' + +class Vernaux(ELFRecord): + DEF = (BiStruct('IHHII'), ['vna_hash', 'vna_flags', 'vna_other', 'vna_name', 'vna_next']) + STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader, strings: bytes) -> None: + super().__init__(data, offset, eh, None) + self.name = _lookup_string(strings, self.vna_name) + + def __repr__(self) -> str: + return f'Veraux(vna_hash={self.vna_hash}, vna_flags={self.vna_flags}, vna_other={self.vna_other}, vna_name={self.vna_name}({self.name!r}), vna_next={self.vna_next})' + +class DynTag(ELFRecord): + STRUCT = { + ELFCLASS32: (BiStruct('II'), ['d_tag', 'd_val']), + ELFCLASS64: (BiStruct('QQ'), ['d_tag', 'd_val']), + } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader, section: Section) -> None: + super().__init__(data, offset, eh, section.sh_entsize) + + def __repr__(self) -> str: + return f'DynTag(d_tag={self.d_tag}, d_val={self.d_val})' + +def _lookup_string(data: bytes, index: int) -> bytes: + '''Look up string by offset in ELF string table.''' + endx = data.find(b'\x00', index) + assert endx != -1 + return data[index:endx] + +VERSYM_S = BiStruct('H') # .gnu_version section has a single 16-bit integer per symbol in the linked section +def _parse_symbol_table(section: Section, strings: bytes, eh: ELFHeader, versym: bytes, verneed: Dict[int, bytes]) -> List[Symbol]: + '''Parse symbol table, return a list of symbols.''' + data = section.contents() + symbols = [] + versym_iter = (verneed.get(v[0]) for v in VERSYM_S[eh.ei_data].iter_unpack(versym)) + for ofs, version in zip(range(0, len(data), section.sh_entsize), versym_iter): + symbols.append(Symbol(data, ofs, eh, section, strings, version)) + return symbols + +def _parse_verneed(section: Section, strings: bytes, eh: ELFHeader) -> Dict[int, bytes]: + '''Parse .gnu.version_r section, return a dictionary of {versym: 'GLIBC_...'}.''' + data = section.contents() + ofs = 0 + result = {} + while True: + verneed = Verneed(data, ofs, eh) + aofs = ofs + verneed.vn_aux + while True: + vernaux = Vernaux(data, aofs, eh, strings) + result[vernaux.vna_other] = vernaux.name + if not vernaux.vna_next: + break + aofs += vernaux.vna_next + + if not verneed.vn_next: + break + ofs += verneed.vn_next + + return result + +def _parse_dyn_tags(section: Section, strings: bytes, eh: ELFHeader) -> List[Tuple[int, Union[bytes, int]]]: + '''Parse dynamic tags. Return array of tuples.''' + data = section.contents() + ofs = 0 + result = [] + for ofs in range(0, len(data), section.sh_entsize): + tag = DynTag(data, ofs, eh, section) + val = _lookup_string(strings, tag.d_val) if tag.d_tag in STRING_TAGS else tag.d_val + result.append((tag.d_tag, val)) + + return result + +class ELFFile: + sections: List[Section] + program_headers: List[ProgramHeader] + dyn_symbols: List[Symbol] + dyn_tags: List[Tuple[int, Union[bytes, int]]] + + def __init__(self, data: bytes) -> None: + self.data = data + self.hdr = ELFHeader(self.data, 0) + self._load_sections() + self._load_program_headers() + self._load_dyn_symbols() + self._load_dyn_tags() + self._section_to_segment_mapping() + + def _load_sections(self) -> None: + self.sections = [] + for idx in range(self.hdr.e_shnum): + offset = self.hdr.e_shoff + idx * self.hdr.e_shentsize + self.sections.append(Section(self.data, offset, self.hdr)) + + shstr = self.sections[self.hdr.e_shstrndx].contents() + for section in self.sections: + section.name = _lookup_string(shstr, section.sh_name) + + def _load_program_headers(self) -> None: + self.program_headers = [] + for idx in range(self.hdr.e_phnum): + offset = self.hdr.e_phoff + idx * self.hdr.e_phentsize + self.program_headers.append(ProgramHeader(self.data, offset, self.hdr)) + + def _load_dyn_symbols(self) -> None: + # first, load 'verneed' section + verneed = None + for section in self.sections: + if section.sh_type == SHT_GNU_verneed: + strtab = self.sections[section.sh_link].contents() # associated string table + assert verneed is None # only one section of this kind please + verneed = _parse_verneed(section, strtab, self.hdr) + assert verneed is not None + + # then, correlate GNU versym sections with dynamic symbol sections + versym = {} + for section in self.sections: + if section.sh_type == SHT_GNU_versym: + versym[section.sh_link] = section + + # finally, load dynsym sections + self.dyn_symbols = [] + for idx, section in enumerate(self.sections): + if section.sh_type == SHT_DYNSYM: # find dynamic symbol tables + strtab_data = self.sections[section.sh_link].contents() # associated string table + versym_data = versym[idx].contents() # associated symbol version table + self.dyn_symbols += _parse_symbol_table(section, strtab_data, self.hdr, versym_data, verneed) + + def _load_dyn_tags(self) -> None: + self.dyn_tags = [] + for idx, section in enumerate(self.sections): + if section.sh_type == SHT_DYNAMIC: # find dynamic tag tables + strtab = self.sections[section.sh_link].contents() # associated string table + self.dyn_tags += _parse_dyn_tags(section, strtab, self.hdr) + + def _section_to_segment_mapping(self) -> None: + for ph in self.program_headers: + ph.sections = [] + for section in self.sections: + if ph.p_vaddr <= section.sh_addr < (ph.p_vaddr + ph.p_memsz): + ph.sections.append(section) + + def query_dyn_tags(self, tag_in: int) -> List[Union[int, bytes]]: + '''Return the values of all dyn tags with the specified tag.''' + return [val for (tag, val) in self.dyn_tags if tag == tag_in] + + +def load(filename: str) -> ELFFile: + with open(filename, 'rb') as f: + data = f.read() + return ELFFile(data) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 26a53babb0..0b59d8eada 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,215 +1,261 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -Perform basic ELF security checks on a series of executables. +Perform basic security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. -Needs `readelf` (for ELF) and `objdump` (for PE). ''' -import subprocess import sys -import os +from typing import List, Optional -READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') -OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') -NONFATAL = {'HIGH_ENTROPY_VA'} # checks which are non-fatal for now but only generate a warning +import lief +import pixie -def check_ELF_PIE(executable): +def check_ELF_PIE(executable) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + elf = pixie.load(executable) + return elf.hdr.e_type == pixie.ET_DYN - ok = False - for line in stdout.splitlines(): - line = line.split() - if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN': - ok = True - return ok - -def get_ELF_program_headers(executable): - '''Return type and flags for ELF program headers''' - p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - in_headers = False - count = 0 - headers = [] - for line in stdout.splitlines(): - if line.startswith('Program Headers:'): - in_headers = True - if line == '': - in_headers = False - if in_headers: - if count == 1: # header line - ofs_typ = line.find('Type') - ofs_offset = line.find('Offset') - ofs_flags = line.find('Flg') - ofs_align = line.find('Align') - if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align == -1: - raise ValueError('Cannot parse elfread -lW output') - elif count > 1: - typ = line[ofs_typ:ofs_offset].rstrip() - flags = line[ofs_flags:ofs_align].rstrip() - headers.append((typ, flags)) - count += 1 - return headers - -def check_ELF_NX(executable): +def check_ELF_NX(executable) -> bool: ''' Check that no sections are writable and executable (including the stack) ''' + elf = pixie.load(executable) have_wx = False have_gnu_stack = False - for (typ, flags) in get_ELF_program_headers(executable): - if typ == 'GNU_STACK': + for ph in elf.program_headers: + if ph.p_type == pixie.PT_GNU_STACK: have_gnu_stack = True - if 'W' in flags and 'E' in flags: # section is both writable and executable + if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0: # section is both writable and executable have_wx = True return have_gnu_stack and not have_wx -def check_ELF_RELRO(executable): +def check_ELF_RELRO(executable) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' + elf = pixie.load(executable) have_gnu_relro = False - for (typ, flags) in get_ELF_program_headers(executable): - # Note: not checking flags == 'R': here as linkers set the permission differently - # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions. + for ph in elf.program_headers: + # Note: not checking p_flags == PF_R: here as linkers set the permission differently + # This does not affect security: the permission flags of the GNU_RELRO program + # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 - if typ == 'GNU_RELRO': + if ph.p_type == pixie.PT_GNU_RELRO: have_gnu_relro = True have_bindnow = False - p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2:]): + for flags in elf.query_dyn_tags(pixie.DT_FLAGS): + assert isinstance(flags, int) + if flags & pixie.DF_BIND_NOW: have_bindnow = True + return have_gnu_relro and have_bindnow -def check_ELF_Canary(executable): +def check_ELF_Canary(executable) -> bool: ''' Check for use of stack canary ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + elf = pixie.load(executable) ok = False - for line in stdout.splitlines(): - if '__stack_chk_fail' in line: + for symbol in elf.dyn_symbols: + if symbol.name == b'__stack_chk_fail': ok = True return ok -def get_PE_dll_characteristics(executable): - ''' - Get PE DllCharacteristics bits. - Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' - and bits is the DllCharacteristics value. - ''' - p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - arch = '' - bits = 0 - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens)>=2 and tokens[0] == 'architecture:': - arch = tokens[1].rstrip(',') - if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': - bits = int(tokens[1],16) - return (arch,bits) - -IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 -IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 -IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100 - -def check_PE_DYNAMIC_BASE(executable): +def check_ELF_separate_code(executable): + ''' + Check that sections are appropriately separated in virtual memory, + based on their permissions. This checks for missing -Wl,-z,separate-code + and potentially other problems. + ''' + elf = pixie.load(executable) + R = pixie.PF_R + W = pixie.PF_W + E = pixie.PF_X + EXPECTED_FLAGS = { + # Read + execute + b'.init': R | E, + b'.plt': R | E, + b'.plt.got': R | E, + b'.plt.sec': R | E, + b'.text': R | E, + b'.fini': R | E, + # Read-only data + b'.interp': R, + b'.note.gnu.property': R, + b'.note.gnu.build-id': R, + b'.note.ABI-tag': R, + b'.gnu.hash': R, + b'.dynsym': R, + b'.dynstr': R, + b'.gnu.version': R, + b'.gnu.version_r': R, + b'.rela.dyn': R, + b'.rela.plt': R, + b'.rodata': R, + b'.eh_frame_hdr': R, + b'.eh_frame': R, + b'.qtmetadata': R, + b'.gcc_except_table': R, + b'.stapsdt.base': R, + # Writable data + b'.init_array': R | W, + b'.fini_array': R | W, + b'.dynamic': R | W, + b'.got': R | W, + b'.data': R | W, + b'.bss': R | W, + } + if elf.hdr.e_machine == pixie.EM_PPC64: + # .plt is RW on ppc64 even with separate-code + EXPECTED_FLAGS[b'.plt'] = R | W + # For all LOAD program headers get mapping to the list of sections, + # and for each section, remember the flags of the associated program header. + flags_per_section = {} + for ph in elf.program_headers: + if ph.p_type == pixie.PT_LOAD: + for section in ph.sections: + assert(section.name not in flags_per_section) + flags_per_section[section.name] = ph.p_flags + # Spot-check ELF LOAD program header flags per section + # If these sections exist, check them against the expected R/W/E flags + for (section, flags) in flags_per_section.items(): + if section in EXPECTED_FLAGS: + if EXPECTED_FLAGS[section] != flags: + return False + return True + +def check_PE_DYNAMIC_BASE(executable) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - (arch,bits) = get_PE_dll_characteristics(executable) - reqbits = IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE - return (bits & reqbits) == reqbits + binary = lief.parse(executable) + return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists -# On 64 bit, must support high-entropy 64-bit address space layout randomization in addition to DYNAMIC_BASE -# to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(executable): +# Must support high-entropy 64-bit address space layout randomization +# in addition to DYNAMIC_BASE to have secure ASLR. +def check_PE_HIGH_ENTROPY_VA(executable) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - (arch,bits) = get_PE_dll_characteristics(executable) - if arch == 'i386:x86-64': - reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA - else: # Unnecessary on 32-bit - assert(arch == 'i386') - reqbits = 0 - return (bits & reqbits) == reqbits - -def check_PE_NX(executable): - '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' - (arch,bits) = get_PE_dll_characteristics(executable) - return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT + binary = lief.parse(executable) + return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists + +def check_PE_RELOC_SECTION(executable) -> bool: + '''Check for a reloc section. This is required for functional ASLR.''' + binary = lief.parse(executable) + return binary.has_relocations + +def check_MACHO_NOUNDEFS(executable) -> bool: + ''' + Check for no undefined references. + ''' + binary = lief.parse(executable) + return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) + +def check_MACHO_LAZY_BINDINGS(executable) -> bool: + ''' + Check for no lazy bindings. + We don't use or check for MH_BINDATLOAD. See #18295. + ''' + binary = lief.parse(executable) + return binary.dyld_info.lazy_bind == (0,0) + +def check_MACHO_Canary(executable) -> bool: + ''' + Check for use of stack canary + ''' + binary = lief.parse(executable) + return binary.has_symbol('___stack_chk_fail') + +def check_PIE(executable) -> bool: + ''' + Check for position independent executable (PIE), + allowing for address space randomization. + ''' + binary = lief.parse(executable) + return binary.is_pie + +def check_NX(executable) -> bool: + ''' + Check for no stack execution + ''' + binary = lief.parse(executable) + return binary.has_nx + +def check_control_flow(executable) -> bool: + ''' + Check for control flow instrumentation + ''' + binary = lief.parse(executable) + + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + CHECKS = { 'ELF': [ ('PIE', check_ELF_PIE), ('NX', check_ELF_NX), ('RELRO', check_ELF_RELRO), - ('Canary', check_ELF_Canary) + ('Canary', check_ELF_Canary), + ('separate_code', check_ELF_separate_code), ], 'PE': [ + ('PIE', check_PIE), ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_PE_NX) + ('NX', check_NX), + ('RELOC_SECTION', check_PE_RELOC_SECTION) +], +'MACHO': [ + ('PIE', check_PIE), + ('NOUNDEFS', check_MACHO_NOUNDEFS), + ('NX', check_NX), + ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), + ('Canary', check_MACHO_Canary), + ('CONTROL_FLOW', check_control_flow), ] } -def identify_executable(executable): +def identify_executable(executable) -> Optional[str]: with open(filename, 'rb') as f: magic = f.read(4) if magic.startswith(b'MZ'): return 'PE' elif magic.startswith(b'\x7fELF'): return 'ELF' + elif magic.startswith(b'\xcf\xfa'): + return 'MACHO' return None if __name__ == '__main__': - retval = 0 + retval: int = 0 for filename in sys.argv[1:]: try: etype = identify_executable(filename) if etype is None: - print('%s: unknown format' % filename) + print(f'{filename}: unknown format') retval = 1 continue - failed = [] - warning = [] + failed: List[str] = [] for (name, func) in CHECKS[etype]: if not func(filename): - if name in NONFATAL: - warning.append(name) - else: - failed.append(name) + failed.append(name) if failed: - print('%s: failed %s' % (filename, ' '.join(failed))) + print(f'{filename}: failed {" ".join(failed)}') retval = 1 - if warning: - print('%s: warning %s' % (filename, ' '.join(warning))) except IOError: - print('%s: cannot open' % filename) + print(f'{filename}: cannot open') retval = 1 sys.exit(retval) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 3e6432b387..61f727fa63 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -3,18 +3,21 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -A script to check that the (Linux) executables produced by gitian only contain -allowed gcc, glibc and libstdc++ version symbols. This makes sure they are -still compatible with the minimum supported Linux distribution versions. +A script to check that release executables only contain certain symbols +and are only linked against allowed libraries. Example usage: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess -import re import sys -import os +from typing import List, Optional + +import lief +import pixie + +from utils import determine_wellknown_cmd # Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases # @@ -39,8 +42,16 @@ # MAX_VERSIONS = { 'GCC': (4,8,0), -'GLIBC': (2,17), -'LIBATOMIC': (1,0) +'GLIBC': { + pixie.EM_386: (2,17), + pixie.EM_X86_64: (2,17), + pixie.EM_ARM: (2,17), + pixie.EM_AARCH64:(2,17), + pixie.EM_PPC64: (2,17), + pixie.EM_RISCV: (2,27), +}, +'LIBATOMIC': (1,0), +'V': (0,5,0), # xkb (bitcoin-qt only) } # See here for a description of _IO_stdin_used: # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 @@ -48,17 +59,15 @@ # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { '_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', -'environ', '_environ', '__environ' +'environ', '_environ', '__environ', } -READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') -CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') + # Allowed NEEDED libraries -ALLOWED_LIBRARIES = { +ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading -'libanl.so.1', # DNS resolve 'libm.so.6', # math library 'librt.so.1', # real-time (clock) 'libatomic.so.1', @@ -66,23 +75,64 @@ 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker +'ld64.so.1', # POWER64 ABIv1 dynamic linker +'ld64.so.2', # POWER64 ABIv2 dynamic linker 'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker # bitcoin-qt only -'libX11-xcb.so.1', # part of X11 -'libX11.so.6', # part of X11 'libxcb.so.1', # part of X11 +'libxkbcommon.so.0', # keyboard keymapping +'libxkbcommon-x11.so.0', # keyboard keymapping 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing 'libdl.so.2' # programming interface to dynamic linker } -ARCH_MIN_GLIBC_VER = { -'80386': (2,1), -'X86-64': (2,2,5), -'ARM': (2,4), -'AArch64':(2,17), -'RISC-V': (2,27) +MACHO_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libc++.1.dylib', # C++ Standard Library +'libSystem.B.dylib', # libc, libm, libpthread, libinfo +# bitcoin-qt only +'AppKit', # user interface +'ApplicationServices', # common application tasks. +'Carbon', # deprecated c back-compat API +'CoreFoundation', # low level func, data types +'CoreGraphics', # 2D rendering +'CoreServices', # operating system services +'CoreText', # interface for laying out text and handling fonts. +'CoreVideo', # video processing +'Foundation', # base layer functionality for apps/frameworks +'ImageIO', # read and write image file formats. +'IOKit', # user-space access to hardware devices and drivers. +'IOSurface', # cross process image/drawing buffers +'libobjc.A.dylib', # Objective-C runtime library +'Metal', # 3D graphics +'Security', # access control and authentication +'QuartzCore', # animation +} + +PE_ALLOWED_LIBRARIES = { +'ADVAPI32.dll', # security & registry +'IPHLPAPI.DLL', # IP helper API +'KERNEL32.dll', # win32 base APIs +'msvcrt.dll', # C standard library for MSVC +'SHELL32.dll', # shell API +'USER32.dll', # user interface +'WS2_32.dll', # sockets +# bitcoin-qt only +'dwmapi.dll', # desktop window manager +'GDI32.dll', # graphics device interface +'IMM32.dll', # input method editor +'NETAPI32.dll', +'ole32.dll', # component object model +'OLEAUT32.dll', # OLE Automation API +'SHLWAPI.dll', # light weight shell API +'USERENV.dll', +'UxTheme.dll', +'VERSION.dll', # version checking +'WINMM.dll', # WinMM audio API +'WTSAPI32.dll', } + class CPPFilt(object): ''' Demangle C++ symbol names. @@ -90,7 +140,7 @@ class CPPFilt(object): Use a pipe to the 'c++filt' command. ''' def __init__(self): - self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + self.proc = subprocess.Popen(determine_wellknown_cmd('CPPFILT', 'c++filt'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) def __call__(self, mangled): self.proc.stdin.write(mangled + '\n') @@ -102,30 +152,7 @@ def close(self): self.proc.stdout.close() self.proc.wait() -def read_symbols(executable, imports=True): - ''' - Parse an ELF executable and return a list of (symbol,version) tuples - for dynamic, imported symbols. - ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) - syms = [] - for line in stdout.splitlines(): - line = line.split() - if 'Machine:' in line: - arch = line[-1] - if len(line)>7 and re.match('[0-9]+:$', line[0]): - (sym, _, version) = line[7].partition('@') - is_import = line[6] == 'UND' - if version.startswith('@'): - version = version[1:] - if is_import == imports: - syms.append((sym, version, arch)) - return syms - -def check_version(max_versions, version, arch): +def check_version(max_versions, version, arch) -> bool: if '_' in version: (lib, _, ver) = version.rpartition('_') else: @@ -134,46 +161,135 @@ def check_version(max_versions, version, arch): ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False - return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch] - -def read_libraries(filename): - p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - libraries = [] - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens)>2 and tokens[1] == '(NEEDED)': - match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) - if match: - libraries.append(match.group(1)) - else: - raise ValueError('Unparseable (NEEDED) specification') - return libraries + if isinstance(max_versions[lib], tuple): + return ver <= max_versions[lib] + else: + return ver <= max_versions[lib][arch] -if __name__ == '__main__': +def check_imported_symbols(filename) -> bool: + elf = pixie.load(filename) + cppfilt = CPPFilt() + ok: bool = True + + for symbol in elf.dyn_symbols: + if not symbol.is_import: + continue + sym = symbol.name.decode() + version = symbol.version.decode() if symbol.version is not None else None + if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine): + print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) + ok = False + return ok + +def check_exported_symbols(filename) -> bool: + elf = pixie.load(filename) cppfilt = CPPFilt() - retval = 0 + ok: bool = True + for symbol in elf.dyn_symbols: + if not symbol.is_export: + continue + sym = symbol.name.decode() + if elf.hdr.e_machine == pixie.EM_RISCV or sym in IGNORE_EXPORTS: + continue + print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) + ok = False + return ok + +def check_ELF_libraries(filename) -> bool: + ok: bool = True + elf = pixie.load(filename) + for library_name in elf.query_dyn_tags(pixie.DT_NEEDED): + assert(isinstance(library_name, bytes)) + if library_name.decode() not in ELF_ALLOWED_LIBRARIES: + print('{}: NEEDED library {} is not allowed'.format(filename, library_name.decode())) + ok = False + return ok + +def check_MACHO_libraries(filename) -> bool: + ok: bool = True + binary = lief.parse(filename) + for dylib in binary.libraries: + split = dylib.name.split('/') + if split[-1] not in MACHO_ALLOWED_LIBRARIES: + print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_min_os(filename) -> bool: + binary = lief.parse(filename) + if binary.build_version.minos == [10,14,0]: + return True + return False + +def check_MACHO_sdk(filename) -> bool: + binary = lief.parse(filename) + if binary.build_version.sdk == [10, 15, 6]: + return True + return False + +def check_PE_libraries(filename) -> bool: + ok: bool = True + binary = lief.parse(filename) + for dylib in binary.libraries: + if dylib not in PE_ALLOWED_LIBRARIES: + print(f'{dylib} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_PE_subsystem_version(filename) -> bool: + binary = lief.parse(filename) + major: int = binary.optional_header.major_subsystem_version + minor: int = binary.optional_header.minor_subsystem_version + if major == 6 and minor == 1: + return True + return False + +CHECKS = { +'ELF': [ + ('IMPORTED_SYMBOLS', check_imported_symbols), + ('EXPORTED_SYMBOLS', check_exported_symbols), + ('LIBRARY_DEPENDENCIES', check_ELF_libraries) +], +'MACHO': [ + ('DYNAMIC_LIBRARIES', check_MACHO_libraries), + ('MIN_OS', check_MACHO_min_os), + ('SDK', check_MACHO_sdk), +], +'PE' : [ + ('DYNAMIC_LIBRARIES', check_PE_libraries), + ('SUBSYSTEM_VERSION', check_PE_subsystem_version), +] +} + +def identify_executable(executable) -> Optional[str]: + with open(filename, 'rb') as f: + magic = f.read(4) + if magic.startswith(b'MZ'): + return 'PE' + elif magic.startswith(b'\x7fELF'): + return 'ELF' + elif magic.startswith(b'\xcf\xfa'): + return 'MACHO' + return None + +if __name__ == '__main__': + retval: int = 0 for filename in sys.argv[1:]: - # Check imported symbols - for sym,version,arch in read_symbols(filename, True): - if version and not check_version(MAX_VERSIONS, version, arch): - print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) - retval = 1 - # Check exported symbols - if arch != 'RISC-V': - for sym,version,arch in read_symbols(filename, False): - if sym in IGNORE_EXPORTS: - continue - print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) - retval = 1 - # Check dependency libraries - for library_name in read_libraries(filename): - if library_name not in ALLOWED_LIBRARIES: - print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + try: + etype = identify_executable(filename) + if etype is None: + print(f'{filename}: unknown format') retval = 1 + continue + failed: List[str] = [] + for (name, func) in CHECKS[etype]: + if not func(filename): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + except IOError: + print(f'{filename}: cannot open') + retval = 1 sys.exit(retval) - - diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 18f9835faa..14058e2cc8 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,16 +1,18 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015-2016 The Bitcoin Core developers +#!/usr/bin/env python3 +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Test script for security-check.py ''' -from __future__ import division,print_function +import os import subprocess import unittest +from utils import determine_wellknown_cmd + def write_testcode(filename): - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: f.write(''' #include int main() @@ -20,43 +22,80 @@ def write_testcode(filename): } ''') +def clean_files(source, executable): + os.remove(source) + os.remove(executable) + def call_security_check(cc, source, executable, options): - subprocess.check_call([cc,source,'-o',executable] + options) - p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (stdout, stderr) = p.communicate() - return (p.returncode, stdout.rstrip()) + subprocess.run([*cc,source,'-o',executable] + options, check=True) + p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + return (p.returncode, p.stdout.rstrip()) class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' - cc = 'gcc' + cc = determine_wellknown_cmd('CC', 'gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), (0, '')) + clean_files(source, executable) + def test_PE(self): source = 'test1.c' executable = 'test1.exe' - cc = 'i686-w64-mingw32-gcc' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, []), - (1, executable+': failed PIE NX')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), - (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), - (0, '')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + (0, '')) + + clean_files(source, executable) + + def test_MACHO(self): + source = 'test1.c' + executable = 'test1' + cc = determine_wellknown_cmd('CC', 'clang') + write_testcode(source) + + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), + (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), + (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), + (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (1, executable+': failed PIE')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (0, '')) + + clean_files(source, executable) if __name__ == '__main__': unittest.main() - diff --git a/contrib/devtools/utils.py b/contrib/devtools/utils.py new file mode 100755 index 0000000000..68ad1c3aba --- /dev/null +++ b/contrib/devtools/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Common utility functions +''' +import shutil +import sys +import os +from typing import List + + +def determine_wellknown_cmd(envvar, progname) -> List[str]: + maybe_env = os.getenv(envvar) + maybe_which = shutil.which(progname) + if maybe_env: + return maybe_env.split(' ') # Well-known vars are often meant to be word-split + elif maybe_which: + return [ maybe_which ] + else: + sys.exit(f"{progname} not found") diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME new file mode 100644 index 0000000000..b94413eed1 --- /dev/null +++ b/contrib/gitian-descriptors/assign_DISTNAME @@ -0,0 +1,12 @@ +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# A helper script to be sourced into the gitian descriptors + +if RECENT_TAG="$(git describe --exact-match HEAD 2> /dev/null)"; then + VERSION="${RECENT_TAG#v}" +else + VERSION="$(git rev-parse --short=12 HEAD)" +fi +DISTNAME="firo-${VERSION}" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index ebb3553766..fa50ac3a41 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,59 +1,64 @@ --- name: "firo-linux-0.14.0" enable_cache: true +distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: -- "curl" -- "g++-aarch64-linux-gnu" -- "g++-8-aarch64-linux-gnu" -- "gcc-8-aarch64-linux-gnu" -- "binutils-aarch64-linux-gnu" -- "g++-arm-linux-gnueabihf" -- "g++-8-arm-linux-gnueabihf" -- "gcc-8-arm-linux-gnueabihf" -- "binutils-arm-linux-gnueabihf" -- "g++-riscv64-linux-gnu" -- "g++-8-riscv64-linux-gnu" -- "gcc-8-riscv64-linux-gnu" -- "binutils-riscv64-linux-gnu" -- "g++-8-multilib" -- "gcc-8-multilib" -- "binutils-gold" -- "git" -- "pkg-config" +# Common dependencies. - "autoconf" -- "libtool" - "automake" -- "faketime" +- "binutils" +- "bison" - "bsdmainutils" - "ca-certificates" -- "python" +- "curl" +- "faketime" +- "g++-8" +- "gcc-8" +- "git" +- "libtool" +- "patch" +- "pkg-config" +- "python3" +- "python3-pip" - "cmake" +# Cross compilation HOSTS: +# - arm-linux-gnueabihf +# - "binutils-arm-linux-gnueabihf" +# - "g++-8-arm-linux-gnueabihf" +# - aarch64-linux-gnu +# - "binutils-aarch64-linux-gnu" +# - "g++-8-aarch64-linux-gnu" +# - powerpc64-linux-gnu +# - "binutils-powerpc64-linux-gnu" +# - "g++-8-powerpc64-linux-gnu" +# - powerpc64le-linux-gnu +# - "binutils-powerpc64le-linux-gnu" +# - "g++-8-powerpc64le-linux-gnu" +# - riscv64-linux-gnu +# - "binutils-riscv64-linux-gnu" +# - "g++-8-riscv64-linux-gnu" remotes: - "url": "https://github.com/firoorg/firo" "dir": "firo" files: [] script: | + set -e -o pipefail WRAP_DIR=$HOME/wrapped - # HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" HOSTS="x86_64-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --enable-crash-hooks --enable-elysium" + CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary --enable-elysium" FAKETIME_HOST_PROGS="gcc g++" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS=-static-libstdc++ + HOST_LDFLAGS_BASE="-static-libstdc++ -Wl,-O2" - export QT_RCC_TEST=1 - export QT_RCC_SOURCE_DATE_OVERRIDE=1 - export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -61,13 +66,14 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -79,60 +85,34 @@ script: | then echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + if [ "${i:0:11}" = "powerpc64le" ]; then + echo "exec \"\$REAL\" -mcpu=power8 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} + elif [ "${i:0:9}" = "powerpc64" ]; then + echo "exec \"\$REAL\" -mcpu=970 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} + else + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} + fi + chmod +x ${WRAP_DIR}/${i}-${prog} fi done done } + pip3 install lief==0.11.5 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} - EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes - mkdir -p $EXTRA_INCLUDES_BASE - - # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, - # but we can't write there. Instead, create a link here and force it to be included in the - # search paths by wrapping gcc/g++. - - mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu - rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm - ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm - - for prog in gcc g++; do - rm -f ${WRAP_DIR}/${prog} - cat << EOF > ${WRAP_DIR}/${prog} - #!/usr/bin/env bash - REAL="`which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1`" - for var in "\$@" - do - if [ "\$var" = "-m32" ]; then - export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" - export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" - break - fi - done - \$REAL \$@ - EOF - chmod +x ${WRAP_DIR}/${prog} - done - cd firo - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do - EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" - if [ -d "$EXTRA_INCLUDES" ]; then - export HOST_ID_SALT="$EXTRA_INCLUDES" - fi - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - unset HOST_ID_SALT + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" CC=${i}-gcc-8 CXX=${i}-g++-8 done # Faketime for binaries @@ -141,55 +121,48 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=`echo firo-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` - # Correct tar file order - mkdir -p temp - pushd temp - tar xf ../$SOURCEDIST - find firo-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST - popd + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" + + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + if [ "${i}" = "powerpc64-linux-gnu" ]; then + # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 + # TODO: remove this when no longer needed + HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" + else + HOST_LDFLAGS="${HOST_LDFLAGS_BASE}" + fi mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST + tar --strip-components=1 -xf "${GIT_ARCHIVE}" - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ - - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --enable-elysium --enable-clientapi --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" + ./autogen.sh + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" CC=${i}-gcc-8 CXX=${i}-g++-8 make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make install DESTDIR=${INSTALLPATH} cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - cp -r ${DISTNAME}/bin ${OUTDIR}/bin_debug - find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; - find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; - cp ../doc/README.md ${DISTNAME}/ - find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz + find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + cp ../README.md ${DISTNAME}/ + find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz + find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ rm -rf distsrc-${i} done - mkdir -p $OUTDIR/src - mv $SOURCEDIST $OUTDIR/src diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 9009225fa9..f0b5320523 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -1,37 +1,53 @@ --- name: "firo-dmg-signer" +distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: - "faketime" +- "xorriso" +- "python3-pip" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" +- "url": "https://github.com/achow101/signapple.git" + "dir": "signapple" + "commit": "b084cbbf44d5330448ffce0c7d118f75781b64bd" files: -- "bitcoin-osx-unsigned.tar.gz" +- "firo-osx-unsigned.tar.gz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} - export PATH=`pwd`:$PATH - FAKETIME_PROGS="dmg genisoimage" + export PATH="$PWD":$PATH + FAKETIME_PROGS="dmg xorrisofs" # Create global faketime wrappers for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done - UNSIGNED=firo-osx-unsigned.tar.gz + # Install signapple + cd signapple + python3 -m pip install -U pip setuptools + python3 -m pip install . + export PATH="$HOME/.local/bin":$PATH + cd .. + + UNSIGNED_TARBALL=firo-osx-unsigned.tar.gz + UNSIGNED_APP=dist/Firo-Qt.app SIGNED=firo-osx-signed.dmg - tar -xf ${UNSIGNED} + tar -xf ${UNSIGNED_TARBALL} OSX_VOLNAME="$(cat osx_volname)" - ./detached-sig-apply.sh ${UNSIGNED} signature/osx - ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app + ./detached-sig-apply.sh ${UNSIGNED_APP} signature/osx/dist + ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 9b1b73d02d..2347fd6025 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,8 +1,9 @@ --- name: "firo-osx-0.14.0" enable_cache: true +distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -20,31 +21,29 @@ packages: - "bsdmainutils" - "cmake" - "imagemagick" -- "libcap-dev" - "libz-dev" -- "libbz2-dev" -- "python" -- "python-dev" -- "python-setuptools" +- "python3" +- "python3-pip" +- "python3-setuptools" - "fonts-tuffy" -- "cmake" +- "xorriso" +- "libtinfo5" remotes: - "url": "https://github.com/firoorg/firo" "dir": "firo" files: -- "MacOSX10.14.sdk.tar.gz" +- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin14" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks --enable-elysium" + HOSTS="x86_64-apple-darwin18" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary --enable-elysium XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg genisoimage" + FAKETIME_PROGS="ar ranlib date dmg xorrisofs" - export QT_RCC_TEST=1 - export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -54,13 +53,14 @@ script: | export ZERO_AR_DATE=1 + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -70,14 +70,16 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } + pip3 install lief==0.11.5 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" @@ -85,10 +87,10 @@ script: | export PATH=${WRAP_DIR}:${PATH} cd firo - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz # Build dependencies for each host for i in $HOSTS; do @@ -101,71 +103,53 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=`echo firo-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME - # Correct tar file order - mkdir -p temp - pushd temp - tar xf ../$SOURCEDIST - find firo-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST - popd + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST + tar --strip-components=1 -xf "${GIT_ARCHIVE}" - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ - - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --enable-elysium --enable-clientapi --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + ./autogen.sh + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} - make -C src osx_debug + make ${MAKEOPTS} -C src check-security + make ${MAKEOPTS} -C src check-symbols make install-strip DESTDIR=${INSTALLPATH} make osx_volname make deploydir - OSX_VOLNAME="$(cat osx_volname)" mkdir -p unsigned-app-${i} cp osx_volname unsigned-app-${i}/ cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate - cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff + cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} mv dist unsigned-app-${i} pushd unsigned-app-${i} - find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz + find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz popd - make deploy - ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg + make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find .. -name *.dSYM -exec cp -ra {} ${DISTNAME}/bin \; - find ${DISTNAME} -not -path '*.dSYM*' | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - find ${DISTNAME} -path '*.dSYM*' | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz + find ${DISTNAME} | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz cd ../../ done - mkdir -p $OUTDIR/src - mv $SOURCEDIST $OUTDIR/src - mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin16.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz - mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin16-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz + + mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 14587850e9..e165f9f4a6 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -1,39 +1,42 @@ --- name: "firo-win-signer" +distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: -# Once osslsigncode supports openssl 1.1, we can change this back to libssl-dev -- "libssl1.0-dev" +- "libssl-dev" - "autoconf" +- "automake" +- "libtool" +- "pkg-config" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" files: -- "osslsigncode-1.7.1.tar.gz" -- "osslsigncode-Backports-to-1.7.1.patch" +- "osslsigncode-2.0.tar.gz" - "firo-win-unsigned.tar.gz" script: | - BUILD_DIR=`pwd` + set -e -o pipefail + + BUILD_DIR="$PWD" SIGDIR=${BUILD_DIR}/signature/win UNSIGNED_DIR=${BUILD_DIR}/unsigned - echo "f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 osslsigncode-1.7.1.tar.gz" | sha256sum -c - echo "a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 osslsigncode-Backports-to-1.7.1.patch" | sha256sum -c + echo "5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f osslsigncode-2.0.tar.gz" | sha256sum -c mkdir -p ${UNSIGNED_DIR} tar -C ${UNSIGNED_DIR} -xf firo-win-unsigned.tar.gz - tar xf osslsigncode-1.7.1.tar.gz - cd osslsigncode-1.7.1 - patch -p1 < ${BUILD_DIR}/osslsigncode-Backports-to-1.7.1.patch + tar xf osslsigncode-2.0.tar.gz + cd osslsigncode-2.0 + ./autogen.sh ./configure --without-gsf --without-curl --disable-dependency-tracking make find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do - INFILE="`basename "${i}"`" - OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`" + INFILE="$(basename "${i}")" + OUTFILE="${INFILE/-unsigned}" ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem" done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 7eac2c2fca..cd016994c3 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,8 +1,9 @@ --- name: "firo-win-0.14.0" enable_cache: true +distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -20,29 +21,26 @@ packages: - "nsis" - "zip" - "ca-certificates" -- "python" -- "rename" +- "python3" +- "python3-pip" - "cmake" remotes: - "url": "https://github.com/firoorg/firo" "dir": "firo" files: [] script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped - # HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" HOSTS="x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --enable-crash-hooks --enable-elysium" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary --enable-elysium" FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" + HOST_CFLAGS="-O2 -g -fno-ident" + HOST_CXXFLAGS="-O2 -g -fno-ident" - export QT_RCC_TEST=1 - export QT_RCC_SOURCE_DATE_OVERRIDE=1 - export GZIP="-9n" - export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -50,13 +48,14 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -66,47 +65,39 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } - function create_per-host_linker_wrapper { - # This is only needed for trusty, as the mingw linker leaks a few bytes of - # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 + function create_per-host_compiler_wrapper { + # -posix variant is required for c++11 threading. for i in $HOSTS; do - mkdir -p ${WRAP_DIR}/${i} - for prog in collect2; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} - REAL=$(${i}-gcc -print-prog-name=${prog}) - echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} - echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} - chmod +x ${WRAP_DIR}/${i}/${prog} - done for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } + pip3 install lief==0.11.5 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_linker_wrapper "2000-01-01 12:00:00" + create_per-host_compiler_wrapper "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} cd firo - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" @@ -116,69 +107,52 @@ script: | export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_linker_wrapper "${REFERENCE_DATETIME}" + create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=`echo firo-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME - # Correct tar file order - mkdir -p temp - pushd temp - tar xf ../$SOURCEDIST - find firo-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST - mkdir -p $OUTDIR/src - cp ../$SOURCEDIST $OUTDIR/src - popd + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST + tar --strip-components=1 -xf "${GIT_ARCHIVE}" - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ - - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --enable-elysium --enable-clientapi --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" + ./autogen.sh + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security - make deploy + make ${MAKEOPTS} -C src check-symbols + make deploy BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" make install DESTDIR=${INSTALLPATH} - rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe - cp -f firo-*setup*.exe $OUTDIR/ cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - cp -r ${DISTNAME}/bin ${OUTDIR}/bin_debug - find ${DISTNAME}/bin -type f -executable -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; - find ${DISTNAME}/lib -type f -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; - find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip - find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip + find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + cp ../doc/README_windows.txt ${DISTNAME}/readme.txt + find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i//x86_64-w64-mingw32/win64}.zip + find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i//x86_64-w64-mingw32/win64}-debug.zip cd ../../ rm -rf distsrc-${i} done + cp -rf contrib/windeploy $BUILD_DIR cd $BUILD_DIR/windeploy mkdir unsigned - cp $OUTDIR/firo-*setup-unsigned.exe unsigned/ - find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz - mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip - # mv ${OUTDIR}/${DISTNAME}-i686-*-debug.zip ${OUTDIR}/${DISTNAME}-win32-debug.zip - mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip - # mv ${OUTDIR}/${DISTNAME}-i686-*.zip ${OUTDIR}/${DISTNAME}-win32.zip + cp ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe unsigned/ + find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md new file mode 100644 index 0000000000..68aae18731 --- /dev/null +++ b/contrib/guix/INSTALL.md @@ -0,0 +1,801 @@ +# Guix Installation and Setup + +This only needs to be done once per machine. If you have already completed the +installation and setup, please proceed to [perform a build](./README.md). + +Otherwise, you may choose from one of the following options to install Guix: + +1. Using the official **shell installer script** [⤓ skip to section][install-script] + - Maintained by Guix developers + - Easiest (automatically performs *most* setup) + - Works on nearly all Linux distributions + - Only installs latest release + - Binary installation only, requires high level of trust + - Note: The script needs to be run as root, so it should be inspected before it's run +2. Using the official **binary tarball** [⤓ skip to section][install-bin-tarball] + - Maintained by Guix developers + - Normal difficulty (full manual setup required) + - Works on nearly all Linux distributions + - Installs any release + - Binary installation only, requires high level of trust +3. Using fanquake's **Docker image** [↗︎ external instructions][install-fanquake-docker] + - Maintained by fanquake + - Easy (automatically performs *some* setup) + - Works wherever Docker images work + - Installs any release + - Binary installation only, requires high level of trust +4. Using a **distribution-maintained package** [⤓ skip to section][install-distro-pkg] + - Maintained by distribution's Guix package maintainer + - Normal difficulty (manual setup required) + - Works only on distributions with Guix packaged, see: https://repology.org/project/guix/versions + - Installs a release decided on by package maintainer + - Source or binary installation depending on the distribution +5. Building **from source** [⤓ skip to section][install-source] + - Maintained by you + - Hard, but rewarding + - Can be made to work on most Linux distributions + - Installs any commit (more granular) + - Source installation, requires lower level of trust + +## Options 1 and 2: Using the official shell installer script or binary tarball + +The installation instructions for both the official shell installer script and +the binary tarballs can be found in the GNU Guix Manual's [Binary Installation +section](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html). + +Note that running through the binary tarball installation steps is largely +equivalent to manually performing what the shell installer script does. + +Note that at the time of writing (July 5th, 2021), the shell installer script +automatically creates an `/etc/profile.d` entry which the binary tarball +installation instructions do not ask you to create. However, you will likely +need this entry for better desktop integration. Please see [this +section](#add-an-etcprofiled-entry) for instructions on how to add a +`/etc/profile.d/guix.sh` entry. + +Regardless of which installation option you chose, the changes to +`/etc/profile.d` will not take effect until the next shell or desktop session, +so you should log out and log back in. + +## Option 3: Using fanquake's Docker image + +Please refer to fanquake's instructions +[here](https://github.com/fanquake/core-review/tree/master/guix). + +Note that the `Dockerfile` is largely equivalent to running through the binary +tarball installation steps. + +## Option 4: Using a distribution-maintained package + +Note that this section is based on the distro packaging situation at the time of +writing (July 2021). Guix is expected to be more widely packaged over time. For +an up-to-date view on Guix's package status/version across distros, please see: +https://repology.org/project/guix/versions + +### Debian 11 (Bullseye)/Ubuntu 21.04 (Hirsute Hippo) + +Guix v1.2.0 is available as a distribution package starting in [Debian +11](https://packages.debian.org/bullseye/guix) and [Ubuntu +21.04](https://packages.ubuntu.com/hirsute/guix). + +Note that if you intend on using Guix without using any substitutes (more +details [here][security-model]), v1.2.0 has a known problem when building GnuTLS +from source. Solutions and workarounds are documented +[here](#gnutls-test-suite-fail-status-request-revoked). + + +To install: +```sh +sudo apt install guix +``` + +For up-to-date information on Debian and Ubuntu's release history: +- [Debian release history](https://www.debian.org/releases/) +- [Ubuntu release history](https://ubuntu.com/about/release-cycle) + +### Arch Linux + +Guix is available in the AUR as +[`guix`](https://aur.archlinux.org/packages/guix/), please follow the +installation instructions in the Arch Linux Wiki ([live +link](https://wiki.archlinux.org/index.php/Guix#AUR_Package_Installation), +[2021/03/30 +permalink](https://wiki.archlinux.org/index.php?title=Guix&oldid=637559#AUR_Package_Installation)) +to install Guix. + +At the time of writing (2021/03/30), the `check` phase will fail if the path to +guix's build directory is longer than 36 characters due to an anachronistic +character limit on the shebang line. Since the `check` phase happens after the +`build` phase, which may take quite a long time, it is recommended that users +either: + +1. Skip the `check` phase + - For `makepkg`: `makepkg --nocheck ...` + - For `yay`: `yay --mflags="--nocheck" ...` + - For `paru`: `paru --nocheck ...` +2. Or, check their build directory's length beforehand + - For those building with `makepkg`: `pwd | wc -c` + +## Option 5: Building from source + +Building Guix from source is a rather involved process but a rewarding one for +those looking to minimize trust and maximize customizability (e.g. building a +particular commit of Guix). Previous experience with using autotools-style build +systems to build packages from source will be helpful. *hic sunt dracones.* + +I strongly urge you to at least skim through the entire section once before you +start issuing commands, as it will save you a lot of unnecessary pain and +anguish. + +### Installing common build tools + +There are a few basic build tools that are required for most things we'll build, +so let's install them now: + +Text transformation/i18n: +- `autopoint` (sometimes packaged in `gettext`) +- `help2man` +- `po4a` +- `texinfo` + +Build system tools: +- `g++` w/ C++11 support +- `libtool` +- `autoconf` +- `automake` +- `pkg-config` (sometimes packaged as `pkgconf`) +- `make` +- `cmake` + +Miscellaneous: +- `git` +- `gnupg` +- `python3` + +### Building and Installing Guix's dependencies + +In order to build Guix itself from source, we need to first make sure that the +necessary dependencies are installed and discoverable. The most up-to-date list +of Guix's dependencies is kept in the ["Requirements" +section](https://guix.gnu.org/manual/en/html_node/Requirements.html) of the Guix +Reference Manual. + +Depending on your distribution, most or all of these dependencies may already be +packaged and installable without manually building and installing. + +For reference, the graphic below outlines Guix v1.3.0's dependency graph: + +![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) + +#### Guile + +##### Choosing a Guile version and sticking to it + +One of the first things you need to decide is which Guile version you want to +use: Guile v2.2 or Guile v3.0. Unlike the python2 to python3 transition, Guile +v2.2 and Guile v3.0 are largely compatible, as evidenced by the fact that most +Guile packages and even [Guix +itself](https://guix.gnu.org/en/blog/2020/guile-3-and-guix/) support running on +both. + +What is important here is that you **choose one**, and you **remain consistent** +with your choice throughout **all Guile-related packages**, no matter if they +are installed via the distribution's package manager or installed from source. +This is because the files for Guile packages are installed to directories which +are separated based on the Guile version. + +###### Example: Checking that Ubuntu's `guile-git` is compatible with your chosen Guile version + +On Ubuntu Focal: + +```sh +$ apt show guile-git +Package: guile-git +... +Depends: guile-2.2, guile-bytestructures, libgit2-dev +... +``` + +As you can see, the package `guile-git` depends on `guile-2.2`, meaning that it +was likely built for Guile v2.2. This means that if you decided to use Guile +v3.0 on Ubuntu Focal, you would need to build guile-git from source instead of +using the distribution package. + +On Ubuntu Hirsute: + +```sh +$ apt show guile-git +Package: guile-git +... +Depends: guile-3.0 | guile-2.2, guile-bytestructures (>= 1.0.7-3~), libgit2-dev (>= 1.0) +... +``` + +In this case, `guile-git` depends on either `guile-3.0` or `guile-2.2`, meaning +that it would work no matter what Guile version you decided to use. + +###### Corner case: Multiple versions of Guile on one system + +It is recommended to only install one version of Guile, so that build systems do +not get confused about which Guile to use. + +However, if you insist on having both Guile v2.2 and Guile v3.0 installed on +your system, then you need to **consistently** specify one of +`GUILE_EFFECTIVE_VERSION=3.0` or `GUILE_EFFECTIVE_VERSION=2.2` to all +`./configure` invocations for Guix and its dependencies. + +##### Installing Guile + +Guile is most likely already packaged for your distribution, so after you have +[chosen a Guile version](#choosing-a-guile-version-and-sticking-to-it), install +it via your distribution's package manager. + +If your distribution splits packages into `-dev`-suffixed and +non-`-dev`-suffixed sub-packages (as is the case for Debian-derived +distributions), please make sure to install both. For example, to install Guile +v2.2 on Debian/Ubuntu: + +```sh +apt install guile-2.2 guile-2.2-dev +``` + +#### Mixing distribution packages and source-built packages + +At the time of writing, most distributions have _some_ of Guix's dependencies +packaged, but not all. This means that you may want to install the distribution +package for some dependencies, and manually build-from-source for others. + +Distribution packages usually install to `/usr`, which is different from the +default `./configure` prefix of source-built packages: `/usr/local`. + +This means that if you mix-and-match distribution packages and source-built +packages and do not specify exactly `--prefix=/usr` to `./configure` for +source-built packages, you will need to augment the `GUILE_LOAD_PATH` and +`GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look +under the right prefix and find your source-built packages. + +For example, if you are using Guile v2.2, and have Guile packages in the +`/usr/local` prefix, either add the following lines to your `.profile` or +`.bash_profile` so that the environment variable is properly set for all future +shell logins, or paste the lines into a POSIX-style shell to temporarily modify +the environment variables of your current shell session. + +```sh +# Help Guile v2.2.x find packages in /usr/local +export GUILE_LOAD_PATH="/usr/local/share/guile/site/2.2${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" +export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/2.2/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" +``` + +Note that these environment variables are used to check for packages during +`./configure`, so they should be set as soon as possible should you want to use +a prefix other than `/usr`. + +#### Building and installing source-built packages + +***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are +documented in the sub-sections immediately below. Please read these sections +before proceeding to build and install these packages.* + +Although you should always refer to the README or INSTALL files for the most +accurate information, most of these dependencies use autoconf-style build +systems (check if there's a `configure.ac` file), and will likely do the right +thing with the following: + +Clone the repository and check out the latest release: +```sh +git clone /.git +cd +git tag -l # check for the latest release +git checkout +``` + +For autoconf-based build systems (if `./autogen.sh` or `configure.ac` exists at +the root of the repository): + +```sh +./autogen.sh || autoreconf -vfi +./configure --prefix= +make +sudo make install +``` + +For CMake-based build systems (if `CMakeLists.txt` exists at the root of the +repository): + +```sh +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX= +sudo cmake --build . --target install +``` + +If you choose not to specify exactly `--prefix=/usr` to `./configure`, please +make sure you've carefully read the [previous section] on mixing distribution +packages and source-built packages. + +##### Binding packages require `-dev`-suffixed packages + +Relevant for: +- Everyone + +When building bindings, the `-dev`-suffixed version of the original package +needs to be installed. For example, building `Guile-zlib` on Debian-derived +distributions requires that `zlib1g-dev` is installed. + +When using bindings, the `-dev`-suffixed version of the original package still +needs to be installed. This is particularly problematic when distribution +packages are mispackaged like `guile-sqlite3` is in Ubuntu Focal such that +installing `guile-sqlite3` does not automatically install `libsqlite3-dev` as a +dependency. + +Below is a list of relevant Guile bindings and their corresponding `-dev` +packages in Debian at the time of writing. + +| Guile binding package | -dev Debian package | +|-----------------------|---------------------| +| guile-gcrypt | libgcrypt-dev | +| guile-git | libgit2-dev | +| guile-lzlib | liblz-dev | +| guile-ssh | libssh-dev | +| guile-sqlite3 | libsqlite3-dev | +| guile-zlib | zlib1g-dev | + +##### `guile-git` actually depends on `libgit2 >= 1.1` + +Relevant for: +- Those building `guile-git` from source against `libgit2 < 1.1` +- Those installing `guile-git` from their distribution where `guile-git` is + built against `libgit2 < 1.1` + +As of v0.4.0, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, +it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a +reference of `origin/keyring`: instead of interpreting the reference as "the +'keyring' branch of the 'origin' remote", the reference is interpreted as "the +branch literally named 'origin/keyring'" + +This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and +`guile-git` is built against it. + +Should you be in this situation, you need to build both `libgit2 v1.1.x` and +`guile-git` from source. + +Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 + +##### `{scheme,guile}-bytestructures` v1.0.8 and v1.0.9 are broken for Guile v2.2 + +Relevant for: +- Those building `{scheme,guile}-bytestructures` from source against Guile v2.2 + +Commit +[707eea3](https://github.com/TaylanUB/scheme-bytestructures/commit/707eea3a85e1e375e86702229ebf73d496377669) +introduced a regression for Guile v2.2 and was first included in v1.0.8, this +was later corrected in commit +[ec9a721](https://github.com/TaylanUB/scheme-bytestructures/commit/ec9a721957c17bcda13148f8faa5f06934431ff7) +and included in v1.1.0. + +TL;DR If you decided to use Guile v2.2, do not use `{scheme,guile}-bytestructures` v1.0.8 or v1.0.9. + +### Building and Installing Guix itself + +Start by cloning Guix: + +``` +git clone https://git.savannah.gnu.org/git/guix.git +cd guix +``` + +You will likely want to build the latest release, however, if the latest release +when you're reading this is still 1.2.0 then you may want to use 95aca29 instead +to avoid a problem in the GnuTLS test suite. + +``` +git branch -a -l 'origin/version-*' # check for the latest release +git checkout +``` + +Bootstrap the build system: +``` +./bootstrap +``` + +Configure with the recommended `--localstatedir` flag: +``` +./configure --localstatedir=/var +``` + +Note: If you intend to hack on Guix in the future, you will need to supply the +same `--localstatedir=` flag for all future Guix `./configure` invocations. See +the last paragraph of this +[section](https://guix.gnu.org/manual/en/html_node/Requirements.html) for more +details. + +Build Guix (this will take a while): +``` +make -j$(nproc) +``` + +Install Guix: + +``` +sudo make install +``` + +### Post-"build from source" Setup + +#### Creating and starting a `guix-daemon-original` service with a fixed `argv[0]` + +At this point, guix will be installed to `${bindir}`, which is likely +`/usr/local/bin` if you did not override directory variables at +`./configure`-time. More information on standard Automake directory variables +can be found +[here](https://www.gnu.org/software/automake/manual/html_node/Standard-Directory-Variables.html). + +However, the Guix init scripts and service configurations for Upstart, systemd, +SysV, and OpenRC are installed (in `${libdir}`) to launch +`${localstatedir}/guix/profiles/per-user/root/current-guix/bin/guix-daemon`, +which does not yet exist, and will only exist after [`root` performs their first +`guix pull`](#guix-pull-as-root). + +We need to create a `-original` version of these init scripts that's pointed to +the binaries we just built and `make install`'ed in `${bindir}` (normally, +`/usr/local/bin`). + +Example for `systemd`, run as `root`: + +```sh +# Create guix-daemon-original.service by modifying guix-daemon.service +libdir=# set according to your PREFIX (default is /usr/local/lib) +bindir="$(dirname $(command -v guix-daemon))" +sed -E -e "s|/\S*/guix/profiles/per-user/root/current-guix/bin/guix-daemon|${bindir}/guix-daemon|" "${libdir}"/systemd/system/guix-daemon.service > /etc/systemd/system/guix-daemon-original.service +chmod 664 /etc/systemd/system/guix-daemon-original.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the non-working guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the working guix-daemon-original.service is started and enabled +systemctl enable guix-daemon-original +systemctl start guix-daemon-original +``` + +#### Creating `guix-daemon` users / groups + +Please see the [relevant +section](https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html) +in the Guix Reference Manual for more details. + +## Optional setup + +At this point, you are set up to [use Guix to build Bitcoin +Core](./README.md#usage). However, if you want to polish your setup a bit and +make it "what Guix intended", then read the next few subsections. + +### Add an `/etc/profile.d` entry + +This section definitely does not apply to you if you installed Guix using: +1. The shell installer script +2. fanquake's Docker image +3. Debian's `guix` package + +#### Background + +Although Guix knows how to update itself and its packages, it does so in a +non-invasive way (it does not modify `/usr/local/bin/guix`). + +Instead, it does the following: + +- After a `guix pull`, it updates + `/var/guix/profiles/per-user/$USER/current-guix`, and creates a symlink + targeting this directory at `$HOME/.config/guix/current` + +- After a `guix install`, it updates + `/var/guix/profiles/per-user/$USER/guix-profile`, and creates a symlink + targeting this directory at `$HOME/.guix-profile` + +Therefore, in order for these operations to affect your shell/desktop sessions +(and for the principle of least astonishment to hold), their corresponding +directories have to be added to well-known environment variables like `$PATH`, +`$INFOPATH`, `$XDG_DATA_DIRS`, etc. + +In other words, if `$HOME/.config/guix/current/bin` does not exist in your +`$PATH`, a `guix pull` will have no effect on what `guix` you are using. Same +goes for `$HOME/.guix-profile/bin`, `guix install`, and installed packages. + +Helpfully, after a `guix pull` or `guix install`, a message will be printed like +so: + +``` +hint: Consider setting the necessary environment variables by running: + + GUIX_PROFILE="$HOME/.guix-profile" + . "$GUIX_PROFILE/etc/profile" + +Alternately, see `guix package --search-paths -p "$HOME/.guix-profile"'. +``` + +However, this is somewhat tedious to do for both `guix pull` and `guix install` +for each user on the system that wants to properly use `guix`. I recommend that +you instead add an entry to `/etc/profile.d` instead. This is done by default +when installing the Debian package later than 1.2.0-4 and when using the shell +script installer. + +#### Instructions + +Create `/etc/profile.d/guix.sh` with the following content: +```sh +# _GUIX_PROFILE: `guix pull` profile +_GUIX_PROFILE="$HOME/.config/guix/current" +if [ -L $_GUIX_PROFILE ]; then + export PATH="$_GUIX_PROFILE/bin${PATH:+:}$PATH" + # Export INFOPATH so that the updated info pages can be found + # and read by both /usr/bin/info and/or $GUIX_PROFILE/bin/info + # When INFOPATH is unset, add a trailing colon so that Emacs + # searches 'Info-default-directory-list'. + export INFOPATH="$_GUIX_PROFILE/share/info:$INFOPATH" +fi + +# GUIX_PROFILE: User's default profile +GUIX_PROFILE="$HOME/.guix-profile" +[ -L $GUIX_PROFILE ] || return +GUIX_LOCPATH="$GUIX_PROFILE/lib/locale" +export GUIX_PROFILE GUIX_LOCPATH + +[ -f "$GUIX_PROFILE/etc/profile" ] && . "$GUIX_PROFILE/etc/profile" + +# set XDG_DATA_DIRS to include Guix installations +export XDG_DATA_DIRS="$GUIX_PROFILE/share:${XDG_DATA_DIRS:-/usr/local/share/:/usr/share/}" +``` + +Please note that this will not take effect until the next shell or desktop +session (log out and log back in). + +### `guix pull` as root + +Before you do this, you need to read the section on [choosing your security +model][security-model] and adjust `guix` and `guix-daemon` flags according to +your choice, as invoking `guix pull` may pull substitutes from substitute +servers (which you may not want). + +As mentioned in a previous section, Guix expects +`${localstatedir}/guix/profiles/per-user/root/current-guix` to be populated with +`root`'s Guix profile, `guix pull`-ed and built by some former version of Guix. +However, this is not the case when we build from source. Therefore, we need to +perform a `guix pull` as `root`: + +```sh +sudo --login guix pull --branch=version- +# or +sudo --login guix pull --commit= +``` + +`guix pull` is quite a long process (especially if you're using +`--no-substitute`). If you encounter build problems, please refer to the +[troubleshooting section](#troubleshooting). + +Note that running a bare `guix pull` with no commit or branch specified will +pull the latest commit on Guix's master branch, which is likely fine, but not +recommended. + +If you installed Guix from source, you may get an error like the following: +```sh +error: while creating symlink '/root/.config/guix/current' No such file or directory +``` +To resolve this, simply: +``` +sudo mkdir -p /root/.config/guix +``` +Then try the `guix pull` command again. + +After the `guix pull` finishes successfully, +`${localstatedir}/guix/profiles/per-user/root/current-guix` should be populated. + +#### Using the newly-pulled `guix` by restarting the daemon + +Depending on how you installed Guix, you should now make sure that your init +scripts and service configurations point to the newly-pulled `guix-daemon`. + +##### If you built Guix from source + +If you followed the instructions for [fixing argv\[0\]][fix-argv0], you can now +do the following: + +```sh +systemctl stop guix-daemon-original +systemctl disable guix-daemon-original + +systemctl enable guix-daemon +systemctl start guix-daemon +``` + +##### If you installed Guix via the Debian/Ubuntu distribution packages + +You will need to create a `guix-daemon-latest` service which points to the new +`guix` rather than a pinned one. + +```sh +# Create guix-daemon-latest.service by modifying guix-daemon.service +sed -E -e "s|/usr/bin/guix-daemon|/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon|" /etc/systemd/system/guix-daemon.service > /lib/systemd/system/guix-daemon-latest.service +chmod 664 /lib/systemd/system/guix-daemon-latest.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the old guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the new guix-daemon-latest.service is started and enabled +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### If you installed Guix via lantw44's Arch Linux AUR package + +At the time of writing (July 5th, 2021) the systemd unit for "updated Guix" is +`guix-daemon-latest.service`, therefore, you should do the following: + +```sh +systemctl stop guix-daemon +systemctl disable guix-daemon + +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### Otherwise... + +Simply do: + +```sh +systemctl restart guix-daemon +``` + +### Checking everything + +If you followed all the steps above to make your Guix setup "prim and proper," +you can check that you did everything properly by running through this +checklist. + +1. `/etc/profile.d/guix.sh` should exist and be sourced at each shell login + +2. `guix describe` should not print `guix describe: error: failed to determine + origin`, but rather something like: + + ``` + Generation 38 Feb 22 2021 16:39:31 (current) + guix f350df4 + repository URL: https://git.savannah.gnu.org/git/guix.git + branch: version-1.2.0 + commit: f350df405fbcd5b9e27e6b6aa500da7f101f41e7 + ``` + +3. `guix-daemon` should be running from `${localstatedir}/guix/profiles/per-user/root/current-guix` + +# Troubleshooting + +## Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion (please don't forget +to add other `guix` flags like `--no-substitutes` as appropriate): + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://bugs.python.org/issue37584 + +### GnuTLS: test-suite FAIL: status-request-revoked + +*The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* + +This unfortunate error is most common for non-substitute builders who installed +Guix v1.2.0. The problem stems from the fact that one of GnuTLS's tests uses a +hardcoded certificate which expired on 2020-10-24. + +What's more unfortunate is that this GnuTLS derivation is somewhat special in +Guix's dependency graph and is not affected by the package transformation flags +like `--without-tests=`. + +The easiest solution for those encountering this problem is to install a newer +version of Guix. However, there are ways to work around this issue: + +#### Workaround 1: Using substitutes for this single derivation + +If you've authorized the official Guix build farm's key (more info +[here](./README.md#step-1-authorize-the-signing-keys)), then you can use +substitutes just for this single derivation by invoking the following: + +```sh +guix build --substitute-urls="https://ci.guix.gnu.org" /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +``` + +See [this section](./README.md#removing-authorized-keys) for instructions on how +to remove authorized keys if you don't want to keep the build farm's key +authorized. + +#### Workaround 2: Temporarily setting the system clock back + +This workaround was described [here](https://issues.guix.gnu.org/44559#5). + +Basically: +1. Turn off networking +2. Turn off NTP +3. Set system time to 2020-10-01 +4. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +5. Set system time back to accurate current time +6. Turn NTP back on +7. Turn networking back on + +### coreutils: FAIL: tests/tail-2/inotify-dir-recreate + +The inotify-dir-create test fails on "remote" filesystems such as overlayfs +(Docker's default filesystem) due to the filesystem being mistakenly recognized +as non-remote. + +A relatively easy workaround to this is to make sure that a somewhat traditional +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For +Docker users, this might mean [using a volume][docker/volumes], [binding +mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) +[mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. + +Please see the following links for more details: + +- An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) +- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935) +- A commit to skip this test in Guix has been merged into the core-updates branch: +[savannah/guix@6ba1058](https://git.savannah.gnu.org/cgit/guix.git/commit/?id=6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) + + +[install-script]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-bin-tarball]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-fanquake-docker]: #option-3-using-fanquakes-docker-image +[install-distro-pkg]: #option-4-using-a-distribution-maintained-package +[install-source]: #option-5-building-from-source + +[fix-argv0]: #creating-and-starting-a-guix-daemon-original-service-with-a-fixed-argv0 +[security-model]: ./README.md#choosing-your-security-model + +[docker/volumes]: https://docs.docker.com/storage/volumes/ +[docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ +[docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ diff --git a/contrib/guix/README.md b/contrib/guix/README.md new file mode 100644 index 0000000000..51a034c26e --- /dev/null +++ b/contrib/guix/README.md @@ -0,0 +1,484 @@ +# Bootstrappable Bitcoin Core Builds + +This directory contains the files necessary to perform bootstrappable Bitcoin +Core builds. + +[Bootstrappability][b17e] furthers our binary security guarantees by allowing us +to _audit and reproduce_ our toolchain instead of blindly _trusting_ binary +downloads. + +We achieve bootstrappability by using Guix as a functional package manager. + +# Requirements + +Conservatively, you will need an x86_64 machine with: + +- 16GB of free disk space on the partition that /gnu/store will reside in +- 8GB of free disk space **per platform triple** you're planning on building + (see the `HOSTS` [environment variable description][env-vars-list]) + +# Installation and Setup + +If you don't have Guix installed and set up, please follow the instructions in +[INSTALL.md](./INSTALL.md) + +# Usage + +If you haven't considered your security model yet, please read [the relevant +section](#choosing-your-security-model) before proceeding to perform a build. + +## Making the Xcode SDK available for macOS cross-compilation + +In order to perform a build for macOS (which is included in the default set of +platform triples to build), you'll need to extract the macOS SDK tarball using +tools found in the [`macdeploy` directory](../macdeploy/README.md). + +You can then either point to the SDK using the `SDK_PATH` environment variable: + +```sh +# Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode---extracted-SDK-with-libcxx-headers +tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar.gz + +# Indicate where to locate the SDK tarball +export SDK_PATH=/path/to/parent/dir/of/extracted/SDK +``` + +or extract it into `depends/SDKs`: + +```sh +mkdir -p depends/SDKs +tar -C depends/SDKs -xaf /path/to/SDK/tarball +``` + +## Building + +*The author highly recommends at least reading over the [common usage patterns +and examples](#common-guix-build-invocation-patterns-and-examples) section below +before starting a build. For a full list of customization options, see the +[recognized environment variables][env-vars-list] section.* + +To build Bitcoin Core reproducibly with all default options, invoke the +following from the top of a clean repository: + +```sh +./contrib/guix/guix-build +``` + +## Codesigning build outputs + +The `guix-codesign` command attaches codesignatures (produced by codesigners) to +existing non-codesigned outputs. Please see the [release process +documentation](/doc/release-process.md) for more context. + +It respects many of the same environment variable flags as `guix-build`, with 2 +crucial differences: + +1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` + environment variable will have a sane default value of `x86_64-w64-mingw32 + x86_64-apple-darwin19` instead of all the platforms. +2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. + * _**DETACHED_SIGS_REPO**_ + + Set the directory where detached codesignatures can be found for the current + Bitcoin Core version being built. + + _REQUIRED environment variable_ + +An invocation with all default options would look like: + +``` +env DETACHED_SIGS_REPO= ./contrib/guix/guix-codesign +``` + +## Cleaning intermediate work directories + +By default, `guix-build` leaves all intermediate files or "work directories" +(e.g. `depends/work`, `guix-build-*/distsrc-*`) intact at the end of a build so +that they are available to the user (to aid in debugging, etc.). However, these +directories usually take up a large amount of disk space. Therefore, a +`guix-clean` convenience script is provided which cleans the current `git` +worktree to save disk space: + +``` +./contrib/guix/guix-clean +``` + + +## Attesting to build outputs + +Much like how Gitian build outputs are attested to in a `gitian.sigs` +repository, Guix build outputs are attested to in the [`guix.sigs` +repository](https://github.com/bitcoin-core/guix.sigs). + +After you've cloned the `guix.sigs` repository, to attest to the current +worktree's commit/tag: + +``` +env GUIX_SIGS_REPO= SIGNER= ./contrib/guix/guix-attest +``` + +See `./contrib/guix/guix-attest --help` for more information on the various ways +`guix-attest` can be invoked. + +## Verifying build output attestations + +After at least one other signer has uploaded their signatures to the `guix.sigs` +repository: + +``` +git -C pull +env GUIX_SIGS_REPO= ./contrib/guix/guix-verify +``` + + +## Common `guix-build` invocation patterns and examples + +### Keeping caches and SDKs outside of the worktree + +If you perform a lot of builds and have a bunch of worktrees, you may find it +more efficient to keep the depends tree's download cache, build cache, and SDKs +outside of the worktrees to avoid duplicate downloads and unnecessary builds. To +help with this situation, the `guix-build` script honours the `SOURCES_PATH`, +`BASE_CACHE`, and `SDK_PATH` environment variables and will pass them on to the +depends tree so that you can do something like: + +```sh +env SOURCES_PATH="$HOME/depends-SOURCES_PATH" BASE_CACHE="$HOME/depends-BASE_CACHE" SDK_PATH="$HOME/macOS-SDKs" ./contrib/guix/guix-build +``` + +Note that the paths that these environment variables point to **must be +directories**, and **NOT symlinks to directories**. + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Building a subset of platform triples + +Sometimes you only want to build a subset of the supported platform triples, in +which case you can override the default list by setting the space-separated +`HOSTS` environment variable: + +```sh +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin19' ./contrib/guix/guix-build +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Controlling the number of threads used by `guix` build commands + +Depending on your system's RAM capacity, you may want to decrease the number of +threads used to decrease RAM usage or vice versa. + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that `guix` build +commands also accept a `--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + + - controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + + - controls how many derivations can be built in parallel + - defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +## Recognized environment variables + +* _**HOSTS**_ + + Override the space-separated list of platform triples for which to perform a + bootstrappable build. + + _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu + riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86\_64-w64-mingw32 x86\_64-apple-darwin19")_ + +* _**SOURCES_PATH**_ + + Set the depends tree download cache for sources. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary redownloading of package sources. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**BASE_CACHE**_ + + Set the depends tree cache for built packages. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary building of packages. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**SDK_PATH**_ + + Set the path where _extracted_ SDKs can be found. This is passed through to + the depends tree. Note that this is should be set to the _parent_ directory of + the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of + `$HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers`). + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**JOBS**_ + + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix environment --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` + + See [here](#controlling-the-number-of-threads-used-by-guix-build-commands) for + more details. + + _(defaults to the value of `nproc` outside the container)_ + +* _**SOURCE_DATE_EPOCH**_ + + Override the reference UNIX timestamp used for bit-for-bit reproducibility, + the variable name conforms to [standard][r12e/source-date-epoch]. + + _(defaults to the output of `$(git log --format=%at -1)`)_ + +* _**V**_ + + If non-empty, will pass `V=1` to all `make` invocations, making `make` output + verbose. + + Note that any given value is ignored. The variable is only checked for + emptiness. More concretely, this means that `V=` (setting `V` to the empty + string) is interpreted the same way as not setting `V` at all, and that `V=0` + has the same effect as `V=1`. + +* _**SUBSTITUTE_URLS**_ + + A whitespace-delimited list of URLs from which to download pre-built packages. + A URL is only used if its signing key is authorized (refer to the [substitute + servers section](#option-1-building-with-substitutes) for more details). + +* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ + + Additional flags to be passed to all `guix` commands. + +* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ + + Additional flags to be passed to `guix time-machine`. + +* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ + + Additional flags to be passed to the invocation of `guix environment` inside + `guix time-machine`. + +# Choosing your security model + +No matter how you installed Guix, you need to decide on your security model for +building packages with Guix. + +Guix allows us to achieve better binary security by using our CPU time to build +everything from scratch. However, it doesn't sacrifice user choice in pursuit of +this: users can decide whether or not to use **substitutes** (pre-built +packages). + +## Option 1: Building with substitutes + +### Step 1: Authorize the signing keys + +Depending on the installation procedure you followed, you may have already +authorized the Guix build farm key. In particular, the official shell installer +script asks you if you want the key installed, and the debian distribution +package authorized the key during installation. + +You can check the current list of authorized keys at `/etc/guix/acl`. + +At the time of writing, a `/etc/guix/acl` with just the Guix build farm key +authorized looks something like: + +```lisp +(acl + (entry + (public-key + (ecc + (curve Ed25519) + (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) + ) + ) + (tag + (guix import) + ) + ) + ) +``` + +If you've determined that the official Guix build farm key hasn't been +authorized, and you would like to authorize it, run the following as root: + +``` +guix archive --authorize < /var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub +``` + +If +`/var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub` +doesn't exist, try: + +```sh +guix archive --authorize < /share/guix/ci.guix.gnu.org.pub +``` + +Where `` is likely: +- `/usr` if you installed from a distribution package +- `/usr/local` if you installed Guix from source and didn't supply any + prefix-modifying flags to Guix's `./configure` + +For dongcarl's substitute server at https://guix.carldong.io, run as root: + +```sh +wget -qO- 'https://guix.carldong.io/signing-key.pub' | guix archive --authorize +``` + +#### Removing authorized keys + +To remove previously authorized keys, simply edit `/etc/guix/acl` and remove the +`(entry (public-key ...))` entry. + +### Step 2: Specify the substitute servers + +Once its key is authorized, the official Guix build farm at +https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag +is supplied. This default list of substitute servers is overridable both on a +`guix-daemon` level and when you invoke `guix` commands. See examples below for +the various ways of adding dongcarl's substitute server after having [authorized +his signing key](#authorize-the-signing-keys). + +Change the **default list** of substitute servers by starting `guix-daemon` with +the `--substitute-urls` option (you will likely need to edit your init script): + +```sh +guix-daemon --substitute-urls='https://guix.carldong.io https://ci.guix.gnu.org' +``` + +Override the default list of substitute servers by passing the +`--substitute-urls` option for invocations of `guix` commands: + +```sh +guix --substitute-urls='https://guix.carldong.io https://ci.guix.gnu.org' +``` + +For scripts under `./contrib/guix`, set the `SUBSTITUTE_URLS` environment +variable: + +```sh +export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' +``` + +## Option 2: Disabling substitutes on an ad-hoc basis + +If you prefer not to use any substitutes, make sure to supply `--no-substitutes` +like in the following snippet. The first build will take a while, but the +resulting packages will be cached for future builds. + +For direct invocations of `guix`: +```sh +guix --no-substitutes +``` + +For the scripts under `./contrib/guix/`: +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' +``` + +## Option 3: Disabling substitutes by default + +`guix-daemon` accepts a `--no-substitutes` flag, which will make sure that, +unless otherwise overridden by a command line invocation, no substitutes will be +used. + +If you start `guix-daemon` using an init script, you can edit said script to +supply this flag. + + +# Purging/Uninstalling Guix + +In the extraordinarily rare case where you messed up your Guix installation in +an irreversible way, you may want to completely purge Guix from your system and +start over. + +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). +2. Remove all build users and groups + + You may check for relevant users and groups using: + + ``` + getent passwd | grep guix + getent group | grep guix + ``` + + Then, you may remove users and groups using: + + ``` + sudo userdel + sudo groupdel + ``` + +3. Remove all possible Guix-related directories + - `/var/guix/` + - `/var/log/guix/` + - `/gnu/` + - `/etc/guix/` + - `/home/*/.config/guix/` + - `/home/*/.cache/guix/` + - `/home/*/.guix-profile/` + - `/root/.config/guix/` + - `/root/.cache/guix/` + - `/root/.guix-profile/` + +[b17e]: https://bootstrappable.org/ +[r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ + +[guix/install.sh]: https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh +[guix/bin-install]: https://www.gnu.org/software/guix/manual/en/html_node/Binary-Installation.html +[guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html +[guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html +[guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html +[guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html + +[debian/guix-bullseye]: https://packages.debian.org/bullseye/guix +[ubuntu/guix-hirsute]: https://packages.ubuntu.com/hirsute/guix +[fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix + +[env-vars-list]: #recognized-environment-variables diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest new file mode 100755 index 0000000000..6e12cbead7 --- /dev/null +++ b/contrib/guix/guix-attest @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat env basename mkdir diff sort +if [ -z "$NO_SIGN" ]; then + check_tools gpg +fi + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < \\ + SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ + [ NO_SIGN=1 ] + ./contrib/guix/guix-attest + +Example w/o overriding signing name: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + ./contrib/guix/guix-attest + +Example overriding signing name: + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs \\ + SIGNER=0x96AB007F1A7ED999=dongcarl \\ + ./contrib/guix/guix-attest + +Example w/o signing, just creating SHA256SUMS: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + NO_SIGN=1 \\ + ./contrib/guix/guix-attest + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +################ +# The key specified in SIGNER should be usable +################ + +IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER" +if [ -z "${signer_name}" ]; then + signer_name="$gpg_key_name" +fi + +if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then + echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" + exit 1 +fi + +################ +# We should be able to find at least one output +################ + +echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}" + +shopt -s nullglob +sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... +shopt -u nullglob + +noncodesigned_fragments=() +codesigned_fragments=() + +if (( ${#sha256sum_fragments[@]} )); then + echo "Found build output SHA256SUMS fragments:" + for outdir in "${sha256sum_fragments[@]}"; do + echo " '$outdir'" + case "$outdir" in + "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part) + codesigned_fragments+=("$outdir") + ;; + *) + noncodesigned_fragments+=("$outdir") + ;; + esac + done + echo +else + echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}" + exit 1 +fi + +############## +## Attest ## +############## + +# Usage: out_name $outdir +# +# HOST: The output directory being attested +# +out_name() { + basename "$(dirname "$1")" +} + +shasum_already_exists() { +cat < "$temp_noncodesigned" + if [ -e noncodesigned.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then + echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists noncodesigned.SHA256SUMS + exit 1 + fi + else + mv "$temp_noncodesigned" noncodesigned.SHA256SUMS + fi + else + echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..." + exit 1 + fi + + temp_all="$(mktemp)" + trap 'rm -rf -- "$temp_all"' EXIT + + if (( ${#codesigned_fragments[@]} )); then + # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is + # not needed if there are no $codesigned_fragments + cat "${sha256sum_fragments[@]}" \ + | sort -u \ + | sort -k2 \ + | basenameify_SHA256SUMS \ + > "$temp_all" + if [ -e all.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u all.SHA256SUMS "$temp_all"; then + echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists all.SHA256SUMS + exit 1 + fi + else + mv "$temp_all" all.SHA256SUMS + fi + else + # It is fine to have the codesigned outputs be missing (perhaps the + # detached codesigs have not been published yet), just print a log + # message instead of erroring out + echo "INFO: No codesigned outputs found for '${VERSION}', skipping..." + fi + + if [ -z "$NO_SIGN" ]; then + echo "Signing SHA256SUMS to produce SHA256SUMS.asc" + for i in *.SHA256SUMS; do + if [ ! -e "$i".asc ]; then + gpg --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" + else + echo "Signature already there" + fi + done + else + echo "Not signing SHA256SUMS as \$NO_SIGN is not empty" + fi + echo "" +) diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build new file mode 100755 index 0000000000..9b60048b35 --- /dev/null +++ b/contrib/guix/guix-build @@ -0,0 +1,477 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir make getent curl git guix + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# The git worktree should not be dirty +################ + +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +fi + +mkdir -p "$VERSION_BASE" + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-linux-gnu + x86_64-w64-mingw32 + x86_64-apple-darwin19}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + +################ +# When building for darwin, the macOS SDK should exists +################ + +for host in $HOSTS; do + case "$host" in + *darwin*) + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" + if [ -e "$OSX_SDK" ]; then + echo "Found macOS SDK at '${OSX_SDK}', using..." + else + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + exit 1 + fi + ;; + esac +done + +################ +# VERSION_BASE should have enough space +################ + +avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)" +total_required_KiB=0 +for host in $HOSTS; do + case "$host" in + *darwin*) required_KiB=1200000 ;; + *mingw*) required_KiB=2900000 ;; + *) required_KiB=2100000 ;; + esac + total_required_KiB=$((total_required_KiB+required_KiB)) +done + +if (( total_required_KiB > avail_KiB )); then + total_required_GiB=$((total_required_KiB / 1048576)) + avail_GiB=$((avail_KiB / 1048576)) + echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build." + exit 1 +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + +################ +# Services database must have basic entries +################ + +if ! getent services http https ftp > /dev/null 2>&1; then +cat << EOF +ERR: Your system's C library can not find service database entries for at least + one of the following services: http, https, ftp. + +Hint: Most likely, /etc/services does not exist yet (common for docker images + and minimal distros), or you don't have permissions to access it. + + If /etc/services does not exist yet, you may want to install the + appropriate package for your distro which provides it. + + On Debian/Ubuntu: netbase + On Arch Linux: iana-etc + + For more information, see: getent(1), services(5) + +EOF + +fi + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Usage: host_to_commonname HOST +# +# HOST: The current platform triple we're building for +# +host_to_commonname() { + case "$1" in + *darwin*) echo osx ;; + *mingw*) echo win ;; + *linux*) echo linux ;; + *) exit 1 ;; + esac +} + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" + +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ + --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + + +# Precious directories are those which should not be cleaned between successive +# guix builds +depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' +precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" + +# Usage: contains IFS-SEPARATED-LIST ITEM +contains() { + for i in ${1}; do + if [ "$i" = "${2}" ]; then + return 0 # Found! + fi + done + return 1 +} + +# If the user explicitly specified a precious directory, create it so we +# can map it into the container +for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if [ -n "$precious_dir_path" ]; then + if [ ! -e "$precious_dir_path" ]; then + mkdir -p "$precious_dir_path" + elif [ -L "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} cannot be a symbolic link" + exit 1 + elif [ ! -d "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} must be a directory" + exit 1 + fi + fi +done + +mkdir -p "$VAR_BASE" + +# Record the _effective_ values of precious directories such that guix-clean can +# avoid clobbering them if appropriate. +# +# shellcheck disable=SC2046,SC2086 +{ + # Get depends precious dir definitions from depends + make -C "${PWD}/depends" \ + --no-print-directory \ + -- $(printf "print-%s\n" $depends_precious_dir_names) + + # Get remaining precious dir definitions from the environment + for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then + echo "${precious_dir_name}=${precious_dir_path}" + fi + done +} > "${VAR_BASE}/precious_dirs" + +# Make sure an output directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Download the depends sources now as we won't have internet access in the build +# container +for host in $HOSTS; do + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + + +######### +# BUILD # +######### + +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before rebuilding + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...running at most ${JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/firo' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' +EOF + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/firo + # + # maps our current working directory to /firo + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/firo' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # --keep-failed keep build tree of failed builds + # + # When builds of the Guix environment itself (not Bitcoin Core) + # fail, it is useful for the build tree to be kept for debugging + # purposes. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/firo \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --expose="$(git rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${BASE_CACHE:+--share="$BASE_CACHE"} \ + ${SDK_PATH:+--share="$SDK_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --root="$(profiledir_for_host "${HOST}")" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + bash -c "cd /firo && bash contrib/guix/libexec/build.sh" + ) + +done diff --git a/contrib/guix/guix-clean b/contrib/guix/guix-clean new file mode 100755 index 0000000000..9af0a793cf --- /dev/null +++ b/contrib/guix/guix-clean @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat mkdir make git guix + + +############# +## Clean ## +############# + +# Usage: under_dir MAYBE_PARENT MAYBE_CHILD +# +# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path +# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code. +# +# NOTE: This does not perform any symlink-resolving or path canonicalization. +# +under_dir() { + local path_residue + path_residue="${2##"${1}"}" + if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then + return 1 + else + echo "$path_residue" + fi +} + +# Usage: dir_under_git_root MAYBE_CHILD +# +# If MAYBE_CHILD is under the current git repository and exists, print the +# relative path from the git repository's top-level directory to MAYBE_CHILD, +# otherwise, exit with an error code. +# +dir_under_git_root() { + local rv + rv="$(under_dir "$(git_root)" "$1")" + [ -n "$rv" ] && echo "$rv" +} + +shopt -s nullglob +found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories... +shopt -u nullglob + +exclude_flags=() + +for precious_dirs_file in "${found_precious_dirs_files[@]}"; do + # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH) + # are excluded from git-clean + echo "Found precious_dirs file: '${precious_dirs_file}'" + + # Exclude the precious_dirs file itself + if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then + exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" ) + fi + + # Read each 'name=dir' pair from the precious_dirs file + while IFS='=' read -r name dir; do + # Add an exclusion flag if the precious directory is under the git root. + if under=$(dir_under_git_root "$dir"); then + echo "Avoiding ${name}: ${under}" + exclude_flags+=( --exclude="$under" ) + fi + done < "$precious_dirs_file" +done + +git clean -xdff "${exclude_flags[@]}" diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign new file mode 100755 index 0000000000..aff897037d --- /dev/null +++ b/contrib/guix/guix-codesign @@ -0,0 +1,392 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir git guix + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { + cat < \\ + ./contrib/guix/guix-codesign + +EOF +} + +if [ -z "$DETACHED_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# The codesignature git worktree should not be dirty +################ + +if ! git -C "$DETACHED_SIGS_REPO" diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then + cat << EOF +ERR: The DETACHED CODESIGNATURE git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF + exit 1 +fi + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin19}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}-codesigned" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + + +################ +# Unsigned tarballs SHOULD exist +################ + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + + +unsigned_tarball_for_host() { + case "$1" in + *mingw*) + echo "$(outdir_for_host "$1")/${DISTNAME}-win-unsigned.tar.gz" + ;; + *darwin*) + echo "$(outdir_for_host "$1")/${DISTNAME}-osx-unsigned.tar.gz" + ;; + *) + exit 1 + ;; + esac +} + +# Accumulate a list of build directories that already exist... +hosts_unsigned_tarball_missing="" +for host in $HOSTS; do + if [ ! -e "$(unsigned_tarball_for_host "$host")" ]; then + hosts_unsigned_tarball_missing+=" ${host}" + fi +done + +if [ -n "$hosts_unsigned_tarball_missing" ]; then + # ...so that we can print them out nicely in an error message + cat << EOF +ERR: Unsigned tarballs do not exist +... + +EOF +for host in $hosts_unsigned_tarball_missing; do + echo " ${host} '$(unsigned_tarball_for_host "$host")'" +done +exit 1 +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then + cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF + exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" + +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ + --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + +# Make sure an output directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + +######### +# BUILD # +######### + +# Function to be called when codesigning for host ${1} and the user interrupts +# the codesign +int_trap() { +cat << EOF +** INT received while codesigning ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before recodesigning + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Codesigning ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST" codesigned)' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)' + ...using detached signatures in: '${DETACHED_SIGS_REPO:?not set}' + ...bind-mounted in container to: '/detached-sigs' +EOF + + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --share="$DETACHED_SIGS_REPO"=/detached-sigs \ + --expose="$(git rev-parse --git-common-dir)" \ + --expose="$(git -C "$DETACHED_SIGS_REPO" rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --root="$(profiledir_for_host "${HOST}" codesigned)" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + DETACHED_SIGS_REPO=/detached-sigs \ + UNSIGNED_TARBALL="$(OUTDIR_BASE=/outdir-base && unsigned_tarball_for_host "$HOST")" \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/codesign.sh" + ) + +done diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify new file mode 100755 index 0000000000..02ae022741 --- /dev/null +++ b/contrib/guix/guix-verify @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat diff gpg + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < [ SIGNER= ] ./contrib/guix/guix-verify + +Example overriding signer's manifest to use as base + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +############## +## Verify ## +############## + +OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" +echo "Looking for signature directories in '${OUTSIGDIR_BASE}'" +echo "" + +# Usage: verify compare_manifest current_manifest +verify() { + local compare_manifest="$1" + local current_manifest="$2" + if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then + echo "ERR: Failed to verify GPG signature in '${current_manifest}'" + echo "" + echo "Hint: Either the signature is invalid or the public key is missing" + echo "" + failure=1 + elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then + echo "ERR: The SHA256SUMS attestation in these two directories differ:" + echo " '${compare_manifest}'" + echo " '${current_manifest}'" + echo "" + failure=1 + else + echo "Verified: '${current_manifest}'" + echo "" + fi +} + +shopt -s nullglob +all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_noncodesigned[@]} )); then + compare_noncodesigned="${all_noncodesigned[0]}" + if [[ -n "$SIGNER" ]]; then + signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" + if [[ -f "$signer_noncodesigned" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_noncodesigned="$signer_noncodesigned" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_noncodesigned[@]}"; do + verify "$compare_noncodesigned" "$current_manifest" + done + + echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with noncodesigned.SHA256SUMS found" + echo "" +fi + +shopt -s nullglob +all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_all[@]} )); then + compare_all="${all_all[0]}" + if [[ -n "$SIGNER" ]]; then + signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" + if [[ -f "$signer_all" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_all="$signer_all" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_all[@]}"; do + verify "$compare_all" "$current_manifest" + done + + # Sanity check: there should be no entries that exist in + # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS + if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then + echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which" + echo " do not exist in all.SHA256SUMS, something went very wrong." + exit 1 + fi + + echo "DONE: Checking output signatures for all.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with all.SHA256SUMS found" + echo "" +fi + +echo "====================" +echo "" +if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then + echo "ERR: Unable to perform any verifications as no signature directories" + echo " were found" + echo "" + exit 1 +fi + +if [ -n "$failure" ]; then + exit 1 +fi diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh new file mode 100755 index 0000000000..798229fa8e --- /dev/null +++ b/contrib/guix/libexec/build.sh @@ -0,0 +1,448 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# environment`. It does make sense for at least `guix environment --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + JOBS: ${JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + +##################### +# Environment Setup # +##################### + +# The depends folder also serves as a base-prefix for depends packages for +# $HOSTs after successfully building. +BASEPREFIX="${PWD}/depends" + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + + +# Set environment variables to point the NATIVE toolchain to the right +# includes/libs +NATIVE_GCC="$(store_path gcc-toolchain)" +NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" + +unset LIBRARY_PATH +unset CPATH +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH + +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64" +export C_INCLUDE_PATH="${NATIVE_GCC}/include" +export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" +export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" +export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" + +prepend_to_search_env_var() { + export "${1}=${2}${!1:+:}${!1}" +} + +case "$HOST" in + *darwin*) + # When targeting darwin, zlib is required by native_libdmg-hfsplus. + zlib_store_path=$(store_path "zlib") + zlib_static_store_path=$(store_path "zlib" static) + + prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib" + prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include" +esac + +# Set environment variables to point the CROSS toolchain to the right +# includes/libs for $HOST +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # The search path ordering is generally: + # 1. gcc-related search paths + # 2. libc-related search paths + # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *darwin*) + # The CROSS toolchain for darwin uses the SDK and ignores environment variables. + # See depends/hosts/darwin.mk for more details. + ;; + *linux*) + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *) + exit 1 ;; +esac + +# Sanity check CROSS_*_PATH directories +IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" +for p in "${PATHS[@]}"; do + if [ -n "$p" ] && [ ! -d "$p" ]; then + echo "'$p' doesn't exist or isn't a directory... Aborting..." + exit 1 + fi +done + +# Disable Guix ld auto-rpath behavior +case "$HOST" in + *darwin*) + # The auto-rpath behavior is necessary for darwin builds as some native + # tools built by depends refer to and depend on Guix-built native + # libraries + # + # After the native packages in depends are built, the ld wrapper should + # no longer affect our build, as clang would instead reach for + # x86_64-apple-darwin19-ld from cctools + ;; + *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; +esac + +# Make /usr/bin if it doesn't exist +[ -e /usr/bin ] || mkdir -p /usr/bin + +# Symlink file and env to a conventional path +[ -e /usr/bin/file ] || ln -s --no-dereference "$(command -v file)" /usr/bin/file +[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env + +# Determine the correct value for -Wl,--dynamic-linker for the current $HOST +case "$HOST" in + *linux*) + glibc_dynamic_linker=$( + case "$HOST" in + i686-linux-gnu) echo /lib/ld-linux.so.2 ;; + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + powerpc64-linux-gnu) echo /lib64/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; + *) exit 1 ;; + esac + ) + ;; +esac + +# Environment variables for determinism +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ="UTC" +case "$HOST" in + *darwin*) + # cctools AR, unlike GNU binutils AR, does not have a deterministic mode + # or a configure flag to enable determinism by default, it only + # understands if this env-var is set or not. See: + # + # https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334 + export ZERO_AR_DATE=yes + ;; +esac + +#################### +# Depends Building # +#################### + +# Build the depends tree, overriding variables that assume multilib gcc +make -C depends --jobs="$JOBS" HOST="$HOST" \ + ${V:+V=1} \ + ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ + i686_linux_CC=i686-linux-gnu-gcc \ + i686_linux_CXX=i686-linux-gnu-g++ \ + i686_linux_AR=i686-linux-gnu-ar \ + i686_linux_RANLIB=i686-linux-gnu-ranlib \ + i686_linux_NM=i686-linux-gnu-nm \ + i686_linux_STRIP=i686-linux-gnu-strip \ + x86_64_linux_CC=x86_64-linux-gnu-gcc \ + x86_64_linux_CXX=x86_64-linux-gnu-g++ \ + x86_64_linux_AR=x86_64-linux-gnu-ar \ + x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ + x86_64_linux_NM=x86_64-linux-gnu-nm \ + x86_64_linux_STRIP=x86_64-linux-gnu-strip \ + qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ + qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ + FORCE_USE_SYSTEM_CLANG=1 + + +########################### +# Source Tarball Building # +########################### + +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" + +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +########################### +# Binary Tarball Building # +########################### + +# CONFIGFLAGS +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" + +# CFLAGS +HOST_CFLAGS="-O2 -g" +case "$HOST" in + *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; + *mingw*) HOST_CFLAGS+=" -fno-ident" ;; + *darwin*) unset HOST_CFLAGS ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +case "$HOST" in + arm-linux-gnueabihf) HOST_CXXFLAGS="${HOST_CXXFLAGS} -Wno-psabi" ;; +esac + +# LDFLAGS +case "$HOST" in + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; +esac + +# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by +# avoiding a PowerPC64 optimisation available in glibc 2.22 and later. +# https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html +case "$HOST" in + *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;; +esac + +case "$HOST" in + powerpc64-linux-*|riscv64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;; +esac + +# Make $HOST-specific native binaries from depends available in $PATH +export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + # Extract the source tarball + tar --strip-components=1 -xf "${GIT_ARCHIVE}" + + ./autogen.sh + + # Configure this DISTSRC for $HOST + # shellcheck disable=SC2086 + env CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" \ + ./configure --prefix=/ \ + --disable-ccache \ + --disable-maintainer-mode \ + --disable-dependency-tracking \ + --enable-elysium \ + --enable-crash-hooks \ + --without-libs \ + ${CONFIGFLAGS} \ + ${HOST_CFLAGS:+CFLAGS="${HOST_CFLAGS}"} \ + ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \ + ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} + + sed -i.old 's/-lstdc++ //g' config.status libtool src/secp256k1/config.status src/secp256k1/libtool src/univalue/config.status src/univalue/libtool + + # Build Bitcoin Core + make --jobs="$JOBS" ${V:+V=1} + + mkdir -p "$OUTDIR" + + # Make the os-specific installers + case "$HOST" in + *mingw*) + make deploy ${V:+V=1} BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + ;; + esac + + # Setup the directory where our Bitcoin Core build for HOST will be + # installed. This directory will also later serve as the input for our + # binary tarballs. + INSTALLPATH="${PWD}/installed/${DISTNAME}" + mkdir -p "${INSTALLPATH}" + # Install built Bitcoin Core to $INSTALLPATH + case "$HOST" in + *darwin*) + make install-strip DESTDIR="${INSTALLPATH}" ${V:+V=1} + ;; + *) + make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + ;; + esac + + case "$HOST" in + *darwin*) + make osx_volname ${V:+V=1} + make deploydir ${V:+V=1} + mkdir -p "unsigned-app-${HOST}" + cp --target-directory="unsigned-app-${HOST}" \ + osx_volname \ + contrib/macdeploy/detached-sig-{apply,create}.sh \ + "${BASEPREFIX}/${HOST}"/native/bin/dmg + mv --target-directory="unsigned-app-${HOST}" dist + ( + cd "unsigned-app-${HOST}" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" && exit 1 ) + ) + make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" + ;; + esac + ( + cd installed + + # Prune libtool and object archives + find . -name "lib*.la" -delete + find . -name "lib*.a" -delete + + # Prune pkg-config files + rm -rf "${DISTNAME}/lib/pkgconfig" + + case "$HOST" in + *darwin*) ;; + *) + # Split binaries and libraries from their debug symbols + { + find "${DISTNAME}/bin" -type f -executable -print0 + } | xargs -0 -n1 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + ;; + esac + + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/README.md" "${DISTNAME}/" + ;; + esac + + # Finally, deterministically produce {non-,}debug binary tarballs ready + # for release + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + *darwin*) + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin19/osx64}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin19/osx64}.tar.gz" && exit 1 ) + ;; + esac + ) # $DISTSRC/installed + + case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir -p unsigned + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + ) + ;; + esac +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh new file mode 100755 index 0000000000..f484ac5774 --- /dev/null +++ b/contrib/guix/libexec/codesign.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# environment`. It does make sense for at least `guix environment --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + UNSIGNED_TARBALL: ${UNSIGNED_TARBALL:?not set} + DETACHED_SIGS_REPO: ${DETACHED_SIGS_REPO:?not set} + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + +git_head_version() { + local recent_tag + if recent_tag="$(git -C "$1" describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git -C "$1" rev-parse --short=12 HEAD + fi +} + +CODESIGNATURE_GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}-codesignatures-$(git_head_version "$DETACHED_SIGS_REPO").tar.gz" + +# Create the codesignature tarball if not already there +if [ ! -e "$CODESIGNATURE_GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$CODESIGNATURE_GIT_ARCHIVE")" + git -C "$DETACHED_SIGS_REPO" archive --output="$CODESIGNATURE_GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + tar -xf "$UNSIGNED_TARBALL" + + mkdir -p codesignatures + tar -C codesignatures -xf "$CODESIGNATURE_GIT_ARCHIVE" + + case "$HOST" in + *mingw*) + find "$PWD" -name "*-unsigned.exe" | while read -r infile; do + infile_base="$(basename "$infile")" + + # Codesigned *-unsigned.exe and output to OUTDIR + osslsigncode attach-signature \ + -in "$infile" \ + -out "${OUTDIR}/${infile_base/-unsigned}" \ + -sigin codesignatures/win/"$infile_base".pem + done + ;; + *darwin*) + # Apply detached codesignatures to dist/ (in-place) + signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist + + # Make an uncompressed DMG from dist/ + xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \ + -o uncompressed.dmg \ + dist \ + -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" + + # Compress uncompressed.dmg and output to OUTDIR + ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-osx-signed.dmg" + ;; + *) + exit 1 + ;; + esac +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$UNSIGNED_TARBALL" + echo "$CODESIGNATURE_GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash new file mode 100644 index 0000000000..a68bb98103 --- /dev/null +++ b/contrib/guix/libexec/prelude.bash @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# shellcheck source=contrib/shell/realpath.bash +source contrib/shell/realpath.bash + +# shellcheck source=contrib/shell/git-utils.bash +source contrib/shell/git-utils.bash + +################ +# Required non-builtin commands should be invocable +################ + +check_tools() { + for cmd in "$@"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi + done +} + +check_tools cat env readlink dirname basename git + +################ +# We should be at the top directory of the repository +################ + +same_dir() { + local resolved1 resolved2 + resolved1="$(bash_realpath "${1}")" + resolved2="$(bash_realpath "${2}")" + [ "$resolved1" = "$resolved2" ] +} + +if ! same_dir "${PWD}" "$(git_root)"; then +cat << EOF +ERR: This script must be invoked from the top level of the git repository + +Hint: This may look something like: + env FOO=BAR ./contrib/guix/guix- + +EOF +exit 1 +fi + +################ +# Set common variables +################ + +VERSION="${FORCE_VERSION:-$(git_head_version)}" +DISTNAME="${DISTNAME:-firo-${VERSION}}" + +version_base_prefix="${PWD}/guix-build-" +VERSION_BASE="${version_base_prefix}${VERSION}" # TOP + +DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" + +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" + +var_base_basename="var" +VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}" + +profiles_base_basename="profiles" +PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm new file mode 100644 index 0000000000..2444b87af7 --- /dev/null +++ b/contrib/guix/manifest.scm @@ -0,0 +1,659 @@ +(use-modules (gnu) + (gnu packages) + (gnu packages autotools) + (gnu packages base) + (gnu packages bash) + (gnu packages bison) + (gnu packages certs) + (gnu packages cdrom) + (gnu packages check) + (gnu packages cmake) + (gnu packages commencement) + (gnu packages compression) + (gnu packages cross-base) + (gnu packages curl) + (gnu packages file) + (gnu packages gawk) + (gnu packages gcc) + (gnu packages gnome) + (gnu packages image) + (gnu packages imagemagick) + (gnu packages installers) + (gnu packages linux) + (gnu packages llvm) + (gnu packages mingw) + (gnu packages moreutils) + (gnu packages perl) + (gnu packages pkg-config) + (gnu packages python) + (gnu packages python-web) + (gnu packages shells) + (gnu packages tls) + (gnu packages version-control) + (guix build-system font) + (guix build-system gnu) + (guix build-system python) + (guix build-system trivial) + (guix download) + (guix gexp) + (guix git-download) + ((guix licenses) #:prefix license:) + (guix packages) + (guix profiles) + (guix utils)) + +(define-syntax-rule (search-our-patches file-name ...) + "Return the list of absolute file names corresponding to each +FILE-NAME found in ./patches relative to the current file." + (parameterize + ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) + (list (search-patch file-name) ...))) + +(define (make-ssp-fixed-gcc xgcc) + "Given a XGCC package, return a modified package that uses the SSP function +from glibc instead of from libssp.so. Our `symbol-check' script will complain if +we link against libssp.so, and thus will ensure that this works properly. + +Taken from: +http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:make-flags flags) + `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + +(define (make-gcc-rpath-link xgcc) + "Given a XGCC package, return a modified package that replace each instance of +-rpath in the default system spec that's inserted by Guix with -rpath-link" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:phases phases) + `(modify-phases ,phases + (add-after 'pre-configure 'replace-rpath-with-rpath-link + (lambda _ + (substitute* (cons "gcc/config/rs6000/sysv4.h" + (find-files "gcc/config" + "^gnu-user.*\\.h$")) + (("-rpath=") "-rpath-link=")) + #t)))))))) + +(define (make-binutils-with-mingw-w64-disable-flags xbinutils) + (package-with-extra-patches xbinutils + (search-our-patches "binutils-mingw-w64-disable-flags.patch"))) + +(define (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + ;; 1. Build a cross-compiling gcc without targeting any libc, derived + ;; from BASE-GCC-FOR-LIBC + (xgcc-sans-libc (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils)) + ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived + ;; from BASE-KERNEL-HEADERS + (xkernel (cross-kernel-headers target + base-kernel-headers + xgcc-sans-libc + xbinutils)) + ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, + ;; derived from BASE-LIBC + (xlibc (cross-libc target + base-libc + xgcc-sans-libc + xbinutils + xkernel)) + ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from + ;; BASE-GCC + (xgcc (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-toolchain")) + (version (package-version xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,xlibc) + ("libc:static" ,xlibc "static") + ("gcc" ,xgcc) + ("gcc-lib" ,xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page xgcc)) + (license (package-license xgcc))))) + +(define base-gcc + (package-with-extra-patches gcc-8 + (search-our-patches "gcc-8-sort-libtool-find-output.patch"))) + +;; Building glibc with stack smashing protector first landed in glibc 2.25, use +;; this function to disable for older glibcs +;; +;; From glibc 2.25 changelog: +;; +;; * Most of glibc can now be built with the stack smashing protector enabled. +;; It is recommended to build glibc with --enable-stack-protector=strong. +;; Implemented by Nick Alcock (Oracle). +(define (make-glibc-without-ssp xglibc) + (package-with-extra-configure-variable + (package-with-extra-configure-variable + xglibc "libc_cv_ssp" "no") + "libc_cv_ssp_strong" "no")) + +(define* (make-bitcoin-cross-toolchain target + #:key + (base-gcc-for-libc gcc-7) + (base-kernel-headers linux-libre-headers-4.9) + (base-libc (make-glibc-without-ssp glibc-2.24)) + (base-gcc (make-gcc-rpath-link base-gcc))) + "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values +desirable for building Bitcoin Core release binaries." + (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc)) + +(define (make-gcc-with-pthreads gcc) + (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (make-binutils-with-mingw-w64-disable-flags (cross-binutils target))) + (pthreads-xlibc mingw-w64-x86_64-winpthreads) + (pthreads-xgcc (make-gcc-with-pthreads + (cross-gcc target + #:xgcc (make-ssp-fixed-gcc base-gcc) + #:xbinutils xbinutils + #:libc pthreads-xlibc)))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,pthreads-xlibc) + ("gcc" ,pthreads-xgcc) + ("gcc-lib" ,pthreads-xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + +(define (make-nsis-with-sde-support base-nsis) + (package-with-extra-patches base-nsis + (search-our-patches "nsis-SConstruct-sde-support.patch"))) + +(define-public font-tuffy + (package + (name "font-tuffy") + (version "20120614") + (source + (origin + (method url-fetch) + (uri (string-append "http://tulrich.com/fonts/tuffy-" version ".tar.gz")) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + "02vf72bgrp30vrbfhxjw82s115z27dwfgnmmzfb0n9wfhxxfpyf6")))) + (build-system font-build-system) + (home-page "http://tulrich.com/fonts/") + (synopsis "The Tuffy Truetype Font Family") + (description + "Thatcher Ulrich's first outline font design. He started with the goal of producing a neutral, readable sans-serif text font. There are lots of \"expressive\" fonts out there, but he wanted to start with something very plain and clean, something he might want to actually use. ") + (license license:public-domain))) + +(define-public lief + (package + (name "python-lief") + (version "0.11.5") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/lief-project/LIEF.git") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0qahjfg1n0x76ps2mbyljvws1l3qhkqvmxqbahps4qgywl2hbdkj")))) + (build-system python-build-system) + (native-inputs + `(("cmake" ,cmake))) + (home-page "https://github.com/lief-project/LIEF") + (synopsis "Library to Instrument Executable Formats") + (description "Python library to to provide a cross platform library which can +parse, modify and abstract ELF, PE and MachO formats.") + (license license:asl2.0))) + +(define osslsigncode + (package + (name "osslsigncode") + (version "2.0") + (source (origin + (method url-fetch) + (uri (string-append "https://github.com/mtrojnar/" + name "/archive/" version ".tar.gz")) + (sha256 + (base32 + "0byri6xny770wwb2nciq44j5071122l14bvv65axdd70nfjf0q2s")))) + (build-system gnu-build-system) + (native-inputs + `(("pkg-config" ,pkg-config) + ("autoconf" ,autoconf) + ("automake" ,automake) + ("libtool" ,libtool))) + (inputs + `(("openssl" ,openssl))) + (arguments + `(#:configure-flags + `("--without-gsf" + "--without-curl" + "--disable-dependency-tracking"))) + (home-page "https://github.com/mtrojnar/osslsigncode") + (synopsis "Authenticode signing and timestamping tool") + (description "osslsigncode is a small tool that implements part of the +functionality of the Microsoft tool signtool.exe - more exactly the Authenticode +signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and +thus should be able to compile on most platforms where these exist.") + (license license:gpl3+))) ; license is with openssl exception + +(define-public python-asn1crypto + (package + (name "python-asn1crypto") + (version "1.4.0") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/wbond/asn1crypto") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "19abibn6jw20mzi1ln4n9jjvpdka8ygm4m439hplyrdfqbvgm01r")))) + (build-system python-build-system) + (arguments + '(#:phases + (modify-phases %standard-phases + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests")))))) + (home-page "https://github.com/wbond/asn1crypto") + (synopsis "ASN.1 parser and serializer in Python") + (description "asn1crypto is an ASN.1 parser and serializer with definitions +for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, +PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") + (license license:expat))) + +(define-public python-elfesteem + (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5")) + (package + (name "python-elfesteem") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/LRGH/elfesteem") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")))) + (build-system python-build-system) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; PYTHONPATH problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/LRGH/elfesteem") + (synopsis "ELF/PE/Mach-O parsing library") + (description "elfesteem parses ELF, PE and Mach-O files.") + (license license:lgpl2.1)))) + +(define-public python-oscrypto + (package + (name "python-oscrypto") + (version "1.2.1") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/wbond/oscrypto") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1d4d8s4z340qhvb3g5m5v3436y3a71yc26wk4749q64m09kxqc3l")) + (patches (search-our-patches "oscrypto-hard-code-openssl.patch")))) + (build-system python-build-system) + (native-search-paths + (list (search-path-specification + (variable "SSL_CERT_FILE") + (file-type 'regular) + (separator #f) ;single entry + (files '("etc/ssl/certs/ca-certificates.crt"))))) + + (propagated-inputs + `(("python-asn1crypto" ,python-asn1crypto) + ("openssl" ,openssl))) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (let ((openssl (assoc-ref inputs "openssl"))) + (substitute* "oscrypto/__init__.py" + (("@GUIX_OSCRYPTO_USE_OPENSSL@") + (string-append openssl "/lib/libcrypto.so" "," openssl "/lib/libssl.so"))) + #t))) + (add-after 'unpack 'disable-broken-tests + (lambda _ + ;; This test is broken as there is no keyboard interrupt. + (substitute* "tests/test_trust_list.py" + (("^(.*)class TrustListTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_tls.py" + (("^(.*)class TLSTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/oscrypto") + (synopsis "Compiler-free Python crypto library backed by the OS") + (description "oscrypto is a compilation-free, always up-to-date encryption library for Python.") + (license license:expat))) + +(define-public python-oscryptotests + (package (inherit python-oscrypto) + (name "python-oscryptotests") + (arguments + `(#:tests? #f + #:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (chdir "tests") + #t))))))) + +(define-public python-certvalidator + (let ((commit "e5bdb4bfcaa09fa0af355eb8867d00dfeecba08c")) + (package + (name "python-certvalidator") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/certvalidator") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "18pvxkvpkfkzgvfylv0kx65pmxfcv1hpsg03cip93krfvrrl4c75")))) + (build-system python-build-system) + (propagated-inputs + `(("python-asn1crypto" ,python-asn1crypto) + ("python-oscrypto" ,python-oscrypto) + ("python-oscryptotests", python-oscryptotests))) ;; certvalidator tests import oscryptotests + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'disable-broken-tests + (lambda _ + (substitute* "tests/test_certificate_validator.py" + (("^(.*)class CertificateValidatorTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_crl_client.py" + (("^(.*)def test_fetch_crl" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_ocsp_client.py" + (("^(.*)def test_fetch_ocsp" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_registry.py" + (("^(.*)def test_build_paths" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_hard" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/certvalidator") + (synopsis "Python library for validating X.509 certificates and paths") + (description "certvalidator is a Python library for validating X.509 +certificates or paths. Supports various options, including: validation at a +specific moment in time, whitelisting and revocation checks.") + (license license:expat)))) + +(define-public python-requests-2.25.1 + (package (inherit python-requests) + (version "2.25.1") + (source (origin + (method url-fetch) + (uri (pypi-uri "requests" version)) + (sha256 + (base32 + "015qflyqsgsz09gnar69s6ga74ivq5kch69s4qxz3904m7a3v5r7")))))) + +(define-public python-altgraph + (package + (name "python-altgraph") + (version "0.17") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/ronaldoussoren/altgraph") + (commit (string-append "v" version)))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "09sm4srvvkw458pn48ga9q7ykr4xlz7q8gh1h9w7nxpf001qgpwb")))) + (build-system python-build-system) + (home-page "https://github.com/ronaldoussoren/altgraph") + (synopsis "Python graph (network) package") + (description "altgraph is a fork of graphlib: a graph (network) package for +constructing graphs, BFS and DFS traversals, topological sort, shortest paths, +etc. with graphviz output.") + (license license:expat))) + + +(define-public python-macholib + (package + (name "python-macholib") + (version "1.14") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/ronaldoussoren/macholib") + (commit (string-append "v" version)))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0aislnnfsza9wl4f0vp45ivzlc0pzhp9d4r08700slrypn5flg42")))) + (build-system python-build-system) + (propagated-inputs + `(("python-altgraph" ,python-altgraph))) + (arguments + '(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'disable-broken-tests + (lambda _ + ;; This test is broken as there is no keyboard interrupt. + (substitute* "macholib_tests/test_command_line.py" + (("^(.*)class TestCmdLine" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "macholib_tests/test_dyld.py" + (("^(.*)def test_\\S+_find" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line)) + (("^(.*)def testBasic" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line)) + ) + #t))))) + (home-page "https://github.com/ronaldoussoren/macholib") + (synopsis "Python library for analyzing and editing Mach-O headers") + (description "macholib is a Macho-O header analyzer and editor. It's +typically used as a dependency analysis tool, and also to rewrite dylib +references in Mach-O headers to be @executable_path relative. Though this tool +targets a platform specific file format, it is pure python code that is platform +and endian independent.") + (license license:expat))) + +(define-public python-signapple + (let ((commit "b084cbbf44d5330448ffce0c7d118f75781b64bd")) + (package + (name "python-signapple") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/signapple") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "0k7inccl2mzac3wq4asbr0kl8s4cghm8982z54kfascqg45shv01")))) + (build-system python-build-system) + (propagated-inputs + `(("python-asn1crypto" ,python-asn1crypto) + ("python-oscrypto" ,python-oscrypto) + ("python-certvalidator" ,python-certvalidator) + ("python-elfesteem" ,python-elfesteem) + ("python-requests" ,python-requests-2.25.1) + ("python-macholib" ,python-macholib) + ("libcrypto" ,openssl))) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/achow101/signapple") + (synopsis "Mach-O binary signature tool") + (description "signapple is a Python tool for creating, verifying, and +inspecting signatures in Mach-O binaries.") + (license license:expat)))) + +(define-public glibc-2.24 + (package + (inherit glibc) + (version "2.24") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "0d7f1ed30969886c8dde62fbf7d2c79967d4bace"))) + (file-name (git-file-name "glibc" "0d7f1ed30969886c8dde62fbf7d2c79967d4bace")) + (sha256 + (base32 + "0g5hryia5v1k0qx97qffgwzrz4lr4jw3s5kj04yllhswsxyjbic3")) + (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-versioned-locpath.patch" + "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" + "glibc-2.24-no-build-time-cxx-header-run.patch")))))) + +(define glibc-2.27/bitcoin-patched + (package-with-extra-patches glibc-2.27 + (search-our-patches "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch"))) + +(packages->manifest + (append + (list ;; The Basics + bash + which + coreutils + util-linux + ;; File(system) inspection + file + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + moreutils + ;; Compression and archiving + tar + bzip2 + gzip + xz + zlib + (list zlib "static") + ;; Build tools + gnu-make + libtool + autoconf + automake + pkg-config + bison + ;; Scripting + perl + python-3 + ;; Git + git + ;; Tests + lief + ;; Native gcc 7 toolchain + gcc-toolchain-7 + (list gcc-toolchain-7 "static") + ;; cmake + cmake) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + ;; Windows + (list zip + (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") + (make-nsis-with-sde-support nsis-x86_64) + osslsigncode)) + ((string-contains target "-linux-") + (list (cond ((string-contains target "riscv64-") + (make-bitcoin-cross-toolchain target + #:base-libc glibc-2.27/bitcoin-patched + #:base-kernel-headers linux-libre-headers-4.19)) + (else + (make-bitcoin-cross-toolchain target))))) + ((string-contains target "darwin") + (list clang-toolchain-10 binutils imagemagick libtiff librsvg font-tuffy xorriso python-signapple)) + (else '()))))) diff --git a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch new file mode 100644 index 0000000000..8f88eb9dfd --- /dev/null +++ b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch @@ -0,0 +1,171 @@ +Description: Add disable opposites to the security-related flags +Author: Stephen Kitt + +This patch adds "no-" variants to disable the various security flags: +"no-dynamicbase", "no-nxcompat", "no-high-entropy-va", "disable-reloc-section". + +--- a/ld/emultempl/pe.em ++++ b/ld/emultempl/pe.em +@@ -259,9 +261,11 @@ + (OPTION_ENABLE_LONG_SECTION_NAMES + 1) + /* DLLCharacteristics flags. */ + #define OPTION_DYNAMIC_BASE (OPTION_DISABLE_LONG_SECTION_NAMES + 1) +-#define OPTION_FORCE_INTEGRITY (OPTION_DYNAMIC_BASE + 1) ++#define OPTION_NO_DYNAMIC_BASE (OPTION_DYNAMIC_BASE + 1) ++#define OPTION_FORCE_INTEGRITY (OPTION_NO_DYNAMIC_BASE + 1) + #define OPTION_NX_COMPAT (OPTION_FORCE_INTEGRITY + 1) +-#define OPTION_NO_ISOLATION (OPTION_NX_COMPAT + 1) ++#define OPTION_NO_NX_COMPAT (OPTION_NX_COMPAT + 1) ++#define OPTION_NO_ISOLATION (OPTION_NO_NX_COMPAT + 1) + #define OPTION_NO_SEH (OPTION_NO_ISOLATION + 1) + #define OPTION_NO_BIND (OPTION_NO_SEH + 1) + #define OPTION_WDM_DRIVER (OPTION_NO_BIND + 1) +@@ -271,6 +275,7 @@ + #define OPTION_NO_INSERT_TIMESTAMP (OPTION_INSERT_TIMESTAMP + 1) + #define OPTION_BUILD_ID (OPTION_NO_INSERT_TIMESTAMP + 1) + #define OPTION_ENABLE_RELOC_SECTION (OPTION_BUILD_ID + 1) ++#define OPTION_DISABLE_RELOC_SECTION (OPTION_ENABLE_RELOC_SECTION + 1) + + static void + gld${EMULATION_NAME}_add_options +@@ -342,8 +347,10 @@ + {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, + {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, + {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, ++ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, + {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, + {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, ++ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, + {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, + {"no-seh", no_argument, NULL, OPTION_NO_SEH}, + {"no-bind", no_argument, NULL, OPTION_NO_BIND}, +@@ -351,6 +358,7 @@ + {"tsaware", no_argument, NULL, OPTION_TERMINAL_SERVER_AWARE}, + {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, + {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, ++ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, + {NULL, no_argument, NULL, 0} + }; + +@@ -485,9 +494,12 @@ + in object files\n")); + fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ + address space layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); + fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); ++ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); + fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); + fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); ++ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); + fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); + fprintf (file, _(" --no-seh Image does not use SEH. No SE handler may\n\ + be called in this image\n")); +@@ -862,12 +874,21 @@ + case OPTION_ENABLE_RELOC_SECTION: + pe_dll_enable_reloc_section = 1; + break; ++ case OPTION_DISABLE_RELOC_SECTION: ++ pe_dll_enable_reloc_section = 0; ++ /* fall through */ ++ case OPTION_NO_DYNAMIC_BASE: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; ++ break; + case OPTION_FORCE_INTEGRITY: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; + break; + case OPTION_NX_COMPAT: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; + break; ++ case OPTION_NO_NX_COMPAT: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; ++ break; + case OPTION_NO_ISOLATION: + pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; + break; +--- a/ld/emultempl/pep.em ++++ b/ld/emultempl/pep.em +@@ -237,9 +240,12 @@ + OPTION_ENABLE_LONG_SECTION_NAMES, + OPTION_DISABLE_LONG_SECTION_NAMES, + OPTION_HIGH_ENTROPY_VA, ++ OPTION_NO_HIGH_ENTROPY_VA, + OPTION_DYNAMIC_BASE, ++ OPTION_NO_DYNAMIC_BASE, + OPTION_FORCE_INTEGRITY, + OPTION_NX_COMPAT, ++ OPTION_NO_NX_COMPAT, + OPTION_NO_ISOLATION, + OPTION_NO_SEH, + OPTION_NO_BIND, +@@ -248,7 +254,8 @@ + OPTION_NO_INSERT_TIMESTAMP, + OPTION_TERMINAL_SERVER_AWARE, + OPTION_BUILD_ID, +- OPTION_ENABLE_RELOC_SECTION ++ OPTION_ENABLE_RELOC_SECTION, ++ OPTION_DISABLE_RELOC_SECTION + }; + + static void +@@ -315,9 +322,12 @@ + {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, + {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, + {"high-entropy-va", no_argument, NULL, OPTION_HIGH_ENTROPY_VA}, ++ {"no-high-entropy-va", no_argument, NULL, OPTION_NO_HIGH_ENTROPY_VA}, + {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, ++ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, + {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, + {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, ++ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, + {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, + {"no-seh", no_argument, NULL, OPTION_NO_SEH}, + {"no-bind", no_argument, NULL, OPTION_NO_BIND}, +@@ -327,6 +337,7 @@ + {"no-insert-timestamp", no_argument, NULL, OPTION_NO_INSERT_TIMESTAMP}, + {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, + {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, ++ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, + {NULL, no_argument, NULL, 0} + }; + +@@ -448,11 +461,15 @@ + in object files\n")); + fprintf (file, _(" --high-entropy-va Image is compatible with 64-bit address space\n\ + layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-high-entropy-va Image is not compatible with 64-bit ASLR\n")); + fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ + address space layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); + fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); ++ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); + fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); + fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); ++ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); + fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); + fprintf (file, _(" --no-seh Image does not use SEH; no SE handler may\n\ + be called in this image\n")); +@@ -809,12 +826,24 @@ + case OPTION_ENABLE_RELOC_SECTION: + pep_dll_enable_reloc_section = 1; + break; ++ case OPTION_DISABLE_RELOC_SECTION: ++ pep_dll_enable_reloc_section = 0; ++ /* fall through */ ++ case OPTION_NO_DYNAMIC_BASE: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; ++ /* fall through */ ++ case OPTION_NO_HIGH_ENTROPY_VA: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA; ++ break; + case OPTION_FORCE_INTEGRITY: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; + break; + case OPTION_NX_COMPAT: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; + break; ++ case OPTION_NO_NX_COMPAT: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; ++ break; + case OPTION_NO_ISOLATION: + pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; + break; diff --git a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch new file mode 100644 index 0000000000..f327c464f3 --- /dev/null +++ b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch @@ -0,0 +1,400 @@ +guix: repro: Sort find output in libtool for gcc-8 + +Otherwise the resulting .a static libraries (e.g. libstdc++.a) will not +be reproducible and end up making the Bitcoin binaries non-reproducible +as well. + +See: https://reproducible-builds.org/docs/archives/#gnu-libtool + +diff --git a/gcc/configure b/gcc/configure +index 97ba7d7d69c..e37a96f0c0c 100755 +--- a/gcc/configure ++++ b/gcc/configure +@@ -19720,20 +19720,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libcc1/configure b/libcc1/configure +index f53a121611c..5740ca90cab 100755 +--- a/libcc1/configure ++++ b/libcc1/configure +@@ -12221,20 +12221,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libffi/configure b/libffi/configure +index 790a291011f..54b1ac18306 100755 +--- a/libffi/configure ++++ b/libffi/configure +@@ -12661,20 +12661,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libgo/config/libtool.m4 b/libgo/config/libtool.m4 +index f7005947454..8a84417b828 100644 +--- a/libgo/config/libtool.m4 ++++ b/libgo/config/libtool.m4 +@@ -6010,20 +6010,20 @@ if test "$_lt_caught_CXX_error" != yes; then + _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libgo/config/ltmain.sh b/libgo/config/ltmain.sh +index ce66b44906a..0f81c401407 100644 +--- a/libgo/config/ltmain.sh ++++ b/libgo/config/ltmain.sh +@@ -2917,7 +2917,7 @@ func_extract_archives () + darwin_file= + darwin_files= + for darwin_file in $darwin_filelist; do +- darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP` ++ darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` + $LIPO -create -output "$darwin_file" $darwin_files + done # $darwin_filelist + $RM -rf unfat-$$ +@@ -2932,7 +2932,7 @@ func_extract_archives () + func_extract_an_archive "$my_xdir" "$my_xabs" + ;; + esac +- my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP` ++ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` + done + + func_extract_archives_result="$my_oldobjs" +diff --git a/libhsail-rt/configure b/libhsail-rt/configure +index a4fcc10c1f9..8e671229fcd 100755 +--- a/libhsail-rt/configure ++++ b/libhsail-rt/configure +@@ -12244,20 +12244,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libitm/configure b/libitm/configure +index dbf386db434..29d4f10611f 100644 +--- a/libitm/configure ++++ b/libitm/configure +@@ -13067,20 +13067,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/liboffloadmic/configure b/liboffloadmic/configure +index f873716991b..7aa9186b10e 100644 +--- a/liboffloadmic/configure ++++ b/liboffloadmic/configure +@@ -12379,20 +12379,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/liboffloadmic/plugin/configure b/liboffloadmic/plugin/configure +index c031eb3e7fa..67fc7368f21 100644 +--- a/liboffloadmic/plugin/configure ++++ b/liboffloadmic/plugin/configure +@@ -12086,20 +12086,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libsanitizer/configure b/libsanitizer/configure +index 4695bc7d4f7..cb7d25c07e6 100755 +--- a/libsanitizer/configure ++++ b/libsanitizer/configure +@@ -13308,20 +13308,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libstdc++-v3/configure b/libstdc++-v3/configure +index 61457e940ec..21ef1f61e41 100755 +--- a/libstdc++-v3/configure ++++ b/libstdc++-v3/configure +@@ -13087,20 +13087,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libtool.m4 b/libtool.m4 +index 24d13f34409..940faaa161d 100644 +--- a/libtool.m4 ++++ b/libtool.m4 +@@ -6005,20 +6005,20 @@ if test "$_lt_caught_CXX_error" != yes; then + _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/libvtv/configure b/libvtv/configure +index a197f750453..31ab3a0637b 100755 +--- a/libvtv/configure ++++ b/libvtv/configure +@@ -13339,20 +13339,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi + prelink_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ +- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' ++ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' + old_archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ +- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ ++ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ + $RANLIB $oldlib' + archive_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' + archive_expsym_cmds_CXX='tpldir=Template.dir~ + rm -rf $tpldir~ + $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ +- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' ++ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' + ;; + *) # Version 6 and above use weak symbols + archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' +diff --git a/ltmain.sh b/ltmain.sh +index 9503ec85d70..79f9ba89af5 100644 +--- a/ltmain.sh ++++ b/ltmain.sh +@@ -2917,7 +2917,7 @@ func_extract_archives () + darwin_file= + darwin_files= + for darwin_file in $darwin_filelist; do +- darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP` ++ darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` + $LIPO -create -output "$darwin_file" $darwin_files + done # $darwin_filelist + $RM -rf unfat-$$ +@@ -2932,7 +2932,7 @@ func_extract_archives () + func_extract_an_archive "$my_xdir" "$my_xabs" + ;; + esac +- my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP` ++ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` + done + + func_extract_archives_result="$my_oldobjs" diff --git a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch new file mode 100644 index 0000000000..5c4d0c6ebe --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch @@ -0,0 +1,62 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=a68ba2f3cd3cbe32c1f31e13c20ed13487727b32 + +commit 6b02af31e9a721bb15a11380cd22d53b621711f8 +Author: Szabolcs Nagy +Date: Wed Oct 18 17:26:23 2017 +0100 + + [AARCH64] Rewrite elf_machine_load_address using _DYNAMIC symbol + + This patch rewrites aarch64 elf_machine_load_address to use special _DYNAMIC + symbol instead of _dl_start. + + The static address of _DYNAMIC symbol is stored in the first GOT entry. + Here is the change which makes this solution work (part of binutils 2.24): + https://sourceware.org/ml/binutils/2013-06/msg00248.html + + i386, x86_64 targets use the same method to do this as well. + + The original implementation relies on a trick that R_AARCH64_ABS32 relocation + being resolved at link time and the static address fits in the 32bits. + However, in LP64, normally, the address is defined to be 64 bit. + + Here is the C version one which should be portable in all cases. + + * sysdeps/aarch64/dl-machine.h (elf_machine_load_address): Use + _DYNAMIC symbol to calculate load address. + +diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h +index e86d8b5b63..5a5b8a5de5 100644 +--- a/sysdeps/aarch64/dl-machine.h ++++ b/sysdeps/aarch64/dl-machine.h +@@ -49,26 +49,11 @@ elf_machine_load_address (void) + /* To figure out the load address we use the definition that for any symbol: + dynamic_addr(symbol) = static_addr(symbol) + load_addr + +- The choice of symbol is arbitrary. The static address we obtain +- by constructing a non GOT reference to the symbol, the dynamic +- address of the symbol we compute using adrp/add to compute the +- symbol's address relative to the PC. +- This depends on 32bit relocations being resolved at link time +- and that the static address fits in the 32bits. */ +- +- ElfW(Addr) static_addr; +- ElfW(Addr) dynamic_addr; +- +- asm (" \n" +-" adrp %1, _dl_start; \n" +-" add %1, %1, #:lo12:_dl_start \n" +-" ldr %w0, 1f \n" +-" b 2f \n" +-"1: \n" +-" .word _dl_start \n" +-"2: \n" +- : "=r" (static_addr), "=r" (dynamic_addr)); +- return dynamic_addr - static_addr; ++ _DYNAMIC sysmbol is used here as its link-time address stored in ++ the special unrelocated first GOT entry. */ ++ ++ extern ElfW(Dyn) _DYNAMIC[] attribute_hidden; ++ return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic (); + } + + /* Set up the loaded object described by L so its unrelocated PLT diff --git a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch new file mode 100644 index 0000000000..11fe7fdc99 --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch @@ -0,0 +1,100 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=fc3e1337be1c6935ab58bd13520f97a535cf70cc + +commit dc23a45db566095e83ff0b7a57afc87fb5ca89a1 +Author: Florian Weimer +Date: Wed Sep 21 10:45:32 2016 +0200 + + Avoid running $(CXX) during build to obtain header file paths + + This reduces the build time somewhat and is particularly noticeable + during rebuilds with few code changes. + +diff --git a/Makerules b/Makerules +index 7e4077ee50..c338850de5 100644 +--- a/Makerules ++++ b/Makerules +@@ -121,14 +121,10 @@ ifneq (,$(CXX)) + # will be used instead of /usr/include/stdlib.h and /usr/include/math.h. + before-compile := $(common-objpfx)cstdlib $(common-objpfx)cmath \ + $(before-compile) +-cstdlib=$(shell echo "\#include " | $(CXX) -M -MP -x c++ - \ +- | sed -n "/cstdlib:/{s/:$$//;p}") +-$(common-objpfx)cstdlib: $(cstdlib) ++$(common-objpfx)cstdlib: $(c++-cstdlib-header) + $(INSTALL_DATA) $< $@T + $(move-if-change) $@T $@ +-cmath=$(shell echo "\#include " | $(CXX) -M -MP -x c++ - \ +- | sed -n "/cmath:/{s/:$$//;p}") +-$(common-objpfx)cmath: $(cmath) ++$(common-objpfx)cmath: $(c++-cmath-header) + $(INSTALL_DATA) $< $@T + $(move-if-change) $@T $@ + endif +diff --git a/config.make.in b/config.make.in +index 95c6f36876..04a8b3ed7f 100644 +--- a/config.make.in ++++ b/config.make.in +@@ -45,6 +45,8 @@ defines = @DEFINES@ + sysheaders = @sysheaders@ + sysincludes = @SYSINCLUDES@ + c++-sysincludes = @CXX_SYSINCLUDES@ ++c++-cstdlib-header = @CXX_CSTDLIB_HEADER@ ++c++-cmath-header = @CXX_CMATH_HEADER@ + all-warnings = @all_warnings@ + enable-werror = @enable_werror@ + +diff --git a/configure b/configure +index 17625e1041..6ff252744b 100755 +--- a/configure ++++ b/configure +@@ -635,6 +635,8 @@ BISON + INSTALL_INFO + PERL + BASH_SHELL ++CXX_CMATH_HEADER ++CXX_CSTDLIB_HEADER + CXX_SYSINCLUDES + SYSINCLUDES + AUTOCONF +@@ -5054,6 +5056,18 @@ fi + + + ++# Obtain some C++ header file paths. This is used to make a local ++# copy of those headers in Makerules. ++if test -n "$CXX"; then ++ find_cxx_header () { ++ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" ++ } ++ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" ++ CXX_CMATH_HEADER="$(find_cxx_header cmath)" ++fi ++ ++ ++ + # Test if LD_LIBRARY_PATH contains the notation for the current directory + # since this would lead to problems installing/building glibc. + # LD_LIBRARY_PATH contains the current directory if one of the following +diff --git a/configure.ac b/configure.ac +index 33bcd62180..9938ab0dc2 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1039,6 +1039,18 @@ fi + AC_SUBST(SYSINCLUDES) + AC_SUBST(CXX_SYSINCLUDES) + ++# Obtain some C++ header file paths. This is used to make a local ++# copy of those headers in Makerules. ++if test -n "$CXX"; then ++ find_cxx_header () { ++ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" ++ } ++ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" ++ CXX_CMATH_HEADER="$(find_cxx_header cmath)" ++fi ++AC_SUBST(CXX_CSTDLIB_HEADER) ++AC_SUBST(CXX_CMATH_HEADER) ++ + # Test if LD_LIBRARY_PATH contains the notation for the current directory + # since this would lead to problems installing/building glibc. + # LD_LIBRARY_PATH contains the current directory if one of the following diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch new file mode 100644 index 0000000000..d6217157ee --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch @@ -0,0 +1,72 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e + +From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Mon, 31 Dec 2018 09:26:42 -0800 +Subject: [PATCH] riscv: Use __has_include__ to include [BZ + #24022] + + has been removed by + +commit 27f8899d6002e11a6e2d995e29b8deab5aa9cc25 +Author: David Abdurachmanov +Date: Thu Nov 8 20:02:39 2018 +0100 + + riscv: add asm/unistd.h UAPI header + + Marcin Juszkiewicz reported issues while generating syscall table for riscv + using 4.20-rc1. The patch refactors our unistd.h files to match some other + architectures. + + - Add asm/unistd.h UAPI header, which has __ARCH_WANT_NEW_STAT only for 64-bit + - Remove asm/syscalls.h UAPI header and merge to asm/unistd.h + - Adjust kernel asm/unistd.h + + So now asm/unistd.h UAPI header should show all syscalls for riscv. + + may be restored by + +Subject: [PATCH] riscv: restore asm/syscalls.h UAPI header +Date: Tue, 11 Dec 2018 09:09:35 +0100 + +UAPI header asm/syscalls.h was merged into UAPI asm/unistd.h header, +which did resolve issue with missing syscalls macros resulting in +glibc (2.28) build failure. It also broke glibc in a different way: +asm/syscalls.h is being used by glibc. I noticed this while doing +Fedora 30/Rawhide mass rebuild. + +The patch returns asm/syscalls.h header and incl. it into asm/unistd.h. +I plan to send a patch to glibc to use asm/unistd.h instead of +asm/syscalls.h + +In the meantime, we use __has_include__, which was added to GCC 5, to +check if exists before including it. Tested with +build-many-glibcs.py for riscv against kernel 4.19.12 and 4.20-rc7. + + [BZ #24022] + * sysdeps/unix/sysv/linux/riscv/flush-icache.c: Check if + exists with __has_include__ before including it. +--- + sysdeps/unix/sysv/linux/riscv/flush-icache.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/sysdeps/unix/sysv/linux/riscv/flush-icache.c b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +index d612ef4c6c..0b2042620b 100644 +--- a/sysdeps/unix/sysv/linux/riscv/flush-icache.c ++++ b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +@@ -21,7 +21,11 @@ + #include + #include + #include +-#include ++#if __has_include__ () ++# include ++#else ++# include ++#endif + + typedef int (*func_type) (void *, void *, unsigned long int); + +-- +2.31.1 + diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch new file mode 100644 index 0000000000..b1b6d5a548 --- /dev/null +++ b/contrib/guix/patches/glibc-ldd-x86_64.patch @@ -0,0 +1,10 @@ +By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas +it's in 'lib/' for us. This patch fixes that. + +--- glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2012-12-25 04:02:13.000000000 +0100 ++++ glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2013-09-15 23:08:03.000000000 +0200 +@@ -1,3 +1,3 @@ + /LD_TRACE_LOADED_OBJECTS=1/a\ + add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" +-s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \264\4-x86-64\6 \2x32\4-x32\6"_ ++s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \2\4-x86-64\6 \2x32\4-x32\6"_ diff --git a/contrib/guix/patches/glibc-versioned-locpath.patch b/contrib/guix/patches/glibc-versioned-locpath.patch new file mode 100644 index 0000000000..bc7652127f --- /dev/null +++ b/contrib/guix/patches/glibc-versioned-locpath.patch @@ -0,0 +1,240 @@ +The format of locale data can be incompatible between libc versions, and +loading incompatible data can lead to 'setlocale' returning EINVAL at best +or triggering an assertion failure at worst. See +https://lists.gnu.org/archive/html/guix-devel/2015-09/msg00717.html +for background information. + +To address that, this patch changes libc to honor a new 'GUIX_LOCPATH' +variable, and to look for locale data in version-specific sub-directories of +that variable. So, if GUIX_LOCPATH=/foo:/bar, locale data is searched for in +/foo/X.Y and /bar/X.Y, where X.Y is the libc version number. + +That way, a single 'GUIX_LOCPATH' setting can work even if different libc +versions coexist on the system. + +--- a/locale/newlocale.c ++++ b/locale/newlocale.c +@@ -30,6 +30,7 @@ + /* Lock for protecting global data. */ + __libc_rwlock_define (extern , __libc_setlocale_lock attribute_hidden) + ++extern error_t compute_locale_search_path (char **, size_t *); + + /* Use this when we come along an error. */ + #define ERROR_RETURN \ +@@ -48,7 +49,6 @@ __newlocale (int category_mask, const char *locale, __locale_t base) + __locale_t result_ptr; + char *locale_path; + size_t locale_path_len; +- const char *locpath_var; + int cnt; + size_t names_len; + +@@ -102,17 +102,8 @@ __newlocale (int category_mask, const char *locale, __locale_t base) + locale_path = NULL; + locale_path_len = 0; + +- locpath_var = getenv ("LOCPATH"); +- if (locpath_var != NULL && locpath_var[0] != '\0') +- { +- if (__argz_create_sep (locpath_var, ':', +- &locale_path, &locale_path_len) != 0) +- return NULL; +- +- if (__argz_add_sep (&locale_path, &locale_path_len, +- _nl_default_locale_path, ':') != 0) +- return NULL; +- } ++ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) ++ return NULL; + + /* Get the names for the locales we are interested in. We either + allow a composite name or a single name. */ +diff --git a/locale/setlocale.c b/locale/setlocale.c +index ead030d..0c0e314 100644 +--- a/locale/setlocale.c ++++ b/locale/setlocale.c +@@ -215,12 +215,65 @@ setdata (int category, struct __locale_data *data) + } + } + ++/* Return in *LOCALE_PATH and *LOCALE_PATH_LEN the locale data search path as ++ a colon-separated list. Return ENOMEN on error, zero otherwise. */ ++error_t ++compute_locale_search_path (char **locale_path, size_t *locale_path_len) ++{ ++ char* guix_locpath_var = getenv ("GUIX_LOCPATH"); ++ char *locpath_var = getenv ("LOCPATH"); ++ ++ if (guix_locpath_var != NULL && guix_locpath_var[0] != '\0') ++ { ++ /* Entries in 'GUIX_LOCPATH' take precedence over 'LOCPATH'. These ++ entries are systematically prefixed with "/X.Y" where "X.Y" is the ++ libc version. */ ++ if (__argz_create_sep (guix_locpath_var, ':', ++ locale_path, locale_path_len) != 0 ++ || __argz_suffix_entries (locale_path, locale_path_len, ++ "/" VERSION) != 0) ++ goto bail_out; ++ } ++ ++ if (locpath_var != NULL && locpath_var[0] != '\0') ++ { ++ char *reg_locale_path = NULL; ++ size_t reg_locale_path_len = 0; ++ ++ if (__argz_create_sep (locpath_var, ':', ++ ®_locale_path, ®_locale_path_len) != 0) ++ goto bail_out; ++ ++ if (__argz_append (locale_path, locale_path_len, ++ reg_locale_path, reg_locale_path_len) != 0) ++ goto bail_out; ++ ++ free (reg_locale_path); ++ } ++ ++ if (*locale_path != NULL) ++ { ++ /* Append the system default locale directory. */ ++ if (__argz_add_sep (locale_path, locale_path_len, ++ _nl_default_locale_path, ':') != 0) ++ goto bail_out; ++ } ++ ++ return 0; ++ ++ bail_out: ++ free (*locale_path); ++ *locale_path = NULL; ++ *locale_path_len = 0; ++ ++ return ENOMEM; ++} ++ + char * + setlocale (int category, const char *locale) + { + char *locale_path; + size_t locale_path_len; +- const char *locpath_var; + char *composite; + + /* Sanity check for CATEGORY argument. */ +@@ -251,17 +304,10 @@ setlocale (int category, const char *locale) + locale_path = NULL; + locale_path_len = 0; + +- locpath_var = getenv ("LOCPATH"); +- if (locpath_var != NULL && locpath_var[0] != '\0') ++ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) + { +- if (__argz_create_sep (locpath_var, ':', +- &locale_path, &locale_path_len) != 0 +- || __argz_add_sep (&locale_path, &locale_path_len, +- _nl_default_locale_path, ':') != 0) +- { +- __libc_rwlock_unlock (__libc_setlocale_lock); +- return NULL; +- } ++ __libc_rwlock_unlock (__libc_setlocale_lock); ++ return NULL; + } + + if (category == LC_ALL) +diff --git a/string/Makefile b/string/Makefile +index 8424a61..f925503 100644 +--- a/string/Makefile ++++ b/string/Makefile +@@ -38,7 +38,7 @@ routines := strcat strchr strcmp strcoll strcpy strcspn \ + swab strfry memfrob memmem rawmemchr strchrnul \ + $(addprefix argz-,append count create ctsep next \ + delete extract insert stringify \ +- addsep replace) \ ++ addsep replace suffix) \ + envz basename \ + strcoll_l strxfrm_l string-inlines memrchr \ + xpg-strerror strerror_l +diff --git a/string/argz-suffix.c b/string/argz-suffix.c +new file mode 100644 +index 0000000..505b0f2 +--- /dev/null ++++ b/string/argz-suffix.c +@@ -0,0 +1,56 @@ ++/* Copyright (C) 2015 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ Contributed by Ludovic Courtès . ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ . */ ++ ++#include ++#include ++#include ++#include ++ ++ ++error_t ++__argz_suffix_entries (char **argz, size_t *argz_len, const char *suffix) ++ ++{ ++ size_t suffix_len = strlen (suffix); ++ size_t count = __argz_count (*argz, *argz_len); ++ size_t new_argz_len = *argz_len + count * suffix_len; ++ char *new_argz = malloc (new_argz_len); ++ ++ if (new_argz) ++ { ++ char *p = new_argz, *entry; ++ ++ for (entry = *argz; ++ entry != NULL; ++ entry = argz_next (*argz, *argz_len, entry)) ++ { ++ p = stpcpy (p, entry); ++ p = stpcpy (p, suffix); ++ p++; ++ } ++ ++ free (*argz); ++ *argz = new_argz; ++ *argz_len = new_argz_len; ++ ++ return 0; ++ } ++ else ++ return ENOMEM; ++} ++weak_alias (__argz_suffix_entries, argz_suffix_entries) +diff --git a/string/argz.h b/string/argz.h +index bb62a31..d276a35 100644 +--- a/string/argz.h ++++ b/string/argz.h +@@ -134,6 +134,16 @@ extern error_t argz_replace (char **__restrict __argz, + const char *__restrict __str, + const char *__restrict __with, + unsigned int *__restrict __replace_count); ++ ++/* Suffix each entry of ARGZ & ARGZ_LEN with SUFFIX. Return 0 on success, ++ and ENOMEN if memory cannot be allocated. */ ++extern error_t __argz_suffix_entries (char **__restrict __argz, ++ size_t *__restrict __argz_len, ++ const char *__restrict __suffix); ++extern error_t argz_suffix_entries (char **__restrict __argz, ++ size_t *__restrict __argz_len, ++ const char *__restrict __suffix); ++ + + /* Returns the next entry in ARGZ & ARGZ_LEN after ENTRY, or NULL if there + are no more. If entry is NULL, then the first entry is returned. This diff --git a/contrib/guix/patches/nsis-SConstruct-sde-support.patch b/contrib/guix/patches/nsis-SConstruct-sde-support.patch new file mode 100644 index 0000000000..f58406a7a0 --- /dev/null +++ b/contrib/guix/patches/nsis-SConstruct-sde-support.patch @@ -0,0 +1,18 @@ +https://github.com/kichik/nsis/pull/13 +https://sourceforge.net/p/nsis/code/7248/ + +diff --git a/SConstruct b/SConstruct +index e8252c9..41786f2 100755 +--- a/SConstruct ++++ b/SConstruct +@@ -95,8 +95,8 @@ default_doctype = 'html' + if defenv.WhereIs('hhc', os.environ['PATH']): + default_doctype = 'chm' + +-from time import strftime, gmtime +-cvs_version = strftime('%d-%b-%Y.cvs', gmtime()) ++import time ++cvs_version = time.strftime('%d-%b-%Y.cvs', time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) + + opts = Variables() + diff --git a/contrib/guix/patches/oscrypto-hard-code-openssl.patch b/contrib/guix/patches/oscrypto-hard-code-openssl.patch new file mode 100644 index 0000000000..32027f2d09 --- /dev/null +++ b/contrib/guix/patches/oscrypto-hard-code-openssl.patch @@ -0,0 +1,13 @@ +diff --git a/oscrypto/__init__.py b/oscrypto/__init__.py +index eb27313..371ab24 100644 +--- a/oscrypto/__init__.py ++++ b/oscrypto/__init__.py +@@ -302,3 +302,8 @@ def load_order(): + 'oscrypto._win.tls', + 'oscrypto.tls', + ] ++ ++ ++paths = '@GUIX_OSCRYPTO_USE_OPENSSL@'.split(',') ++assert len(paths) == 2, 'Value for OSCRYPTO_USE_OPENSSL env var must be two paths separated by a comma' ++use_openssl(*paths) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index be36422f87..4f654865ac 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -1,15 +1,120 @@ -### MacDeploy ### +# MacOS Deployment -For Snow Leopard (which uses [Python 2.6](http://www.python.org/download/releases/2.6/)), you will need the param_parser package: - - sudo easy_install argparse +The `macdeployqtplus` script should not be run manually. Instead, after building as usual: -This script should not be run manually, instead, after building as usual: +```bash +make deploy +``` - make deploy +When complete, it will have produced `Firo-Core.dmg`. -During the process, the disk image window will pop up briefly where the fancy -settings are applied. This is normal, please do not interfere. +## SDK Extraction -When finished, it will produce `Firo-Core.dmg`. +### Step 1: Obtaining `Xcode.app` +Our current macOS SDK +(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be +extracted from +[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip). +An Apple ID is needed to download this. + +After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` +archive. This makes the SDK less-trivial to extract on non-macOS machines. One +approach (tested on Debian Buster) is outlined below: + +```bash +# Install/clone tools needed for extracting Xcode.app +apt install cpio +git clone https://github.com/bitcoin-core/apple-sdk-tools.git + +# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current +# working directory +python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i +``` + +On macOS the process is more straightforward: + +```bash +xip -x Xcode_12.1.xip +``` + +### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` + +To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run +the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the +previous stage) as the first argument. + +```bash +# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from +# the supplied Xcode.app +./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' +``` + +## Deterministic macOS DMG Notes +Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple +`binutils` (`ld`, `ar`, etc) and DMG authoring tools. + +Apple uses `clang` extensively for development and has upstreamed the necessary +functionality so that a vanilla clang can take advantage. It supports the use of `-F`, +`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when +building for macOS. + +Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the +FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several +other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These +do not build under Linux, so they have been patched to do so. The work here was used as +a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). + +In order to build a working toolchain, the following source packages are needed from +Apple: `cctools`, `dyld`, and `ld64`. + +These tools inject timestamps by default, which produce non-deterministic binaries. The +`ZERO_AR_DATE` environment variable is used to disable that. + +This version of `cctools` has been patched to use the current version of `clang`'s headers +and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`. + +To complicate things further, all builds must target an Apple SDK. These SDKs are free to +download, but not redistributable. To obtain it, register for an Apple Developer Account, +then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). + +This file is many gigabytes in size, but most (but not all) of what we need is +contained only in a single directory: + +```bash +Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk +``` + +See the SDK Extraction notes above for how to obtain it. + +The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are +created using these tools. The build process has been designed to avoid including the +SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely +redistributed. + +[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. + +`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the +`libdmg-hfsplus` project is used to compress it. There are several bugs in this +tool and its maintainer has seemingly abandoned the project. + +The DMG tool has the ability to create DMGs from scratch as well, but this functionality is +broken. Only the compression feature is currently used. Ideally, the creation could be fixed +and `xorrisofs` would no longer be necessary. + +Background images and other features can be added to DMG files by inserting a +`.DS_Store` during creation. + +As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in +order to satisfy the new Gatekeeper requirements. Because this private key cannot be +shared, we'll have to be a bit creative in order for the build process to remain somewhat +deterministic. Here's how it works: + +- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which + users may choose to bless and run. It also outputs an unsigned app structure in the form + of a tarball, which also contains all of the tools that have been previously (deterministically) + built in order to create a final DMG. +- The Apple keyholder uses this unsigned app to create a detached signature, using the + script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). +- Builders feed the unsigned app + detached signature back into Gitian. It uses the + pre-built tools to recombine the pieces into a deterministic DMG. diff --git a/contrib/macdeploy/custom_dsstore.py b/contrib/macdeploy/custom_dsstore.py deleted file mode 100755 index 40f02f8c53..0000000000 --- a/contrib/macdeploy/custom_dsstore.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2013-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -from __future__ import division,print_function,unicode_literals -import biplist -from ds_store import DSStore -from mac_alias import Alias -import sys - -output_file = sys.argv[1] -package_name_ns = sys.argv[2] - -ds = DSStore.open(output_file, 'w+') -ds['.']['bwsp'] = { - 'ShowStatusBar': False, - 'WindowBounds': b'{{300, 280}, {500, 343}}', - 'ContainerShowSidebar': False, - 'SidebarWidth': 0, - 'ShowTabView': False, - 'PreviewPaneVisibility': False, - 'ShowToolbar': False, - 'ShowSidebar': False, - 'ShowPathbar': True -} - -icvp = { - 'gridOffsetX': 0.0, - 'textSize': 12.0, - 'viewOptionsVersion': 1, - 'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00', - 'backgroundColorBlue': 1.0, - 'iconSize': 96.0, - 'backgroundColorGreen': 1.0, - 'arrangeBy': 'none', - 'showIconPreview': True, - 'gridSpacing': 100.0, - 'gridOffsetY': 0.0, - 'showItemInfo': False, - 'labelOnBottom': True, - 'backgroundType': 2, - 'backgroundColorRed': 1.0 -} -alias = Alias.from_bytes(icvp['backgroundImageAlias']) -alias.volume.name = package_name_ns -alias.volume.posix_path = '/Volumes/' + package_name_ns -alias.volume.disk_image_alias.target.filename = package_name_ns + '.temp.dmg' -alias.volume.disk_image_alias.target.carbon_path = 'Macintosh HD:Users:\x00bitcoinuser:\x00Documents:\x00firo:\x00firo:\x00' + package_name_ns + '.temp.dmg' -alias.volume.disk_image_alias.target.posix_path = 'Users/bitcoinuser/Documents/firo/firo/' + package_name_ns + '.temp.dmg' -alias.target.carbon_path = package_name_ns + ':.background:\x00background.tiff' -icvp['backgroundImageAlias'] = biplist.Data(alias.to_bytes()) -ds['.']['icvp'] = icvp - -ds['.']['vSrn'] = ('long', 1) - -ds['Applications']['Iloc'] = (370, 156) -ds['Firo-Qt.app']['Iloc'] = (128, 156) - -ds.flush() -ds.close() diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index 91674a92e6..d481413cc3 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -1,16 +1,16 @@ #!/bin/sh -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e UNSIGNED="$1" SIGNATURE="$2" -ARCH=x86_64 ROOTDIR=dist -TEMPDIR=signed.temp OUTDIR=signed-app +SIGNAPPLE=signapple if [ -z "$UNSIGNED" ]; then echo "usage: $0 " @@ -22,35 +22,6 @@ if [ -z "$SIGNATURE" ]; then exit 1 fi -rm -rf ${TEMPDIR} && mkdir -p ${TEMPDIR} -tar -C ${TEMPDIR} -xf ${UNSIGNED} -cp -rf "${SIGNATURE}"/* ${TEMPDIR} - -if [ -z "${PAGESTUFF}" ]; then - PAGESTUFF=${TEMPDIR}/pagestuff -fi - -if [ -z "${CODESIGN_ALLOCATE}" ]; then - CODESIGN_ALLOCATE=${TEMPDIR}/codesign_allocate -fi - -find ${TEMPDIR} -name "*.sign" | while read i; do - SIZE=`stat -c %s "${i}"` - TARGET_FILE="`echo "${i}" | sed 's/\.sign$//'`" - - echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}" - ${CODESIGN_ALLOCATE} -i "${TARGET_FILE}" -a ${ARCH} ${SIZE} -o "${i}.tmp" - - OFFSET=`${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'` - if [ -z ${QUIET} ]; then - echo "Attaching signature at offset ${OFFSET}" - fi - - dd if="$i" of="${i}.tmp" bs=1 seek=${OFFSET} count=${SIZE} 2>/dev/null - mv "${i}.tmp" "${TARGET_FILE}" - rm "${i}" - echo "Success." -done -mv ${TEMPDIR}/${ROOTDIR} ${OUTDIR} -rm -rf ${TEMPDIR} +${SIGNAPPLE} apply ${UNSIGNED} ${SIGNATURE} +mv ${ROOTDIR} ${OUTDIR} echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index ac9f458fd2..2eae7c3259 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -1,50 +1,28 @@ #!/bin/sh -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e ROOTDIR=dist BUNDLE="${ROOTDIR}/Firo-Qt.app" -CODESIGN=codesign +SIGNAPPLE=signapple TEMPDIR=sign.temp -TEMPLIST=${TEMPDIR}/signatures.txt -OUT=signature.tar.gz -OUTROOT=osx +OUT=signature-osx.tar.gz +OUTROOT=osx/dist -if [ ! -n "$1" ]; then - echo "usage: $0 " - echo "example: $0 -s MyIdentity" +if [ -z "$1" ]; then + echo "usage: $0 " + echo "example: $0 " exit 1 fi -rm -rf ${TEMPDIR} ${TEMPLIST} +rm -rf ${TEMPDIR} mkdir -p ${TEMPDIR} -${CODESIGN} -f --file-list ${TEMPLIST} "$@" "${BUNDLE}" - -grep -v CodeResources < "${TEMPLIST}" | while read i; do - TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`" - SIZE=`pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g'` - OFFSET=`pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g'` - SIGNFILE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}.sign" - DIRNAME="`dirname "${SIGNFILE}"`" - mkdir -p "${DIRNAME}" - echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}" - dd if="$i" of="${SIGNFILE}" bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null -done - -grep CodeResources < "${TEMPLIST}" | while read i; do - TARGETFILE="${BUNDLE}/`echo "${i}" | sed "s|.*${BUNDLE}/||"`" - RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}" - DIRNAME="`dirname "${RESOURCE}"`" - mkdir -p "${DIRNAME}" - echo "Adding resource for: "${TARGETFILE}"" - cp "${i}" "${RESOURCE}" -done - -rm ${TEMPLIST} +${SIGNAPPLE} sign -f --detach "${TEMPDIR}/${OUTROOT}" "$@" "${BUNDLE}" tar -C "${TEMPDIR}" -czf "${OUT}" . rm -rf "${TEMPDIR}" diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh deleted file mode 100755 index 4c175156f4..0000000000 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -set -e - -INPUTFILE="Xcode_7.3.1.dmg" -HFSFILENAME="5.hfs" -SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk" - -7z x "${INPUTFILE}" "${HFSFILENAME}" -SDKNAME="$(basename "${SDKDIR}")" -SDKDIRINODE=$(ifind -n "${SDKDIR}" "${HFSFILENAME}") -fls "${HFSFILENAME}" -rpF ${SDKDIRINODE} | - while read type inode filename; do - inode="${inode::-1}" - if [ "${filename:0:14}" = "usr/share/man/" ]; then - continue - fi - filename="${SDKNAME}/$filename" - echo "Extracting $filename ..." - mkdir -p "$(dirname "$filename")" - if [ "$type" = "l/l" ]; then - ln -s "$(icat "${HFSFILENAME}" $inode)" "$filename" - else - icat "${HFSFILENAME}" $inode >"$filename" - fi -done -echo "Building ${SDKNAME}.tar.gz ..." -MTIME="$(istat "${HFSFILENAME}" "${SDKDIRINODE}" | perl -nle 'm/Content Modified:\s+(.*?)\s\(/ && print $1')" -find "${SDKNAME}" | sort | tar --no-recursion --mtime="${MTIME}" --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > "${SDKNAME}.tar.gz" -echo 'All done!' diff --git a/contrib/macdeploy/fancy.plist b/contrib/macdeploy/fancy.plist deleted file mode 100644 index 1027657e7d..0000000000 --- a/contrib/macdeploy/fancy.plist +++ /dev/null @@ -1,32 +0,0 @@ - - - - - window_bounds - - 300 - 300 - 800 - 620 - - background_picture - background.svg - icon_size - 96 - applications_symlink - - items_position - - Applications - - 370 - 156 - - Firo-Qt.app - - 128 - 156 - - - - diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk new file mode 100755 index 0000000000..457d8f5e64 --- /dev/null +++ b/contrib/macdeploy/gen-sdk @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import argparse +import plistlib +import pathlib +import sys +import tarfile +import gzip +import os +import contextlib + +@contextlib.contextmanager +def cd(path): + """Context manager that restores PWD even if an exception was raised.""" + old_pwd = os.getcwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(old_pwd) + +def run(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1) + parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False) + + args = parser.parse_args() + + xcode_app = pathlib.Path(args.xcode_app[0]).resolve() + assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app) + + xcode_app_plist = xcode_app.joinpath("Contents/version.plist") + with xcode_app_plist.open('rb') as fp: + pl = plistlib.load(fp) + xcode_version = pl['CFBundleShortVersionString'] + xcode_build_id = pl['ProductBuildVersion'] + print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)) + + sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") + sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist") + with sdk_plist.open('rb') as fp: + pl = plistlib.load(fp) + sdk_version = pl['ProductVersion'] + sdk_build_id = pl['ProductBuildVersion'] + print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id)) + + out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) + + xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1") + assert xcode_libcxx_dir.is_dir() + + if args.out_sdktgz: + out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) + else: + # Construct our own out_sdktgz if not specified on the command line + out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) + + def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): + """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files' + names + + e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking: + + tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir") + + would result in the following members being added to tarfp: + + foo/bar/ -> corresponding to /root/bazdir + foo/bar/qux -> corresponding to /root/bazdir/qux + + """ + def change_tarinfo_base(tarinfo): + if tarinfo.name and tarinfo.name.startswith("./"): + tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) + if tarinfo.linkname and tarinfo.linkname.startswith("./"): + tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + return tarinfo + with cd(dir_to_add): + tarfp.add(".", recursive=True, filter=change_tarinfo_base) + + print("Creating output .tar.gz file...") + with out_sdktgz_path.open("wb") as fp: + with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: + with tarfile.open(mode="w", fileobj=gzf) as tarfp: + print("Adding MacOSX SDK {} files...".format(sdk_version)) + tarfp_add_with_base_change(tarfp, sdk_dir, out_name) + print("Adding libc++ headers...") + tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name)) + print("Done! Find the resulting gzipped tarball at:") + print(out_sdktgz_path.resolve()) + +if __name__ == '__main__': + run() diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 4cb37aed80..e9c809d422 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -1,5 +1,4 @@ -#!/usr/bin/env python -from __future__ import division, print_function, unicode_literals +#!/usr/bin/env python3 # # Copyright (C) 2011 Patrick "p2k" Schneider # @@ -17,9 +16,13 @@ from __future__ import division, print_function, unicode_literals # along with this program. If not, see . # -import subprocess, sys, re, os, shutil, stat, os.path, time -from string import Template +import sys, re, os, shutil, stat, os.path from argparse import ArgumentParser +from ds_store import DSStore +from mac_alias import Alias +from pathlib import Path +from subprocess import PIPE, run +from typing import List, Optional # This is ported from the original macdeployqt with modifications @@ -49,28 +52,18 @@ class FrameworkInfo(object): return False def __str__(self): - return """ Framework name: %s - Framework directory: %s - Framework path: %s - Binary name: %s - Binary directory: %s - Binary path: %s - Version: %s - Install name: %s - Deployed install name: %s - Source file Path: %s - Deployed Directory (relative to bundle): %s -""" % (self.frameworkName, - self.frameworkDirectory, - self.frameworkPath, - self.binaryName, - self.binaryDirectory, - self.binaryPath, - self.version, - self.installName, - self.deployedInstallName, - self.sourceFilePath, - self.destinationDirectory) + return f""" Framework name: {self.frameworkName} + Framework directory: {self.frameworkDirectory} + Framework path: {self.frameworkPath} + Binary name: {self.binaryName} + Binary directory: {self.binaryDirectory} + Binary path: {self.binaryPath} + Version: {self.version} + Install name: {self.installName} + Deployed install name: {self.deployedInstallName} + Source file Path: {self.sourceFilePath} + Deployed Directory (relative to bundle): {self.destinationDirectory} +""" def isDylib(self): return self.frameworkName.endswith(".dylib") @@ -86,18 +79,18 @@ class FrameworkInfo(object): bundleBinaryDirectory = "Contents/MacOS" @classmethod - def fromOtoolLibraryLine(cls, line): + def fromOtoolLibraryLine(cls, line: str) -> Optional['FrameworkInfo']: # Note: line must be trimmed if line == "": return None - # Don't deploy system libraries (exception for libQtuitools and libQtlucene). - if line.startswith("/System/Library/") or line.startswith("@executable_path") or (line.startswith("/usr/lib/") and "libQt" not in line): + # Don't deploy system libraries + if line.startswith("/System/Library/") or line.startswith("@executable_path") or line.startswith("/usr/lib/"): return None m = cls.reOLine.match(line) if m is None: - raise RuntimeError("otool line could not be parsed: " + line) + raise RuntimeError(f"otool line could not be parsed: {line}") path = m.group(1) @@ -117,7 +110,7 @@ class FrameworkInfo(object): info.version = "-" info.installName = path - info.deployedInstallName = "@executable_path/../Frameworks/" + info.binaryName + info.deployedInstallName = f"@executable_path/../Frameworks/{info.binaryName}" info.sourceFilePath = path info.destinationDirectory = cls.bundleFrameworkDirectory else: @@ -129,7 +122,7 @@ class FrameworkInfo(object): break i += 1 if i == len(parts): - raise RuntimeError("Could not find .framework or .dylib in otool line: " + line) + raise RuntimeError(f"Could not find .framework or .dylib in otool line: {line}") info.frameworkName = parts[i] info.frameworkDirectory = "/".join(parts[:i]) @@ -140,25 +133,24 @@ class FrameworkInfo(object): info.binaryPath = os.path.join(info.binaryDirectory, info.binaryName) info.version = parts[i+2] - info.deployedInstallName = "@executable_path/../Frameworks/" + os.path.join(info.frameworkName, info.binaryPath) + info.deployedInstallName = f"@executable_path/../Frameworks/{os.path.join(info.frameworkName, info.binaryPath)}" info.destinationDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, info.binaryDirectory) info.sourceResourcesDirectory = os.path.join(info.frameworkPath, "Resources") info.sourceContentsDirectory = os.path.join(info.frameworkPath, "Contents") info.sourceVersionContentsDirectory = os.path.join(info.frameworkPath, "Versions", info.version, "Contents") info.destinationResourcesDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, "Resources") - info.destinationContentsDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, "Contents") info.destinationVersionContentsDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, "Versions", info.version, "Contents") return info class ApplicationBundleInfo(object): - def __init__(self, path): + def __init__(self, path: str): self.path = path - appName = "Firo-Qt" - self.binaryPath = os.path.join(path, "Contents", "MacOS", appName) + # for backwards compatibility reasons, this must remain as Firo-Qt + self.binaryPath = os.path.join(path, "Contents", "MacOS", "Firo-Qt") if not os.path.exists(self.binaryPath): - raise RuntimeError("Could not find bundle binary for " + path) + raise RuntimeError(f"Could not find bundle binary for {path}") self.resourcesPath = os.path.join(path, "Contents", "Resources") self.pluginPath = os.path.join(path, "Contents", "PlugIns") @@ -168,17 +160,11 @@ class DeploymentInfo(object): self.pluginPath = None self.deployedFrameworks = [] - def detectQtPath(self, frameworkDirectory): + def detectQtPath(self, frameworkDirectory: str): parentDir = os.path.dirname(frameworkDirectory) if os.path.exists(os.path.join(parentDir, "translations")): # Classic layout, e.g. "/usr/local/Trolltech/Qt-4.x.x" self.qtPath = parentDir - elif os.path.exists(os.path.join(parentDir, "share", "qt4", "translations")): - # MacPorts layout, e.g. "/opt/local/share/qt4" - self.qtPath = os.path.join(parentDir, "share", "qt4") - elif os.path.exists(os.path.join(os.path.dirname(parentDir), "share", "qt4", "translations")): - # Newer Macports layout - self.qtPath = os.path.join(os.path.dirname(parentDir), "share", "qt4") else: self.qtPath = os.getenv("QTDIR", None) @@ -187,31 +173,27 @@ class DeploymentInfo(object): if os.path.exists(pluginPath): self.pluginPath = pluginPath - def usesFramework(self, name): - nameDot = "%s." % name - libNameDot = "lib%s." % name + def usesFramework(self, name: str) -> bool: for framework in self.deployedFrameworks: if framework.endswith(".framework"): - if framework.startswith(nameDot): + if framework.startswith(f"{name}."): return True elif framework.endswith(".dylib"): - if framework.startswith(libNameDot): + if framework.startswith(f"lib{name}."): return True return False -def getFrameworks(binaryPath, verbose): - if verbose >= 3: - print("Inspecting with otool: " + binaryPath) +def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: + if verbose: + print(f"Inspecting with otool: {binaryPath}") otoolbin=os.getenv("OTOOL", "otool") - otool = subprocess.Popen([otoolbin, "-L", binaryPath], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - o_stdout, o_stderr = otool.communicate() + otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, universal_newlines=True) if otool.returncode != 0: - if verbose >= 1: - sys.stderr.write(o_stderr) - sys.stderr.flush() - raise RuntimeError("otool failed with return code %d" % otool.returncode) + sys.stderr.write(otool.stderr) + sys.stderr.flush() + raise RuntimeError(f"otool failed with return code {otool.returncode}") - otoolLines = o_stdout.decode().split("\n") + otoolLines = otool.stdout.split("\n") otoolLines.pop(0) # First line is the inspected binary if ".framework" in binaryPath or binaryPath.endswith(".dylib"): otoolLines.pop(0) # Frameworks and dylibs list themselves as a dependency. @@ -221,50 +203,50 @@ def getFrameworks(binaryPath, verbose): line = line.replace("@loader_path", os.path.dirname(binaryPath)) info = FrameworkInfo.fromOtoolLibraryLine(line.strip()) if info is not None: - if verbose >= 3: + if verbose: print("Found framework:") print(info) libraries.append(info) return libraries -def runInstallNameTool(action, *args): +def runInstallNameTool(action: str, *args): installnametoolbin=os.getenv("INSTALLNAMETOOL", "install_name_tool") - subprocess.check_call([installnametoolbin, "-"+action] + list(args)) + run([installnametoolbin, "-"+action] + list(args), check=True) -def changeInstallName(oldName, newName, binaryPath, verbose): - if verbose >= 3: +def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): + if verbose: print("Using install_name_tool:") print(" in", binaryPath) print(" change reference", oldName) print(" to", newName) runInstallNameTool("change", oldName, newName, binaryPath) -def changeIdentification(id, binaryPath, verbose): - if verbose >= 3: +def changeIdentification(id: str, binaryPath: str, verbose: int): + if verbose: print("Using install_name_tool:") print(" change identification in", binaryPath) print(" to", id) runInstallNameTool("id", id, binaryPath) -def runStrip(binaryPath, verbose): +def runStrip(binaryPath: str, verbose: int): stripbin=os.getenv("STRIP", "strip") - if verbose >= 3: + if verbose: print("Using strip:") print(" stripped", binaryPath) - subprocess.check_call([stripbin, "-x", binaryPath]) + run([stripbin, "-x", binaryPath], check=True) -def copyFramework(framework, path, verbose): +def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional[str]: if framework.sourceFilePath.startswith("Qt"): #standard place for Nokia Qt installer's frameworks - fromPath = "/Library/Frameworks/" + framework.sourceFilePath + fromPath = f"/Library/Frameworks/{framework.sourceFilePath}" else: fromPath = framework.sourceFilePath toDir = os.path.join(path, framework.destinationDirectory) toPath = os.path.join(toDir, framework.binaryName) if not os.path.exists(fromPath): - raise RuntimeError("No file at " + fromPath) + raise RuntimeError(f"No file at {fromPath}") if os.path.exists(toPath): return None # Already there @@ -273,7 +255,7 @@ def copyFramework(framework, path, verbose): os.makedirs(toDir) shutil.copy2(fromPath, toPath) - if verbose >= 3: + if verbose: print("Copied:", fromPath) print(" to:", toPath) @@ -287,13 +269,12 @@ def copyFramework(framework, path, verbose): linkto = framework.version if not os.path.exists(linkfrom): os.symlink(linkto, linkfrom) - if verbose >= 2: - print("Linked:", linkfrom, "->", linkto) + print("Linked:", linkfrom, "->", linkto) fromResourcesDir = framework.sourceResourcesDirectory if os.path.exists(fromResourcesDir): toResourcesDir = os.path.join(path, framework.destinationResourcesDirectory) shutil.copytree(fromResourcesDir, toResourcesDir, symlinks=True) - if verbose >= 3: + if verbose: print("Copied resources:", fromResourcesDir) print(" to:", toResourcesDir) fromContentsDir = framework.sourceVersionContentsDirectory @@ -302,22 +283,13 @@ def copyFramework(framework, path, verbose): if os.path.exists(fromContentsDir): toContentsDir = os.path.join(path, framework.destinationVersionContentsDirectory) shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) - contentslinkfrom = os.path.join(path, framework.destinationContentsDirectory) - if verbose >= 3: + if verbose: print("Copied Contents:", fromContentsDir) print(" to:", toContentsDir) - elif framework.frameworkName.startswith("libQtGui"): # Copy qt_menu.nib (applies to non-framework layout) - qtMenuNibSourcePath = os.path.join(framework.frameworkDirectory, "Resources", "qt_menu.nib") - qtMenuNibDestinationPath = os.path.join(path, "Contents", "Resources", "qt_menu.nib") - if os.path.exists(qtMenuNibSourcePath) and not os.path.exists(qtMenuNibDestinationPath): - shutil.copytree(qtMenuNibSourcePath, qtMenuNibDestinationPath, symlinks=True) - if verbose >= 3: - print("Copied for libQtGui:", qtMenuNibSourcePath) - print(" to:", qtMenuNibDestinationPath) return toPath -def deployFrameworks(frameworks, bundlePath, binaryPath, strip, verbose, deploymentInfo=None): +def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPath: str, strip: bool, verbose: int, deploymentInfo: Optional[DeploymentInfo] = None) -> DeploymentInfo: if deploymentInfo is None: deploymentInfo = DeploymentInfo() @@ -325,16 +297,14 @@ def deployFrameworks(frameworks, bundlePath, binaryPath, strip, verbose, deploym framework = frameworks.pop(0) deploymentInfo.deployedFrameworks.append(framework.frameworkName) - if verbose >= 2: - print("Processing", framework.frameworkName, "...") + print("Processing", framework.frameworkName, "...") # Get the Qt path from one of the Qt frameworks if deploymentInfo.qtPath is None and framework.isQtFramework(): deploymentInfo.detectQtPath(framework.frameworkDirectory) if framework.installName.startswith("@executable_path") or framework.installName.startswith(bundlePath): - if verbose >= 2: - print(framework.frameworkName, "already deployed, skipping.") + print(framework.frameworkName, "already deployed, skipping.") continue # install_name_tool the new id into the binary @@ -363,88 +333,34 @@ def deployFrameworks(frameworks, bundlePath, binaryPath, strip, verbose, deploym return deploymentInfo -def deployFrameworksForAppBundle(applicationBundle, strip, verbose): +def deployFrameworksForAppBundle(applicationBundle: ApplicationBundleInfo, strip: bool, verbose: int) -> DeploymentInfo: frameworks = getFrameworks(applicationBundle.binaryPath, verbose) - if len(frameworks) == 0 and verbose >= 1: - print("Warning: Could not find any external frameworks to deploy in %s." % (applicationBundle.path)) + if len(frameworks) == 0: + print(f"Warning: Could not find any external frameworks to deploy in {applicationBundle.path}.") return DeploymentInfo() else: return deployFrameworks(frameworks, applicationBundle.path, applicationBundle.binaryPath, strip, verbose) -def deployPlugins(appBundleInfo, deploymentInfo, strip, verbose): - # Lookup available plugins, exclude unneeded +def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: DeploymentInfo, strip: bool, verbose: int): plugins = [] if deploymentInfo.pluginPath is None: return for dirpath, dirnames, filenames in os.walk(deploymentInfo.pluginPath): pluginDirectory = os.path.relpath(dirpath, deploymentInfo.pluginPath) - if pluginDirectory == "designer": - # Skip designer plugins + + if pluginDirectory not in ['styles', 'platforms']: continue - elif pluginDirectory == "phonon" or pluginDirectory == "phonon_backend": - # Deploy the phonon plugins only if phonon is in use - if not deploymentInfo.usesFramework("phonon"): - continue - elif pluginDirectory == "sqldrivers": - # Deploy the sql plugins only if QtSql is in use - if not deploymentInfo.usesFramework("QtSql"): - continue - elif pluginDirectory == "script": - # Deploy the script plugins only if QtScript is in use - if not deploymentInfo.usesFramework("QtScript"): - continue - elif pluginDirectory == "qmltooling" or pluginDirectory == "qml1tooling": - # Deploy the qml plugins only if QtDeclarative is in use - if not deploymentInfo.usesFramework("QtDeclarative"): - continue - elif pluginDirectory == "bearer": - # Deploy the bearer plugins only if QtNetwork is in use - if not deploymentInfo.usesFramework("QtNetwork"): - continue - elif pluginDirectory == "position": - # Deploy the position plugins only if QtPositioning is in use - if not deploymentInfo.usesFramework("QtPositioning"): - continue - elif pluginDirectory == "sensors" or pluginDirectory == "sensorgestures": - # Deploy the sensor plugins only if QtSensors is in use - if not deploymentInfo.usesFramework("QtSensors"): - continue - elif pluginDirectory == "audio" or pluginDirectory == "playlistformats": - # Deploy the audio plugins only if QtMultimedia is in use - if not deploymentInfo.usesFramework("QtMultimedia"): - continue - elif pluginDirectory == "mediaservice": - # Deploy the mediaservice plugins only if QtMultimediaWidgets is in use - if not deploymentInfo.usesFramework("QtMultimediaWidgets"): - continue for pluginName in filenames: pluginPath = os.path.join(pluginDirectory, pluginName) - if pluginName.endswith("_debug.dylib"): - # Skip debug plugins + + if pluginName.split('.')[0] not in ['libqminimal', 'libqcocoa', 'libqmacstyle']: continue - elif pluginPath == "imageformats/libqsvg.dylib" or pluginPath == "iconengines/libqsvgicon.dylib": - # Deploy the svg plugins only if QtSvg is in use - if not deploymentInfo.usesFramework("QtSvg"): - continue - elif pluginPath == "accessible/libqtaccessiblecompatwidgets.dylib": - # Deploy accessibility for Qt3Support only if the Qt3Support is in use - if not deploymentInfo.usesFramework("Qt3Support"): - continue - elif pluginPath == "graphicssystems/libqglgraphicssystem.dylib": - # Deploy the opengl graphicssystem plugin only if QtOpenGL is in use - if not deploymentInfo.usesFramework("QtOpenGL"): - continue - elif pluginPath == "accessible/libqtaccessiblequick.dylib": - # Deploy the accessible qtquick plugin only if QtQuick is in use - if not deploymentInfo.usesFramework("QtQuick"): - continue plugins.append((pluginDirectory, pluginName)) for pluginDirectory, pluginName in plugins: - if verbose >= 2: - print("Processing plugin", os.path.join(pluginDirectory, pluginName), "...") + print("Processing plugin", os.path.join(pluginDirectory, pluginName), "...") sourcePath = os.path.join(deploymentInfo.pluginPath, pluginDirectory, pluginName) destinationDirectory = os.path.join(appBundleInfo.pluginPath, pluginDirectory) @@ -453,7 +369,7 @@ def deployPlugins(appBundleInfo, deploymentInfo, strip, verbose): destinationPath = os.path.join(destinationDirectory, pluginName) shutil.copy2(sourcePath, destinationPath) - if verbose >= 3: + if verbose: print("Copied:", sourcePath) print(" to:", destinationPath) @@ -469,146 +385,53 @@ def deployPlugins(appBundleInfo, deploymentInfo, strip, verbose): if dependency.frameworkName not in deploymentInfo.deployedFrameworks: deployFrameworks([dependency], appBundleInfo.path, destinationPath, strip, verbose, deploymentInfo) -qt_conf="""[Paths] -Translations=Resources -Plugins=PlugIns -""" - ap = ArgumentParser(description="""Improved version of macdeployqt. Outputs a ready-to-deploy app in a folder "dist" and optionally wraps it in a .dmg file. Note, that the "dist" folder will be deleted before deploying on each run. -Optionally, Qt translation files (.qm) and additional resources can be added to the bundle. - -Also optionally signs the .app bundle; set the CODESIGNARGS environment variable to pass arguments -to the codesign tool. -E.g. CODESIGNARGS='--sign "Developer ID Application: ..." --keychain /encrypted/foo.keychain'""") +Optionally, Qt translation files (.qm) can be added to the bundle.""") ap.add_argument("app_bundle", nargs=1, metavar="app-bundle", help="application bundle to be deployed") -ap.add_argument("-verbose", type=int, nargs=1, default=[1], metavar="<0-3>", help="0 = no output, 1 = error/warning (default), 2 = normal, 3 = debug") +ap.add_argument("appname", nargs=1, metavar="appname", help="name of the app being deployed") +ap.add_argument("-verbose", nargs="?", const=True, help="Output additional debugging information") ap.add_argument("-no-plugins", dest="plugins", action="store_false", default=True, help="skip plugin deployment") ap.add_argument("-no-strip", dest="strip", action="store_false", default=True, help="don't run 'strip' on the binaries") -ap.add_argument("-sign", dest="sign", action="store_true", default=False, help="sign .app bundle with codesign tool") -ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image; if basename is not specified, a camel-cased version of the app name is used") -ap.add_argument("-fancy", nargs=1, metavar="plist", default=[], help="make a fancy looking disk image using the given plist file with instructions; requires -dmg to work") -ap.add_argument("-add-qt-tr", nargs=1, metavar="languages", default=[], help="add Qt translation files to the bundle's resources; the language list must be separated with commas, not with whitespace") -ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translation files") -ap.add_argument("-add-resources", nargs="+", metavar="path", default=[], help="list of additional files or folders to be copied into the bundle's resources; must be the last argument") -ap.add_argument("-volname", nargs=1, metavar="volname", default=[], help="custom volume name for dmg") +ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image") +ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translations. Base translations will automatically be added to the bundle's resources.") config = ap.parse_args() -verbose = config.verbose[0] +verbose = config.verbose # ------------------------------------------------ app_bundle = config.app_bundle[0] +appname = config.appname[0] if not os.path.exists(app_bundle): - if verbose >= 1: - sys.stderr.write("Error: Could not find app bundle \"%s\"\n" % (app_bundle)) + sys.stderr.write(f"Error: Could not find app bundle \"{app_bundle}\"\n") sys.exit(1) -app_bundle_name = os.path.splitext(os.path.basename(app_bundle))[0] - -# ------------------------------------------------ -translations_dir = None -if config.translations_dir and config.translations_dir[0]: - if os.path.exists(config.translations_dir[0]): - translations_dir = config.translations_dir[0] - else: - if verbose >= 1: - sys.stderr.write("Error: Could not find translation dir \"%s\"\n" % (translations_dir)) - sys.exit(1) -# ------------------------------------------------ - -for p in config.add_resources: - if verbose >= 3: - print("Checking for \"%s\"..." % p) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find additional resource file \"%s\"\n" % (p)) - sys.exit(1) - -# ------------------------------------------------ - -if len(config.fancy) == 1: - if verbose >= 3: - print("Fancy: Importing plistlib...") - try: - import plistlib - except ImportError: - if verbose >= 1: - sys.stderr.write("Error: Could not import plistlib which is required for fancy disk images.\n") - sys.exit(1) - - p = config.fancy[0] - if verbose >= 3: - print("Fancy: Loading \"%s\"..." % p) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find fancy disk image plist at \"%s\"\n" % (p)) - sys.exit(1) - - try: - fancy = plistlib.readPlist(p) - except: - if verbose >= 1: - sys.stderr.write("Error: Could not parse fancy disk image plist at \"%s\"\n" % (p)) - sys.exit(1) - - try: - assert "window_bounds" not in fancy or (isinstance(fancy["window_bounds"], list) and len(fancy["window_bounds"]) == 4) - assert "background_picture" not in fancy or isinstance(fancy["background_picture"], str) - assert "icon_size" not in fancy or isinstance(fancy["icon_size"], int) - assert "applications_symlink" not in fancy or isinstance(fancy["applications_symlink"], bool) - if "items_position" in fancy: - assert isinstance(fancy["items_position"], dict) - for key, value in fancy["items_position"].items(): - assert isinstance(value, list) and len(value) == 2 and isinstance(value[0], int) and isinstance(value[1], int) - except: - if verbose >= 1: - sys.stderr.write("Error: Bad format of fancy disk image plist at \"%s\"\n" % (p)) - sys.exit(1) - - if "background_picture" in fancy: - bp = fancy["background_picture"] - if verbose >= 3: - print("Fancy: Resolving background picture \"%s\"..." % bp) - if not os.path.exists(bp): - bp = os.path.join(os.path.dirname(p), bp) - if not os.path.exists(bp): - if verbose >= 1: - sys.stderr.write("Error: Could not find background picture at \"%s\" or \"%s\"\n" % (fancy["background_picture"], bp)) - sys.exit(1) - else: - fancy["background_picture"] = bp -else: - fancy = None - # ------------------------------------------------ if os.path.exists("dist"): - if verbose >= 2: - print("+ Removing old dist folder +") - + print("+ Removing existing dist folder +") shutil.rmtree("dist") -# ------------------------------------------------ +if os.path.exists(appname + ".dmg"): + print("+ Removing existing DMG +") + os.unlink(appname + ".dmg") -if len(config.volname) == 1: - volname = config.volname[0] -else: - volname = app_bundle_name +if os.path.exists(appname + ".temp.dmg"): + os.unlink(appname + ".temp.dmg") # ------------------------------------------------ target = os.path.join("dist", "Firo-Qt.app") -if verbose >= 2: - print("+ Copying source bundle +") -if verbose >= 3: +print("+ Copying source bundle +") +if verbose: print(app_bundle, "->", target) os.mkdir("dist") @@ -618,276 +441,155 @@ applicationBundle = ApplicationBundleInfo(target) # ------------------------------------------------ -if verbose >= 2: - print("+ Deploying frameworks +") +print("+ Deploying frameworks +") try: deploymentInfo = deployFrameworksForAppBundle(applicationBundle, config.strip, verbose) if deploymentInfo.qtPath is None: deploymentInfo.qtPath = os.getenv("QTDIR", None) if deploymentInfo.qtPath is None: - if verbose >= 1: - sys.stderr.write("Warning: Could not detect Qt's path, skipping plugin deployment!\n") + sys.stderr.write("Warning: Could not detect Qt's path, skipping plugin deployment!\n") config.plugins = False except RuntimeError as e: - if verbose >= 1: - sys.stderr.write("Error: %s\n" % str(e)) + sys.stderr.write(f"Error: {str(e)}\n") sys.exit(1) # ------------------------------------------------ if config.plugins: - if verbose >= 2: - print("+ Deploying plugins +") + print("+ Deploying plugins +") try: deployPlugins(applicationBundle, deploymentInfo, config.strip, verbose) except RuntimeError as e: - if verbose >= 1: - sys.stderr.write("Error: %s\n" % str(e)) + sys.stderr.write(f"Error: {str(e)}\n") sys.exit(1) # ------------------------------------------------ -if len(config.add_qt_tr) == 0: - add_qt_tr = [] -else: - if translations_dir is not None: - qt_tr_dir = translations_dir - else: - if deploymentInfo.qtPath is not None: - qt_tr_dir = os.path.join(deploymentInfo.qtPath, "translations") - else: - sys.stderr.write("Error: Could not find Qt translation path\n") - sys.exit(1) - add_qt_tr = ["qt_%s.qm" % lng for lng in config.add_qt_tr[0].split(",")] - for lng_file in add_qt_tr: - p = os.path.join(qt_tr_dir, lng_file) - if verbose >= 3: - print("Checking for \"%s\"..." % p) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find Qt translation file \"%s\"\n" % (lng_file)) - sys.exit(1) - -# ------------------------------------------------ +if config.translations_dir: + if not Path(config.translations_dir[0]).exists(): + sys.stderr.write(f"Error: Could not find translation dir \"{config.translations_dir[0]}\"\n") + sys.exit(1) -if verbose >= 2: - print("+ Installing qt.conf +") +print("+ Adding Qt translations +") -f = open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") -f.write(qt_conf.encode()) -f.close() +translations = Path(config.translations_dir[0]) -# ------------------------------------------------ +regex = re.compile('qt_[a-z]*(.qm|_[A-Z]*.qm)') -if len(add_qt_tr) > 0 and verbose >= 2: - print("+ Adding Qt translations +") +lang_files = [x for x in translations.iterdir() if regex.match(x.name)] -for lng_file in add_qt_tr: - if verbose >= 3: - print(os.path.join(qt_tr_dir, lng_file), "->", os.path.join(applicationBundle.resourcesPath, lng_file)) - shutil.copy2(os.path.join(qt_tr_dir, lng_file), os.path.join(applicationBundle.resourcesPath, lng_file)) +for file in lang_files: + if verbose: + print(file.as_posix(), "->", os.path.join(applicationBundle.resourcesPath, file.name)) + shutil.copy2(file.as_posix(), os.path.join(applicationBundle.resourcesPath, file.name)) # ------------------------------------------------ -if len(config.add_resources) > 0 and verbose >= 2: - print("+ Adding additional resources +") +print("+ Installing qt.conf +") -for p in config.add_resources: - t = os.path.join(applicationBundle.resourcesPath, os.path.basename(p)) - if verbose >= 3: - print(p, "->", t) - if os.path.isdir(p): - shutil.copytree(p, t, symlinks=True) - else: - shutil.copy2(p, t) +qt_conf="""[Paths] +Translations=Resources +Plugins=PlugIns +""" + +with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: + f.write(qt_conf.encode()) # ------------------------------------------------ -if config.sign and 'CODESIGNARGS' not in os.environ: - print("You must set the CODESIGNARGS environment variable. Skipping signing.") -elif config.sign: - if verbose >= 1: - print("Code-signing app bundle %s"%(target,)) - subprocess.check_call("codesign --force %s %s"%(os.environ['CODESIGNARGS'], target), shell=True) +print("+ Generating .DS_Store +") + +output_file = os.path.join("dist", ".DS_Store") + +ds = DSStore.open(output_file, 'w+') + +ds['.']['bwsp'] = { + 'WindowBounds': '{{300, 280}, {500, 343}}', + 'PreviewPaneVisibility': False, +} + + +icvp = { + 'gridOffsetX': 0.0, + 'textSize': 12.0, + 'viewOptionsVersion': 1, + 'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00', + 'backgroundColorBlue': 1.0, + 'iconSize': 96.0, + 'backgroundColorGreen': 1.0, + 'arrangeBy': 'none', + 'showIconPreview': True, + 'gridSpacing': 100.0, + 'gridOffsetY': 0.0, + 'showItemInfo': False, + 'labelOnBottom': True, + 'backgroundType': 2, + 'backgroundColorRed': 1.0 +} +alias = Alias().from_bytes(icvp['backgroundImageAlias']) +alias.volume.name = appname +alias.volume.posix_path = '/Volumes/' + appname +icvp['backgroundImageAlias'] = alias.to_bytes() +ds['.']['icvp'] = icvp + +ds['.']['vSrn'] = ('long', 1) + +ds['Applications']['Iloc'] = (370, 156) +ds['Firo-Qt.app']['Iloc'] = (128, 156) + +ds.flush() +ds.close() # ------------------------------------------------ if config.dmg is not None: - #Patch in check_output for Python 2.6 - if "check_output" not in dir( subprocess ): - def f(*popenargs, **kwargs): - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return output - subprocess.check_output = f - - def runHDIUtil(verb, image_basename, **kwargs): - hdiutil_args = ["hdiutil", verb, image_basename + ".dmg"] - if "capture_stdout" in kwargs: - del kwargs["capture_stdout"] - run = subprocess.check_output - else: - if verbose < 2: - hdiutil_args.append("-quiet") - elif verbose >= 3: - hdiutil_args.append("-verbose") - run = subprocess.check_call - - for key, value in kwargs.items(): - hdiutil_args.append("-" + key) - if not value is True: - hdiutil_args.append(str(value)) - - return run(hdiutil_args) - - if verbose >= 2: - if fancy is None: - print("+ Creating .dmg disk image +") - else: - print("+ Preparing .dmg disk image +") - - if config.dmg != "": - dmg_name = config.dmg - else: - spl = app_bundle_name.split(" ") - dmg_name = spl[0] + "".join(p.capitalize() for p in spl[1:]) - - if fancy is None: - try: - runHDIUtil("create", dmg_name, srcfolder="dist", format="UDBZ", volname=volname, ov=True, fs="HFS+") - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - else: - if verbose >= 3: - print("Determining size of \"dist\"...") - size = 0 - for path, dirs, files in os.walk("dist"): - for file in files: - size += os.path.getsize(os.path.join(path, file)) - size += int(size * 0.15) - - if verbose >= 3: - print("Creating temp image for modification...") - try: - runHDIUtil("create", dmg_name + ".temp", srcfolder="dist", format="UDRW", size=size, volname=volname, ov=True, fs="HFS+") - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - if verbose >= 3: - print("Attaching temp image...") - try: - output = runHDIUtil("attach", dmg_name + ".temp", readwrite=True, noverify=True, noautoopen=True, capture_stdout=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - m = re.search("/Volumes/(.+$)", output.decode()) - disk_root = m.group(0) - disk_name = m.group(1) - - if verbose >= 2: - print("+ Applying fancy settings +") - - if "background_picture" in fancy: - print ("--test", disk_root) - print ("--test", os.path.basename(fancy["background_picture"])) - bg_path = os.path.join(disk_root, ".background", os.path.basename(fancy["background_picture"])) - os.mkdir(os.path.dirname(bg_path)) - if verbose >= 3: - print(fancy["background_picture"], "->", bg_path) - shutil.copy2(fancy["background_picture"], bg_path) - else: - bg_path = None - - if fancy.get("applications_symlink", False): - os.symlink("/Applications", os.path.join(disk_root, "Applications")) - - # The Python appscript package broke with OSX 10.8 and isn't being fixed. - # So we now build up an AppleScript string and use the osascript command - # to make the .dmg file pretty: - appscript = Template( """ - on run argv - tell application "Finder" - tell disk "$disk" - open - set current view of container window to icon view - set toolbar visible of container window to false - set statusbar visible of container window to false - set the bounds of container window to {$window_bounds} - set theViewOptions to the icon view options of container window - set arrangement of theViewOptions to not arranged - set icon size of theViewOptions to $icon_size - $background_commands - $items_positions - close -- close/reopen works around a bug... - open - update without registering applications - delay 5 - eject - end tell - end tell - end run - """) - - itemscript = Template('set position of item "${item}" of container window to {${position}}') - items_positions = [] - if "items_position" in fancy: - for name, position in fancy["items_position"].items(): - params = { "item" : name, "position" : ",".join([str(p) for p in position]) } - items_positions.append(itemscript.substitute(params)) - - params = { - "disk" : volname, - "window_bounds" : "300,300,800,620", - "icon_size" : "96", - "background_commands" : "", - "items_positions" : "\n ".join(items_positions) - } - if "window_bounds" in fancy: - params["window_bounds"] = ",".join([str(p) for p in fancy["window_bounds"]]) - if "icon_size" in fancy: - params["icon_size"] = str(fancy["icon_size"]) - if bg_path is not None: - # Set background file, then call SetFile to make it invisible. - # (note: making it invisible first makes set background picture fail) - bgscript = Template("""set background picture of theViewOptions to file ".background:$bgpic" - do shell script "SetFile -a V /Volumes/$disk/.background/$bgpic" """) - params["background_commands"] = bgscript.substitute({"bgpic" : os.path.basename(bg_path), "disk" : params["disk"]}) - - s = appscript.substitute(params) - if verbose >= 2: - print("Running AppleScript:") - print(s) - - p = subprocess.Popen(['osascript', '-'], stdin=subprocess.PIPE) - p.communicate(input=s.encode('utf-8')) - if p.returncode: - print("Error running osascript.") - - if verbose >= 2: - print("+ Finalizing .dmg disk image +") - time.sleep(5) - - try: - runHDIUtil("convert", dmg_name + ".temp", format="UDBZ", o=dmg_name + ".dmg", ov=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - os.unlink(dmg_name + ".temp.dmg") + print("+ Preparing .dmg disk image +") + + if verbose: + print("Determining size of \"dist\"...") + size = 0 + for path, dirs, files in os.walk("dist"): + for file in files: + size += os.path.getsize(os.path.join(path, file)) + size += int(size * 0.15) + + if verbose: + print("Creating temp image for modification...") + + tempname: str = appname + ".temp.dmg" + + run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, universal_newlines=True) + + if verbose: + print("Attaching temp image...") + output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout + + m = re.search(r"/Volumes/(.+$)", output) + disk_root = m.group(0) + + print("+ Applying fancy settings +") + + bg_path = os.path.join(disk_root, ".background", os.path.basename('background.tiff')) + os.mkdir(os.path.dirname(bg_path)) + if verbose: + print('background.tiff', "->", bg_path) + shutil.copy2('background.tiff', bg_path) + + os.symlink("/Applications", os.path.join(disk_root, "Applications")) + + print("+ Finalizing .dmg disk image +") + + run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) + + run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, universal_newlines=True) + + os.unlink(tempname) # ------------------------------------------------ -if verbose >= 2: - print("+ Done +") +print("+ Done +") sys.exit(0) diff --git a/contrib/shell/git-utils.bash b/contrib/shell/git-utils.bash new file mode 100644 index 0000000000..37bac1f38d --- /dev/null +++ b/contrib/shell/git-utils.bash @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +git_root() { + git rev-parse --show-toplevel 2> /dev/null +} + +git_head_version() { + local recent_tag + if recent_tag="$(git describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git rev-parse --short=12 HEAD + fi +} diff --git a/contrib/shell/realpath.bash b/contrib/shell/realpath.bash new file mode 100644 index 0000000000..389b77b562 --- /dev/null +++ b/contrib/shell/realpath.bash @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Based on realpath.sh written by Michael Kropat +# Found at: https://github.com/mkropat/sh-realpath/blob/65512368b8155b176b67122aa395ac580d9acc5b/realpath.sh + +bash_realpath() { + canonicalize_path "$(resolve_symlinks "$1")" +} + +resolve_symlinks() { + _resolve_symlinks "$1" +} + +_resolve_symlinks() { + _assert_no_path_cycles "$@" || return + + local dir_context path + if path=$(readlink -- "$1"); then + dir_context=$(dirname -- "$1") + _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" + else + printf '%s\n' "$1" + fi +} + +_prepend_dir_context_if_necessary() { + if [ "$1" = . ]; then + printf '%s\n' "$2" + else + _prepend_path_if_relative "$1" "$2" + fi +} + +_prepend_path_if_relative() { + case "$2" in + /* ) printf '%s\n' "$2" ;; + * ) printf '%s\n' "$1/$2" ;; + esac +} + +_assert_no_path_cycles() { + local target path + + target=$1 + shift + + for path in "$@"; do + if [ "$path" = "$target" ]; then + return 1 + fi + done +} + +canonicalize_path() { + if [ -d "$1" ]; then + _canonicalize_dir_path "$1" + else + _canonicalize_file_path "$1" + fi +} + +_canonicalize_dir_path() { + (cd "$1" 2>/dev/null && pwd -P) +} + +_canonicalize_file_path() { + local dir file + dir=$(dirname -- "$1") + file=$(basename -- "$1") + (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") +} diff --git a/depends/Makefile b/depends/Makefile index d885113c02..1c8f815a7b 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -1,11 +1,46 @@ .NOTPARALLEL : +# Pattern rule to print variables, e.g. make print-top_srcdir +print-%: FORCE + @echo '$*'='$($*)' + +# When invoking a sub-make, keep only the command line variable definitions +# matching the pattern in the filter function. +# +# e.g. invoking: +# $ make A=1 C=1 print-MAKEOVERRIDES print-MAKEFLAGS +# +# with the following in the Makefile: +# MAKEOVERRIDES := $(filter A=% B=%,$(MAKEOVERRIDES)) +# +# will print: +# MAKEOVERRIDES = A=1 +# MAKEFLAGS = -- A=1 +# +# this is because as the GNU make manual says: +# The command line variable definitions really appear in the variable +# MAKEOVERRIDES, and MAKEFLAGS contains a reference to this variable. +# +# and since the GNU make manual also says: +# variables defined on the command line are passed to the sub-make through +# MAKEFLAGS +# +# this means that sub-makes will be invoked as if: +# $(MAKE) A=1 blah blah +MAKEOVERRIDES := $(filter V=%,$(MAKEOVERRIDES)) SOURCES_PATH ?= $(BASEDIR)/sources +WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs NO_QT ?= +NO_QR ?= +NO_BDB ?= +NO_SQLITE ?= NO_WALLET ?= +NO_ZMQ ?= NO_UPNP ?= +NO_NATPMP ?= +MULTIPROCESS ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources BUILD = $(shell ./config.guess) @@ -21,7 +56,6 @@ BUILD_ID_SALT ?= salt host:=$(BUILD) ifneq ($(HOST),) host:=$(HOST) -host_toolchain:=$(HOST)- endif ifneq ($(DEBUG),) @@ -30,9 +64,9 @@ else release_type=release endif -base_build_dir=$(BASEDIR)/work/build -base_staging_dir=$(BASEDIR)/work/staging -base_download_dir=$(BASEDIR)/work/download +base_build_dir=$(WORK_PATH)/build +base_staging_dir=$(WORK_PATH)/staging +base_download_dir=$(WORK_PATH)/download canonical_host:=$(shell ./config.sub $(HOST)) build:=$(shell ./config.sub $(BUILD)) @@ -52,6 +86,11 @@ full_host_os:=$(subst $(host_arch)-$(host_vendor)-,,$(canonical_host)) host_os:=$(findstring linux,$(full_host_os)) host_os+=$(findstring darwin,$(full_host_os)) host_os+=$(findstring mingw32,$(full_host_os)) + +ifeq (android,$(findstring android,$(full_host_os))) +host_os:=android +endif + host_os:=$(strip $(host_os)) ifeq ($(host_os),) host_os=$(full_host_os) @@ -75,40 +114,71 @@ include builders/$(build_os).mk include builders/default.mk include packages/packages.mk -build_id_string:=$(BUILD_ID_SALT) -build_id_string+=$(shell $(build_CC) --version 2>/dev/null) -build_id_string+=$(shell $(build_AR) --version 2>/dev/null) -build_id_string+=$(shell $(build_CXX) --version 2>/dev/null) -build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null) -build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null) - -$(host_arch)_$(host_os)_id_string:=$(HOST_ID_SALT) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) --version 2>/dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) - -qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) -wallet_packages_$(NO_WALLET) = $(wallet_packages) +# Previously, we directly invoked the well-known programs using $(shell ...) +# to construct build_id_string. However, that was problematic because: +# +# 1. When invoking a shell, GNU Make special-cases exit code 127 (command not +# found) by not capturing the output but instead passing it through. This is +# not done for any other exit code. +# +# 2. Characters like '#' (from these programs' output) would end up in make +# variables like build_id_string, which would be wrongly interpreted by make +# when these variables were used. +# +# Therefore, we should avoid having arbitrary strings in make variables where +# possible. The gen_id script used here hashes the output to construct a +# "make-safe" id. +# +# Also note that these lines need to be: +# +# 1. After including {hosts,builders}/*.mk, since they rely on the tool +# variables (e.g. build_CC, host_STRIP, etc.) to be set. +# +# 2. Before including packages/*.mk (excluding packages/packages.mk), since +# they rely on the build_id variables +# +build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') + +qrencode_packages_$(NO_QR) = $(qrencode_packages) + +qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) + +bdb_packages_$(NO_BDB) = $(bdb_packages) +sqlite_packages_$(NO_SQLITE) = $(sqlite_packages) +wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_) + upnp_packages_$(NO_UPNP) = $(upnp_packages) +natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) + +zmq_packages_$(NO_ZMQ) = $(zmq_packages) +multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(qt_packages_),) native_packages += $(qt_native_packages) endif +ifneq ($(zmq_packages_),) +packages += $(zmq_packages) +endif + +ifeq ($(multiprocess_packages_),) +packages += $(multiprocess_packages) +native_packages += $(multiprocess_native_packages) +endif + all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk +$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils) $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk -toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) @@ -119,24 +189,55 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ +# $PATH is not preserved between ./configure and make by convention. Its +# modification and overriding at ./configure time is (as I understand it) +# supposed to be captured by the AC_{PROG_{,OBJ}CXX,PATH_{PROG,TOOL}} macros, +# which will expand the program names to their full absolute paths. The notable +# exception is command line overriding: ./configure CC=clang, which skips the +# program name expansion step, and works because the user implicitly indicates +# with CC=clang that clang will be available in $PATH at all times, and is most +# likely part of the user's system. +# +# Therefore, when we "seed the autoconf cache"/"override well-known program +# vars" by setting AR= in our config.site, either one of two things needs +# to be true for the build system to work correctly: +# +# 1. If we refer to the program by name (e.g. AR=riscv64-gnu-linux-ar), the +# tool needs to be available in $PATH at all times. +# +# 2. If the tool is _**not**_ expected to be available in $PATH at all times +# (such as is the case for our native_cctools binutils tools), it needs to +# be referred to by its absolute path, such as would be output by the +# AC_PATH_{PROG,TOOL} macros. +# +# Minor note: it is also okay to refer to tools by their absolute path even if +# we expect them to be available in $PATH at all times, more specificity does +# not hurt. $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) $(AT)@mkdir -p $(@D) $(AT)sed -e 's|@HOST@|$(host)|' \ - -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ - -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ - -e 's|@AR@|$(toolchain_path)$(host_AR)|' \ - -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \ - -e 's|@NM@|$(toolchain_path)$(host_NM)|' \ - -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \ + -e 's|@CC@|$(host_CC)|' \ + -e 's|@CXX@|$(host_CXX)|' \ + -e 's|@AR@|$(host_AR)|' \ + -e 's|@RANLIB@|$(host_RANLIB)|' \ + -e 's|@NM@|$(host_NM)|' \ + -e 's|@STRIP@|$(host_STRIP)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ -e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \ -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \ -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ + -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ -e 's|@no_qt@|$(NO_QT)|' \ + -e 's|@no_qr@|$(NO_QR)|' \ + -e 's|@no_zmq@|$(NO_ZMQ)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ + -e 's|@no_bdb@|$(NO_BDB)|' \ + -e 's|@no_sqlite@|$(NO_SQLITE)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ + -e 's|@no_natpmp@|$(NO_NATPMP)|' \ + -e 's|@multiprocess@|$(MULTIPROCESS)|' \ -e 's|@debug@|$(DEBUG)|' \ $< > $@ $(AT)touch $@ @@ -165,18 +266,26 @@ $(host_prefix)/share/config.site: check-packages check-packages: check-sources +clean-all: clean + @rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* powerpc* riscv32* riscv64* s390x* + +clean: + @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) + install: check-packages $(host_prefix)/share/config.site download-one: check-sources $(all_sources) download-osx: - @$(MAKE) -s HOST=x86_64-apple-darwin11 download-one + @$(MAKE) -s HOST=x86_64-apple-darwin download-one download-linux: @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win +$(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) -.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources +.PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources +.PHONY: FORCE diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 7990b872fd..001c928424 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -1,17 +1,17 @@ -build_darwin_CC: = $(shell xcrun -f clang) -build_darwin_CXX: = $(shell xcrun -f clang++) -build_darwin_AR: = $(shell xcrun -f ar) -build_darwin_RANLIB: = $(shell xcrun -f ranlib) -build_darwin_STRIP: = $(shell xcrun -f strip) -build_darwin_OTOOL: = $(shell xcrun -f otool) -build_darwin_NM: = $(shell xcrun -f nm) +build_darwin_CC:=$(shell xcrun -f clang) -isysroot$(shell xcrun --show-sdk-path) +build_darwin_CXX:=$(shell xcrun -f clang++) -isysroot$(shell xcrun --show-sdk-path) +build_darwin_AR:=$(shell xcrun -f ar) +build_darwin_RANLIB:=$(shell xcrun -f ranlib) +build_darwin_STRIP:=$(shell xcrun -f strip) +build_darwin_OTOOL:=$(shell xcrun -f otool) +build_darwin_NM:=$(shell xcrun -f nm) build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) -build_darwin_SHA256SUM = shasum -a 256 -build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o +build_darwin_SHA256SUM=shasum -a 256 +build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o #darwin host on darwin builder. overrides darwin host preferences. -darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(shell xcrun --show-sdk-path) -darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ --sysroot $(shell xcrun --show-sdk-path) +darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot$(shell xcrun --show-sdk-path) +darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path) darwin_AR:=$(shell xcrun -f ar) darwin_RANLIB:=$(shell xcrun -f ranlib) darwin_STRIP:=$(shell xcrun -f strip) @@ -19,4 +19,5 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool) darwin_OTOOL:=$(shell xcrun -f otool) darwin_NM:=$(shell xcrun -f nm) darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +darwin_native_binutils= darwin_native_toolchain= diff --git a/depends/config.site.in b/depends/config.site.in index 3d7c9fd43c..87d0011b1c 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -1,84 +1,114 @@ -depends_prefix="`dirname ${ac_site_file}`/.." +# shellcheck shell=sh disable=SC2034 # Many variables set will be used in + # ./configure but shellcheck doesn't know + # that, hence: disable=SC2034 + +true # Dummy command because shellcheck treats all directives before first + # command as file-wide, and we only want to disable for one line. + # + # See: https://github.com/koalaman/shellcheck/wiki/Directive + +# shellcheck disable=SC2154 +depends_prefix="$(cd "$(dirname "$ac_site_file")/.." && pwd)" cross_compiling=maybe -host_alias=@HOST@ -ac_tool_prefix=${host_alias}- +host_alias="@HOST@" +ac_tool_prefix="${host_alias}-" -if test -z $with_boost; then - with_boost=$depends_prefix +if test -z "$with_boost"; then + with_boost="$depends_prefix" fi -if test -z $with_qt_plugindir; then - with_qt_plugindir=$depends_prefix/plugins +if test -z "$with_qt_plugindir"; then + with_qt_plugindir="${depends_prefix}/plugins" fi -if test -z $with_qt_translationdir; then - with_qt_translationdir=$depends_prefix/translations +if test -z "$with_qt_translationdir"; then + with_qt_translationdir="${depends_prefix}/translations" fi -if test -z $with_qt_bindir; then - with_qt_bindir=$depends_prefix/native/bin +if test -z "$with_qt_bindir" && test -z "@no_qt@"; then + with_qt_bindir="${depends_prefix}/native/bin" fi -if test -z $with_protoc_bindir; then - with_protoc_bindir=$depends_prefix/native/bin +if test -z "$with_mpgen" && test -n "@multiprocess@"; then + with_mpgen="${depends_prefix}/native" fi +if test -z "$with_qrencode" && test -n "@no_qr@"; then + with_qrencode=no +fi -if test -z $enable_wallet && test -n "@no_wallet@"; then +if test -z "$enable_wallet" && test -n "@no_wallet@"; then enable_wallet=no fi -if test -z $with_miniupnpc && test -n "@no_upnp@"; then +if test -z "$with_bdb" && test -n "@no_bdb@"; then + with_bdb=no +fi + +if test -z "$with_sqlite" && test -n "@no_sqlite@"; then + with_sqlite=no +fi + +if test -z "$enable_multiprocess" && test -n "@multiprocess@"; then + enable_multiprocess=yes +fi + +if test -z "$with_miniupnpc" && test -n "@no_upnp@"; then with_miniupnpc=no fi -if test -z $with_gui && test -n "@no_qt@"; then +if test -z "$with_natpmp" && test -n "@no_natpmp@"; then + with_natpmp=no +fi + +if test -z "$with_gui" && test -n "@no_qt@"; then with_gui=no fi -if test x@host_os@ = xdarwin; then - BREW=no - PORT=no +if test -n "@debug@" && test -z "@no_qt@" && test "x$with_gui" != xno; then + with_gui=qt5_debug fi -if test x@host_os@ = xmingw32; then - if test -z $with_qt_incdir; then - with_qt_incdir=$depends_prefix/include - fi - if test -z $with_qt_libdir; then - with_qt_libdir=$depends_prefix/lib - fi +if test -z "$enable_zmq" && test -n "@no_zmq@"; then + enable_zmq=no +fi + +if test "@host_os@" = darwin; then + BREW=no fi -PATH=$depends_prefix/native/bin:$PATH -PKG_CONFIG="`which pkg-config` --static" +PATH="${depends_prefix}/native/bin:${PATH}" +PKG_CONFIG="$(which pkg-config) --static" # These two need to remain exported because pkg-config does not see them # otherwise. That means they must be unexported at the end of configure.ac to # avoid ruining the cache. Sigh. +export PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig" +if test -z "@allow_host_packages@"; then + export PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig" +fi -export PKG_CONFIG_LIBDIR=$depends_prefix/lib/pkgconfig -export PKG_CONFIG_PATH=$depends_prefix/share/pkgconfig - -CPPFLAGS="-I$depends_prefix/include/ $CPPFLAGS" -LDFLAGS="-L$depends_prefix/lib $LDFLAGS" +CPPFLAGS="-I${depends_prefix}/include/ ${CPPFLAGS}" +LDFLAGS="-L${depends_prefix}/lib ${LDFLAGS}" -CC="@CC@" -CXX="@CXX@" -OBJC="${CC}" -CCACHE=$depends_prefix/native/bin/ccache -PYTHONPATH=$depends_prefix/native/lib/python/dist-packages:$PYTHONPATH +if test -n "@CC@" -a -z "${CC}"; then + CC="@CC@" +fi +if test -n "@CXX@" -a -z "${CXX}"; then + CXX="@CXX@" +fi +PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PATH_SEPARATOR}}${PYTHONPATH}" if test -n "@AR@"; then - AR=@AR@ - ac_cv_path_ac_pt_AR=${AR} + AR="@AR@" + ac_cv_path_ac_pt_AR="${AR}" fi if test -n "@RANLIB@"; then - RANLIB=@RANLIB@ - ac_cv_path_ac_pt_RANLIB=${RANLIB} + RANLIB="@RANLIB@" + ac_cv_path_ac_pt_RANLIB="${RANLIB}" fi if test -n "@NM@"; then - NM=@NM@ - ac_cv_path_ac_pt_NM=${NM} + NM="@NM@" + ac_cv_path_ac_pt_NM="${NM}" fi if test -n "@debug@"; then @@ -86,14 +116,14 @@ if test -n "@debug@"; then fi if test -n "@CFLAGS@"; then - CFLAGS="@CFLAGS@ $CFLAGS" + CFLAGS="@CFLAGS@ ${CFLAGS}" fi if test -n "@CXXFLAGS@"; then - CXXFLAGS="@CXXFLAGS@ $CXXFLAGS" + CXXFLAGS="@CXXFLAGS@ ${CXXFLAGS}" fi if test -n "@CPPFLAGS@"; then - CPPFLAGS="@CPPFLAGS@ $CPPFLAGS" + CPPFLAGS="@CPPFLAGS@ ${CPPFLAGS}" fi if test -n "@LDFLAGS@"; then - LDFLAGS="@LDFLAGS@ $LDFLAGS" + LDFLAGS="@LDFLAGS@ ${LDFLAGS}" fi diff --git a/depends/funcs.mk b/depends/funcs.mk index fabbb25356..34a030fab7 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -1,17 +1,23 @@ define int_vars #Set defaults for vars which may be overridden per-package -$(1)_cc=$($($(1)_type)_CC) -$(1)_cxx=$($($(1)_type)_CXX) -$(1)_objc=$($($(1)_type)_OBJC) -$(1)_objcxx=$($($(1)_type)_OBJCXX) -$(1)_ar=$($($(1)_type)_AR) -$(1)_ranlib=$($($(1)_type)_RANLIB) -$(1)_libtool=$($($(1)_type)_LIBTOOL) -$(1)_nm=$($($(1)_type)_NM) -$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS) -$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS) -$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib -$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include +$(1)_cc=$$($$($(1)_type)_CC) +$(1)_cxx=$$($$($(1)_type)_CXX) +$(1)_objc=$$($$($(1)_type)_OBJC) +$(1)_objcxx=$$($$($(1)_type)_OBJCXX) +$(1)_ar=$$($$($(1)_type)_AR) +$(1)_ranlib=$$($$($(1)_type)_RANLIB) +$(1)_libtool=$$($$($(1)_type)_LIBTOOL) +$(1)_nm=$$($$($(1)_type)_NM) +$(1)_cflags=$$($$($(1)_type)_CFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CFLAGS) +$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CXXFLAGS) +$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \ + $$($$($(1)_type)_$$(release_type)_LDFLAGS) \ + -L$$($($(1)_type)_prefix)/lib +$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \ + -I$$($$($(1)_type)_prefix)/include $(1)_recipe_hash:= endef @@ -41,9 +47,9 @@ endef define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) -$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies))) +$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) -$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string)) +$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) final_build_id_long+=$($(package)_build_id_long) @@ -76,6 +82,7 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) #default commands +# The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) $(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --no-same-owner --strip-components=1 -xf $$($(1)_source) $(1)_preprocess_cmds ?= @@ -129,11 +136,17 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$( $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig +$(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH) -$(1)_autoconf=./configure --host=$($($(1)_type)_host) --disable-dependency-tracking --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" +# Setting a --build type that differs from --host will explicitly enable +# cross-compilation mode. Note that --build defaults to the output of +# config.guess, which is what we set it too here. This also quells autoconf +# warnings, "If you wanted to set the --build type, don't use --host.", +# when using versions older than 2.70. +$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif @@ -155,6 +168,22 @@ endif ifneq ($($(1)_ldflags),) $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" endif + +$(1)_cmake=env CC="$$($(1)_cc)" \ + CFLAGS="$$($(1)_cppflags) $$($(1)_cflags)" \ + CXX="$$($(1)_cxx)" \ + CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ + LDFLAGS="$$($(1)_ldflags)" \ + cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" +ifeq ($($(1)_type),build) +$(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib" +else +ifneq ($(host),$(build)) +$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) +$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) +$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host) +endif +endif endef define int_add_cmds @@ -170,13 +199,13 @@ $($(1)_extracted): | $($(1)_fetched) $(AT)mkdir -p $$(@D) $(AT)cd $$(@D); $(call $(1)_extract_cmds,$(1)) $(AT)touch $$@ -$($(1)_preprocessed): | $($(1)_dependencies) $($(1)_extracted) +$($(1)_preprocessed): | $($(1)_extracted) $(AT)echo Preprocessing $(1)... $(AT)mkdir -p $$(@D) $($(1)_patch_dir) $(AT)$(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) $(AT)cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) $(AT)touch $$@ -$($(1)_configured): | $($(1)_preprocessed) +$($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) $(AT)echo Configuring $(1)... $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar --no-same-owner -xf $($(package)_cached); ) $(AT)mkdir -p $$(@D) @@ -199,7 +228,7 @@ $($(1)_postprocessed): | $($(1)_staged) $(AT)touch $$@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) $(AT)echo Caching $(1)... - $(AT)cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | tar --no-same-owner --no-recursion -cvzf $$($(1)_staging_dir)/$$(@F) -T - + $(AT)cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | tar --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - $(AT)mkdir -p $$(@D) $(AT)rm -rf $$(@D) && mkdir -p $$(@D) $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) @@ -213,6 +242,14 @@ $(1): | $($(1)_cached_checksum) endef +stages = fetched extracted preprocessed configured built staged postprocessed cached cached_checksum + +define ext_add_stages +$(foreach stage,$(stages), + $(1)_$(stage): $($(1)_$(stage)) + .PHONY: $(1)_$(stage)) +endef + # These functions create the build targets for each package. They must be # broken down into small steps so that each part is done for all packages # before moving on to the next step. Otherwise, a package's info @@ -242,4 +279,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$ $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package)))) #special exception: if a toolchain package exists, all non-native packages depend on it -$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) )) +$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) )) diff --git a/depends/gen_id b/depends/gen_id new file mode 100755 index 0000000000..ac69ca7ee1 --- /dev/null +++ b/depends/gen_id @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Usage: env [ CC=... ] [ CXX=... ] [ AR=... ] [ RANLIB=... ] [ STRIP=... ] \ +# [ DEBUG=... ] ./build-id [ID_SALT]... +# +# Prints to stdout a SHA256 hash representing the current toolset, used by +# depends/Makefile as a build id for caching purposes (detecting when the +# toolset has changed and the cache needs to be invalidated). +# +# If the DEBUG environment variable is non-empty and the system has `tee` +# available in its $PATH, the pre-image to the SHA256 hash will be printed to +# stderr. This is to help developers debug caching issues in depends. + +# This script explicitly does not `set -e` because id determination is mostly +# opportunistic: it is fine that things fail, as long as they fail consistently. + +# Command variables (CC/CXX/AR) which can be blank are invoked with `bash -c`, +# because the "command not found" error message printed by shells often include +# the line number, like so: +# +# ./depends/gen_id: line 43: --version: command not found +# +# By invoking with `bash -c`, we ensure that the line number is always 1 + +( + # Redirect stderr to stdout + exec 2>&1 + + echo "BEGIN ALL" + + # Include any ID salts supplied via command line + echo "BEGIN ID SALT" + echo "$@" + echo "END ID SALT" + + # GCC only prints COLLECT_LTO_WRAPPER when invoked with just "-v", but we want + # the information from "-v -E -" as well, so just include both. + echo "BEGIN CC" + bash -c "${CC} -v" + bash -c "${CC} -v -E -xc -o /dev/null - < /dev/null" + bash -c "${CC} -v -E -xobjective-c -o /dev/null - < /dev/null" + echo "END CC" + + echo "BEGIN CXX" + bash -c "${CXX} -v" + bash -c "${CXX} -v -E -xc++ -o /dev/null - < /dev/null" + bash -c "${CXX} -v -E -xobjective-c++ -o /dev/null - < /dev/null" + echo "END CXX" + + echo "BEGIN AR" + bash -c "${AR} --version" + env | grep '^AR_' + echo "ZERO_AR_DATE=${ZERO_AR_DATE}" + echo "END AR" + + echo "BEGIN RANLIB" + bash -c "${RANLIB} --version" + env | grep '^RANLIB_' + echo "END RANLIB" + + echo "BEGIN STRIP" + bash -c "${STRIP} --version" + env | grep '^STRIP_' + echo "END STRIP" + + echo "END ALL" +) | if [ -n "$DEBUG" ] && command -v tee > /dev/null 2>&1; then + # When debugging and `tee` is available, output the preimage to stderr + # in addition to passing through stdin to stdout + tee >(cat 1>&2) + else + # Otherwise, passthrough stdin to stdout + cat + fi | ${SHA256SUM} - | cut -d' ' -f1 diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 1bc4fb8189..ea92bb7793 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,8 +1,113 @@ -OSX_MIN_VERSION=10.12 -OSX_SDK_VERSION=10.14 -OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ +OSX_MIN_VERSION=10.15 +OSX_SDK_VERSION=10.15.6 +XCODE_VERSION=12.1 +XCODE_BUILD_ID=12A7403 +LD64_VERSION=609 + +OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers + +darwin_native_binutils=native_cctools + +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang +# from llvm.org + +# Clang is a dependency of native_cctools when FORCE_USE_SYSTEM_CLANG is empty +darwin_native_toolchain=native_cctools + +clang_prog=$(build_prefix)/bin/clang +clangxx_prog=$(clang_prog)++ + +clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version) +else +# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's +# system + +darwin_native_toolchain= + +# We can't just use $(shell command -v clang) because GNU Make handles builtins +# in a special way and doesn't know that `command` is a POSIX-standard builtin +# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. +# At the time of writing, GNU Make v4.2.1 is still being used in supported +# distro releases. +# +# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html +clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") +clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") + +clang_resource_dir=$(shell clang -print-resource-dir) +endif + +cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL + +# Make-only lowercase function +lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) + +# For well-known tools provided by cctools, make sure that their well-known +# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG} +# would. +$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL)))) + +# Flag explanations: +# +# -mlinker-version +# +# Ensures that modern linker features are enabled. See here for more +# details: https://github.com/bitcoin/bitcoin/pull/19407. +# +# -B$(build_prefix)/bin +# +# Explicitly point to our binaries (e.g. cctools) so that they are +# ensured to be found and preferred over other possibilities. +# +# -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 +# +# Forces clang to use the libc++ headers from our SDK and completely +# forget about the libc++ headers from the standard directories +# +# -Xclang -*system \ +# -Xclang -*system \ +# -Xclang -*system ... +# +# Adds path_a, path_b, and path_c to the bottom of clang's list of +# include search paths. This is used to explicitly specify the list of +# system include search paths and its ordering, rather than rely on +# clang's autodetection routine. This routine has been shown to: +# 1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1 +# when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034) +# 2. Fail to pickup C headers in $SYSROOT/usr/include when +# C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9) +# +# Talking directly to cc1 with -Xclang here grants us access to specify +# more granular categories for these system include search paths, and we +# can use the correct categories that these search paths would have been +# placed in if the autodetection routine had worked correctly. (see: +# https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment) +# +# Furthermore, it places these search paths after any "non-Xclang" +# specified search paths. This prevents any additional clang options or +# environment variables from coming after or in between these system +# include search paths, as that would be wrong in general but would also +# break #include_next's. +# +darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + -isysroot$(OSX_SDK) \ + -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include +darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + -isysroot$(OSX_SDK) \ + -stdlib=libc++ \ + -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \ + -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) @@ -13,4 +118,4 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) -darwin_native_toolchain=native_cctools +darwin_cmake_system=Darwin diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index 6f60d6b3fd..258619a9d0 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -1,3 +1,7 @@ +ifneq ($(host),$(build)) +host_toolchain:=$(host)- +endif + default_host_CC = $(host_toolchain)gcc default_host_CXX = $(host_toolchain)g++ default_host_AR = $(host_toolchain)ar @@ -9,9 +13,18 @@ default_host_OTOOL = $(host_toolchain)otool default_host_NM = $(host_toolchain)nm define add_host_tool_func +ifneq ($(filter $(origin $1),undefined default),) +# Do not consider the well-known var $1 if it is undefined or is taking a value +# that is predefined by "make" (e.g. the make variable "CC" has a predefined +# value of "cc") $(host_os)_$1?=$$(default_host_$1) $(host_arch)_$(host_os)_$1?=$$($(host_os)_$1) $(host_arch)_$(host_os)_$(release_type)_$1?=$$($(host_os)_$1) +else +$(host_os)_$1=$(or $($1),$($(host_os)_$1),$(default_host_$1)) +$(host_arch)_$(host_os)_$1=$(or $($1),$($(host_arch)_$(host_os)_$1),$$($(host_os)_$1)) +$(host_arch)_$(host_os)_$(release_type)_$1=$(or $($1),$($(host_arch)_$(host_os)_$(release_type)_$1),$$($(host_os)_$1)) +endif host_$1=$$($(host_arch)_$(host_os)_$1) endef diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index 4260c9a902..be5fec570c 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -9,4 +9,4 @@ mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -mingw_CC= +mingw_cmake_system=Windows diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index fe112a84f8..0a0208e2c3 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,45 +1,46 @@ package=boost -$(package)_version=1_70_0 -$(package)_download_path=https://dl.bintray.com/boostorg/release/1.70.0/source/ -$(package)_file_name=$(package)_$($(package)_version).tar.bz2 -$(package)_sha256_hash=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 +$(package)_version=1_77_0 +$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$(subst _,.,$($(package)_version))/source/ +$(package)_file_name=boost_$($(package)_version).tar.bz2 +$(package)_sha256_hash=fc9f85fc030e233142908241af7a846e60630aa7388de9a5fafb1f3a26840854 +$(package)_dependencies=native_b2 define $(package)_set_vars $(package)_config_opts_release=variant=release $(package)_config_opts_debug=variant=debug $(package)_config_opts=--layout=tagged --build-type=complete --user-config=user-config.jam -$(package)_config_opts+=threading=multi link=static -sNO_BZIP2=1 -sNO_ZLIB=1 -$(package)_config_opts_linux=threadapi=pthread runtime-link=shared -$(package)_config_opts_darwin=--toolset=clang-darwin runtime-link=shared -$(package)_config_opts_mingw32=binary-format=pe target-os=windows threadapi=win32 runtime-link=static -$(package)_config_opts_x86_64_mingw32=address-model=64 -$(package)_config_opts_i686_mingw32=address-model=32 -$(package)_config_opts_i686_linux=address-model=32 architecture=x86 -$(package)_config_opts_i686_android=address-model=32 -$(package)_config_opts_aarch64_android=address-model=64 -$(package)_config_opts_x86_64_android=address-model=64 -$(package)_config_opts_armv7a_android=address-model=32 +$(package)_config_opts+=threading=multi link=static -sNO_COMPRESSION=1 +$(package)_config_opts_linux=target-os=linux threadapi=pthread runtime-link=shared +$(package)_config_opts_darwin=target-os=darwin runtime-link=shared +$(package)_config_opts_mingw32=target-os=windows binary-format=pe threadapi=win32 runtime-link=static +$(package)_config_opts_x86_64=architecture=x86 address-model=64 +$(package)_config_opts_i686=architecture=x86 address-model=32 +$(package)_config_opts_aarch64=address-model=64 +$(package)_config_opts_armv7a=address-model=32 +ifneq (,$(findstring clang,$($(package)_cxx))) +$(package)_toolset_$(host_os)=clang +else $(package)_toolset_$(host_os)=gcc -$(package)_archiver_$(host_os)=$($(package)_ar) -$(package)_toolset_darwin=clang-darwin +endif $(package)_config_libraries=chrono,filesystem,program_options,system,thread,test -$(package)_cxxflags=-std=c++11 -fvisibility=hidden +$(package)_cxxflags+=-std=c++17 $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_android=-fPIC +$(package)_cxxflags_x86_64_darwin=-fcf-protection=full endef define $(package)_preprocess_cmds - echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : \"$($(package)_cxxflags) $($(package)_cppflags)\" \"$($(package)_ldflags)\" \"$($(package)_archiver_$(host_os))\" \"$(host_STRIP)\" \"$(host_RANLIB)\" \"$(host_WINDRES)\" : ;" > user-config.jam + echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : \"$($(package)_cflags)\" \"$($(package)_cxxflags)\" \"$($(package)_cppflags)\" \"$($(package)_ldflags)\" \"$($(package)_ar)\" \"$(host_STRIP)\" \"$(host_RANLIB)\" \"$(host_WINDRES)\" : ;" > user-config.jam endef define $(package)_config_cmds - ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) + ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) --with-toolset=$($(package)_toolset_$(host_os)) --with-bjam=b2 endef define $(package)_build_cmds - ./b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) stage + b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) define=BOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX define=BOOST_FILESYSTEM_DISABLE_STATX define=BOOST_FILESYSTEM_DISABLE_GETRANDOM define=BOOST_FILESYSTEM_DISABLE_GETENTROPY stage endef define $(package)_stage_cmds - ./b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) install + b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) --no-cmake-config install endef diff --git a/depends/packages/libxkbcommon.mk b/depends/packages/libxkbcommon.mk new file mode 100644 index 0000000000..8c6c56545f --- /dev/null +++ b/depends/packages/libxkbcommon.mk @@ -0,0 +1,32 @@ +package=libxkbcommon +$(package)_version=0.8.4 +$(package)_download_path=https://xkbcommon.org/download/ +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=60ddcff932b7fd352752d51a5c4f04f3d0403230a584df9a2e0d5ed87c486c8b +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts = --enable-option-checking --disable-dependency-tracking +$(package)_config_opts += --disable-static --disable-docs +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/depends/packages/native_b2.mk b/depends/packages/native_b2.mk new file mode 100644 index 0000000000..aaa37cdcfa --- /dev/null +++ b/depends/packages/native_b2.mk @@ -0,0 +1,20 @@ +package=native_b2 +$(package)_version=$(boost_version) +$(package)_download_path=$(boost_download_path) +$(package)_file_name=$(boost_file_name) +$(package)_sha256_hash=$(boost_sha256_hash) +$(package)_build_subdir=tools/build/src/engine +ifneq (,$(findstring clang,$($(package)_cxx))) +$(package)_toolset_$(host_os)=clang +else +$(package)_toolset_$(host_os)=gcc +endif + +define $(package)_build_cmds + CXX="$($(package)_cxx)" CXXFLAGS="$($(package)_cxxflags)" ./build.sh "$($(package)_toolset_$(host_os))" +endef + +define $(package)_stage_cmds + mkdir -p "$($(package)_staging_prefix_dir)"/bin/ && \ + cp b2 "$($(package)_staging_prefix_dir)"/bin/ +endef diff --git a/depends/packages/native_biplist.mk b/depends/packages/native_biplist.mk deleted file mode 100644 index 3c6e8900f6..0000000000 --- a/depends/packages/native_biplist.mk +++ /dev/null @@ -1,20 +0,0 @@ -package=native_biplist -$(package)_version=0.9 -$(package)_download_path=https://pypi.python.org/packages/source/b/biplist -$(package)_file_name=biplist-$($(package)_version).tar.gz -$(package)_sha256_hash=b57cadfd26e4754efdf89e9e37de87885f9b5c847b2615688ca04adfaf6ca604 -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages -$(package)_patches=sorted_list.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/sorted_list.patch -endef - -define $(package)_build_cmds - python setup.py build -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) -endef diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 4195230b40..d169eb6723 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,55 +1,23 @@ package=native_cctools -$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f +$(package)_version=2ef2e931cf641547eb8a68cfebde61003587c9fd $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04 +$(package)_sha256_hash=6b73269efdf5c58a070e7357b66ee760501388549d6a12b423723f45888b074b $(package)_build_subdir=cctools -$(package)_clang_version=6.0.1 -$(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) -$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0 - -$(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef -$(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive -$(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz -$(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz -$(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3 - -$(package)_extra_sources=$($(package)_clang_file_name) -$(package)_extra_sources += $($(package)_libtapi_file_name) - -define $(package)_fetch_cmds -$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) -endef - -define $(package)_extract_cmds - mkdir -p $($(package)_extract_dir) && \ - echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_libtapi_sha256_hash) $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - mkdir -p toolchain/bin toolchain/lib/clang/$($(package)_clang_version)/include && \ - mkdir -p libtapi && \ - tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \ - tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ - rm -f toolchain/lib/libc++abi.so* && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source) -endef +$(package)_dependencies=native_libtapi define $(package)_set_vars - $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir) + $(package)_config_opts=--target=$(host) $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib - $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang - $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ + ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) + $(package)_config_opts+=--enable-lto-support --with-llvm-config=$(build_prefix)/bin/llvm-config + endif + $(package)_cc=$(clang_prog) + $(package)_cxx=$(clangxx_prog) endef define $(package)_preprocess_cmds - CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \ - CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \ - sed -i.old "/define HAVE_PTHREADS/d" $($(package)_build_subdir)/ld64/src/ld/InputFiles.h + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools endef define $(package)_config_cmds @@ -61,18 +29,9 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ - mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ - cd $($(package)_extract_dir) && \ - cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ && \ - cd $($(package)_extract_dir)/toolchain && \ - mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \ - mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ - cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\ - cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ - cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ - cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ - if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ - if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share endef diff --git a/depends/packages/native_cdrkit.mk b/depends/packages/native_cdrkit.mk deleted file mode 100644 index cf694edb30..0000000000 --- a/depends/packages/native_cdrkit.mk +++ /dev/null @@ -1,26 +0,0 @@ -package=native_cdrkit -$(package)_version=1.1.11 -$(package)_download_path=http://distro.ibiblio.org/fatdog/source/600/c -$(package)_file_name=cdrkit-$($(package)_version).tar.bz2 -$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 -$(package)_patches=cdrkit-deterministic.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch -endef - -define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) -endef - -define $(package)_build_cmds - $(MAKE) genisoimage -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install -endef - -define $(package)_postprocess_cmds - rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump -endef diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk new file mode 100644 index 0000000000..25ac77c1a3 --- /dev/null +++ b/depends/packages/native_clang.mk @@ -0,0 +1,32 @@ +package=native_clang +$(package)_version=10.0.1 +$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) +ifneq (,$(findstring aarch64,$(BUILD))) +$(package)_download_file=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz +$(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz +$(package)_sha256_hash=90dc69a4758ca15cd0ffa45d07fbf5bf4309d47d2c7745a9f0735ecffde9c31f +else +$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz +$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz +$(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 +endif + +define $(package)_preprocess_cmds + rm -f $($(package)_extract_dir)/lib/libc++abi.so* +endef + +define $(package)_stage_cmds + mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ + mkdir -p $($(package)_staging_prefix_dir)/bin && \ + mkdir -p $($(package)_staging_prefix_dir)/include && \ + cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ + cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \ + cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ + cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \ + cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ + cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ +endef + +define $(package)_postprocess_cmds + rmdir include +endef diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk index 0e6dad1353..44108925a4 100644 --- a/depends/packages/native_ds_store.mk +++ b/depends/packages/native_ds_store.mk @@ -1,17 +1,15 @@ package=native_ds_store -$(package)_version=1.1.0 -$(package)_download_path=https://github.com/al45tair/ds_store/archive -$(package)_download_file=v$($(package)_version).tar.gz -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=a9f4c0755c6be7224ff7029e188dd262e830bb81e801424841db9eb0780ec8ed -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages -$(package)_dependencies=native_biplist +$(package)_version=1.3.0 +$(package)_download_path=https://github.com/al45tair/ds_store/archive/ +$(package)_file_name=v$($(package)_version).tar.gz +$(package)_sha256_hash=76b3280cd4e19e5179defa23fb594a9dd32643b0c80d774bd3108361d94fb46d +$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk index a4ffb6046c..c7c8adef41 100644 --- a/depends/packages/native_libdmg-hfsplus.mk +++ b/depends/packages/native_libdmg-hfsplus.mk @@ -1,16 +1,18 @@ package=native_libdmg-hfsplus -$(package)_version=0.1 -$(package)_download_path=https://github.com/theuni/libdmg-hfsplus/archive -$(package)_file_name=libdmg-hfsplus-v$($(package)_version).tar.gz -$(package)_sha256_hash=6569a02eb31c2827080d7d59001869ea14484c281efab0ae7f2b86af5c3120b3 +$(package)_version=7ac55ec64c96f7800d9818ce64c79670e7f02b67 +$(package)_download_path=https://github.com/planetbeing/libdmg-hfsplus/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=56fbdc48ec110966342f0ecddd6f8f89202f4143ed2a3336e42bbf88f940850c $(package)_build_subdir=build +$(package)_patches=remove-libcrypto-dependency.patch define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/remove-libcrypto-dependency.patch && \ mkdir build endef define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix)/bin .. + $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" -DCMAKE_SKIP_RPATH="ON" -DCMAKE_EXE_LINKER_FLAGS="-static" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" .. endef define $(package)_build_cmds diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk new file mode 100644 index 0000000000..60b898da5f --- /dev/null +++ b/depends/packages/native_libtapi.mk @@ -0,0 +1,20 @@ +package=native_libtapi +$(package)_version=664b8414f89612f2dfd35a9b679c345aa5389026 +$(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive +$(package)_download_file=$($(package)_version).tar.gz +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61 + +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +$(package)_dependencies=native_clang +endif + +define $(package)_build_cmds + CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh +endef + +define $(package)_stage_cmds + ./install.sh && \ + mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ + cp src/llvm/include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c +endef diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index ed8835acad..783f87ca7c 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,21 +1,15 @@ package=native_mac_alias -$(package)_version=1.1.0 -$(package)_download_path=https://github.com/al45tair/mac_alias/archive -$(package)_download_file=v$($(package)_version).tar.gz -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=b10cb44ecb64fc25283fae7a9cf365d2829377d84e37b9c21100aca8757509be -$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages -$(package)_patches=python3.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/python3.patch -endef +$(package)_version=2.2.0 +$(package)_download_path=https://github.com/al45tair/mac_alias/archive/ +$(package)_file_name=v$($(package)_version).tar.gz +$(package)_sha256_hash=421e6d7586d1f155c7db3e7da01ca0dacc9649a509a253ad7077b70174426499 +$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_protobuf.mk b/depends/packages/native_protobuf.mk index ce50b366fa..1de8c37d36 100644 --- a/depends/packages/native_protobuf.mk +++ b/depends/packages/native_protobuf.mk @@ -5,7 +5,7 @@ $(package)_file_name=protobuf-$($(package)_version).tar.bz2 $(package)_sha256_hash=ee445612d544d885ae240ffbcbf9267faa9f593b7b101f21d58beceb92661910 define $(package)_set_vars -$(package)_config_opts=--disable-shared +$(package)_config_opts=--disable-shared --without-zlib endef define $(package)_config_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 73b16a6936..f37ef97a58 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -4,20 +4,28 @@ linux_packages:=zeromq native_packages := native_ccache native_comparisontool qt_native_packages = native_protobuf -qt_packages = qrencode protobuf zlib +qt_packages = qrencode protobuf -qt_x86_64_linux_packages:=qt expat dbus libxcb xcb_proto libXau xproto freetype fontconfig libX11 xextproto libXext xtrans -qt_i686_linux_packages:=$(qt_x86_64_linux_packages) +qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon qt_darwin_packages=qt qt_mingw32_packages=qt -wallet_packages=bdb +bdb_packages=bdb +sqlite_packages=sqlite upnp_packages=miniupnpc -darwin_native_packages = native_biplist native_ds_store native_mac_alias +darwin_native_packages = native_ds_store native_mac_alias + +$(host_arch)_$(host_os)_native_packages += native_b2 ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus +darwin_native_packages += native_cctools native_libtapi native_libdmg-hfsplus + +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +darwin_native_packages+= native_clang endif + +endif + diff --git a/depends/packages/protobuf.mk b/depends/packages/protobuf.mk index 54d3fd9245..9dcac58f87 100644 --- a/depends/packages/protobuf.mk +++ b/depends/packages/protobuf.mk @@ -11,8 +11,14 @@ define $(package)_set_vars $(package)_config_opts_linux=--with-pic endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . &&\ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub gtest/build-aux +endef + define $(package)_config_cmds - $($(package)_autoconf) + $($(package)_autoconf) &&\ + sed -i.old 's/-lstdc++ //g' libtool config.status endef define $(package)_build_cmds @@ -25,5 +31,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/libprotoc.a + rm lib/libprotoc.a lib/*.la endef diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 5adc6b7498..e46ad88eef 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,29 +1,37 @@ PACKAGE=qt -$(package)_version=5.9.8 -$(package)_download_path=https://download.qt.io/official_releases/qt/5.9/$($(package)_version)/submodules -$(package)_suffix=opensource-src-$($(package)_version).tar.xz +$(package)_version=5.12.11 +$(package)_download_path=https://download.qt.io/official_releases/qt/5.12/$($(package)_version)/submodules +$(package)_suffix=everywhere-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=9b9dec1f67df1f94bce2955c5604de992d529dde72050239154c56352da0907d +$(package)_sha256_hash=1c1b4e33137ca77881074c140d54c3c9747e845a31338cfe8680f171f0bc3a39 $(package)_dependencies=zlib openssl -$(package)_linux_dependencies=freetype fontconfig libxcb -$(package)_build_subdir=qtbase +$(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch +$(package)_linguist_tools = lrelease lupdate lconvert +$(package)_patches = qt.pro qttools_src.pro +$(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch +$(package)_patches += support_new_android_ndks.patch fix_android_jni_static.patch dont_hardcode_pwd.patch +$(package)_patches += dont_hardcode_x86_64.patch no_sdk_version_check.patch +$(package)_patches+= fix_lib_paths.patch fix_android_pch.patch +$(package)_patches+= qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch +$(package)_patches+= fix_montery_include.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=fb5a47799754af73d3bf501fe513342cfe2fc37f64e80df5533f6110e804220c +$(package)_qttranslations_sha256_hash=577b0668a777eb2b451c61e8d026d79285371597ce9df06b6dee6c814164b7c3 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=a97556eb7b2f30252cdd8a598c396cfce2b2f79d2bae883af6d3b26a2cdcc63c +$(package)_qttools_sha256_hash=98b2aaca230458f65996f3534fd471d2ffd038dd58ac997c0589c06dc2385b4f $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) define $(package)_set_vars $(package)_config_opts_release = -release +$(package)_config_opts_release += -silent $(package)_config_opts_debug = -debug +$(package)_config_opts_debug += -optimized-tools $(package)_config_opts += -bindir $(build_prefix)/bin -$(package)_config_opts += -c++std c++11 +$(package)_config_opts += -c++std c++1z $(package)_config_opts += -confirm-license $(package)_config_opts += -hostprefix $(build_prefix) $(package)_config_opts += -no-compile-examples @@ -44,7 +52,6 @@ $(package)_config_opts += -no-libudev $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-openvg $(package)_config_opts += -no-reduce-relocations -$(package)_config_opts += -no-qml-debug $(package)_config_opts += -no-sctp $(package)_config_opts += -no-sql-db2 $(package)_config_opts += -no-sql-ibase @@ -57,12 +64,10 @@ $(package)_config_opts += -no-sql-sqlite $(package)_config_opts += -no-sql-sqlite2 $(package)_config_opts += -no-system-proxies $(package)_config_opts += -no-use-gold-linker -$(package)_config_opts += -no-xinput2 $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests +$(package)_config_opts += -nomake tools $(package)_config_opts += -opensource -$(package)_config_opts += -optimized-tools -$(package)_config_opts += -pch $(package)_config_opts += -pkg-config $(package)_config_opts += -prefix $(host_prefix) $(package)_config_opts += -qt-libpng @@ -70,14 +75,12 @@ $(package)_config_opts += -qt-pcre $(package)_config_opts += -qt-harfbuzz $(package)_config_opts += -system-zlib $(package)_config_opts += -static -$(package)_config_opts += -silent $(package)_config_opts += -v $(package)_config_opts += -no-feature-bearermanagement $(package)_config_opts += -no-feature-colordialog $(package)_config_opts += -no-feature-commandlineparser $(package)_config_opts += -no-feature-concurrent $(package)_config_opts += -no-feature-dial -$(package)_config_opts += -no-feature-filesystemwatcher $(package)_config_opts += -no-feature-fontcombobox $(package)_config_opts += -no-feature-ftp $(package)_config_opts += -no-feature-http @@ -90,16 +93,15 @@ $(package)_config_opts += -no-feature-printdialog $(package)_config_opts += -no-feature-printer $(package)_config_opts += -no-feature-printpreviewdialog $(package)_config_opts += -no-feature-printpreviewwidget -$(package)_config_opts += -no-feature-regularexpression $(package)_config_opts += -no-feature-sessionmanager $(package)_config_opts += -no-feature-socks5 $(package)_config_opts += -no-feature-sql +$(package)_config_opts += -no-feature-sqlmodel $(package)_config_opts += -no-feature-statemachine $(package)_config_opts += -no-feature-syntaxhighlighter $(package)_config_opts += -no-feature-textbrowser $(package)_config_opts += -no-feature-textodfwriter $(package)_config_opts += -no-feature-topleveldomain -$(package)_config_opts += -no-feature-udpsocket $(package)_config_opts += -no-feature-undocommand $(package)_config_opts += -no-feature-undogroup $(package)_config_opts += -no-feature-undostack @@ -109,23 +111,31 @@ $(package)_config_opts += -no-feature-xml $(package)_config_opts_darwin = -no-dbus $(package)_config_opts_darwin += -no-opengl +$(package)_config_opts_darwin += -pch +$(package)_config_opts_darwin += -no-feature-corewlan +$(package)_config_opts_darwin += QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) ifneq ($(build_os),darwin) $(package)_config_opts_darwin += -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) $(package)_config_opts_darwin += -device-option MAC_SDK_VERSION=$(OSX_SDK_VERSION) $(package)_config_opts_darwin += -device-option CROSS_COMPILE="$(host)-" -$(package)_config_opts_darwin += -device-option MAC_MIN_VERSION=$(OSX_MIN_VERSION) $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) +$(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif -$(package)_config_opts_linux = -qt-xkbcommon-x11 -$(package)_config_opts_linux += -qt-xcb +ifneq ($(build_arch),$(host_arch)) +$(package)_config_opts_aarch64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=arm64 +$(package)_config_opts_x86_64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=x86_64 +endif + +$(package)_config_opts_linux = -qt-xcb $(package)_config_opts_linux += -no-xcb-xlib $(package)_config_opts_linux += -no-feature-xlib $(package)_config_opts_linux += -system-freetype $(package)_config_opts_linux += -fontconfig $(package)_config_opts_linux += -no-opengl +$(package)_config_opts_linux += -no-feature-vulkan $(package)_config_opts_linux += -dbus-runtime $(package)_config_opts_arm_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_i686_linux = -xplatform linux-g++-32 @@ -139,8 +149,11 @@ $(package)_config_opts_s390x_linux = -platform linux-g++ -xplatform bitcoin-linu $(package)_config_opts_mingw32 = -no-opengl $(package)_config_opts_mingw32 += -no-dbus $(package)_config_opts_mingw32 += -xplatform win32-g++ +$(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" +$(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" +$(package)_config_opts_mingw32 += "QMAKE_LFLAGS = '$($(package)_ldflags)'" $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" -$(package)_config_opts_mingw32 += -ssl -openssl-runtime +$(package)_config_opts_mingw32 += -pch $(package)_config_opts_android = -xplatform android-clang $(package)_config_opts_android += -android-sdk $(ANDROID_SDK) @@ -156,14 +169,13 @@ $(package)_config_opts_android += -qt-freetype $(package)_config_opts_android += -no-fontconfig $(package)_config_opts_android += -L $(host_prefix)/lib $(package)_config_opts_android += -I $(host_prefix)/include +$(package)_config_opts_android += -pch +$(package)_config_opts_android += -no-feature-vulkan $(package)_config_opts_aarch64_android += -android-arch arm64-v8a $(package)_config_opts_armv7a_android += -android-arch armeabi-v7a $(package)_config_opts_x86_64_android += -android-arch x86_64 $(package)_config_opts_i686_android += -android-arch i686 - -$(package)_build_env = QT_RCC_TEST=1 -$(package)_build_env += QT_RCC_SOURCE_DATE_OVERRIDE=1 endef define $(package)_fetch_cmds @@ -186,73 +198,72 @@ define $(package)_extract_cmds tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools endef +# Preprocessing steps work as follows: +# +# 1. Apply our patches to the extracted source. See each patch for more info. +# +# 2. Create a macOS-Clang-Linux mkspec using our mac-qmake.conf. +# +# 3. After making a copy of the mkspec for the linux-arm-gnueabi host, named +# bitcoin-linux-g++, replace instances of linux-arm-gnueabi with $(host). This +# way we can generically support hosts like riscv64-linux-gnu, which Qt doesn't +# ship a mkspec for. See it's usage in config_opts_* above. +# +# 4. Put our C, CXX and LD FLAGS into gcc-base.conf. Only used for non-host builds. +# +# 5. Do similar for the win32-g++ mkspec. +# +# 6. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. +# +# 7. Adjust a regex in toolchain.prf, to accommodate Guix's usage of +# CROSS_LIBRARY_PATH. See #15277. define $(package)_preprocess_cmds - sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \ - sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ - sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \ - sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \ - sed -i.old -e 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' -e 's|/bin/pwd|pwd|' qtbase/configure && \ + cp $($(package)_patch_dir)/qt.pro qt.pro && \ + cp $($(package)_patch_dir)/qttools_src.pro qttools/src/src.pro && \ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ + patch -p1 -i $($(package)_patch_dir)/support_new_android_ndks.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_android_pch.patch && \ + patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ + patch -p1 -i $($(package)_patch_dir)/no_sdk_version_check.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ + patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ - cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ - cp -f qtbase/mkspecs/macx-clang/Info.plist.app qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \ sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ - patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_configure_mac.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_rcc_determinism.patch &&\ - patch -p1 -i $($(package)_patch_dir)/xkb-default.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch &&\ echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ - patch -p1 -i $($(package)_patch_dir)/fix_riscv64_arch.patch &&\ - patch -p1 -i $($(package)_patch_dir)/no-xlib.patch &&\ - echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf &&\ - echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf &&\ - sed -i.old "s|QMAKE_CFLAGS += |!host_build: QMAKE_CFLAGS = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ - sed -i.old "s|QMAKE_CXXFLAGS += |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ - sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \ - sed -i.old "s|QMAKE_CC = clang|QMAKE_CC = $($(package)_cc)|" qtbase/mkspecs/common/clang.conf && \ - sed -i.old "s|QMAKE_CXX = clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf && \ + sed -i.old "s|QMAKE_CC = \$$$$\$$$${CROSS_COMPILE}clang|QMAKE_CC = $($(package)_cc)|" qtbase/mkspecs/common/clang.conf && \ + sed -i.old "s|QMAKE_CXX = \$$$$\$$$${CROSS_COMPILE}clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf && \ sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ - ./configure $($(package)_config_opts) && \ - echo "host_build: QT_CONFIG ~= s/system-zlib/zlib" >> mkspecs/qconfig.pri && \ - echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri && \ - $(MAKE) sub-src-clean && \ - cd ../qttranslations && ../qtbase/bin/qmake qttranslations.pro -o Makefile && \ - cd translations && ../../qtbase/bin/qmake translations.pro -o Makefile && cd ../.. && \ - cd qttools/src/linguist/lrelease/ && ../../../../qtbase/bin/qmake lrelease.pro -o Makefile && \ - cd ../lupdate/ && ../../../../qtbase/bin/qmake lupdate.pro -o Makefile && cd ../../../.. + cd qtbase && \ + ./configure -top-level $($(package)_config_opts) endef define $(package)_build_cmds - $(MAKE) -C src $(addprefix sub-,$($(package)_qt_libs)) && \ - $(MAKE) -C ../qttools/src/linguist/lrelease && \ - $(MAKE) -C ../qttools/src/linguist/lupdate && \ - $(MAKE) -C ../qttranslations + $(MAKE) endef define $(package)_stage_cmds - $(MAKE) -C src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && cd .. && \ - $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install_target && \ - $(MAKE) -C qttools/src/linguist/lupdate INSTALL_ROOT=$($(package)_staging_dir) install_target && \ - $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets && \ - if `test -f qtbase/src/plugins/platforms/xcb/xcb-static/libxcb-static.a`; then \ - cp qtbase/src/plugins/platforms/xcb/xcb-static/libxcb-static.a $($(package)_staging_prefix_dir)/lib; \ - fi + $(MAKE) -C qtbase/src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && \ + $(MAKE) -C qttools/src/linguist INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_linguist_tools))) && \ + $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets endef define $(package)_postprocess_cmds rm -rf native/mkspecs/ native/lib/ lib/cmake/ && \ - rm -f lib/lib*.la lib/*.prl plugins/*/*.prl + rm -f lib/lib*.la endef diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk new file mode 100644 index 0000000000..af5e0d09c9 --- /dev/null +++ b/depends/packages/sqlite.mk @@ -0,0 +1,30 @@ +package=sqlite +$(package)_version=3320100 +$(package)_download_path=https://sqlite.org/2020/ +$(package)_file_name=sqlite-autoconf-$($(package)_version).tar.gz +$(package)_sha256_hash=486748abfb16abd8af664e3a5f03b228e5f124682b0c942e157644bf6fff7d10 + +define $(package)_set_vars +$(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking +$(package)_config_opts_linux=--with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) libsqlite3.la +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install-libLTLIBRARIES install-includeHEADERS install-pkgconfigDATA +endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index cdb77f43a0..3b7f3690a4 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -3,18 +3,21 @@ $(package)_version=4.3.1 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb -$(package)_patches=0001-fix-build-with-older-mingw64.patch 0002-disable-pthread_set_name_np.patch +$(package)_patches=remove_libstd_link.patch define $(package)_set_vars - $(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf --disable-Werror + $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf + $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci + $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking + $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic - $(package)_cxxflags=-std=c++11 + $(package)_config_opts_android=--with-pic + $(package)_cxxflags=-std=c++17 endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/0001-fix-build-with-older-mingw64.patch && \ - patch -p1 < $($(package)_patch_dir)/0002-disable-pthread_set_name_np.patch && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config + patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config endef define $(package)_config_cmds @@ -30,6 +33,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \ - rm -rf bin share -endef \ No newline at end of file + rm -rf bin share lib/*.la +endef diff --git a/depends/patches/native_biplist/sorted_list.patch b/depends/patches/native_biplist/sorted_list.patch deleted file mode 100644 index 89abdb1b71..0000000000 --- a/depends/patches/native_biplist/sorted_list.patch +++ /dev/null @@ -1,29 +0,0 @@ ---- a/biplist/__init__.py 2014-10-26 19:03:11.000000000 +0000 -+++ b/biplist/__init__.py 2016-07-19 19:30:17.663521999 +0000 -@@ -541,7 +541,7 @@ - return HashableWrapper(n) - elif isinstance(root, dict): - n = {} -- for key, value in iteritems(root): -+ for key, value in sorted(iteritems(root)): - n[self.wrapRoot(key)] = self.wrapRoot(value) - return HashableWrapper(n) - elif isinstance(root, list): -@@ -616,7 +616,7 @@ - elif isinstance(obj, dict): - size = proc_size(len(obj)) - self.incrementByteCount('dictBytes', incr=1+size) -- for key, value in iteritems(obj): -+ for key, value in sorted(iteritems(obj)): - check_key(key) - self.computeOffsets(key, asReference=True) - self.computeOffsets(value, asReference=True) -@@ -714,7 +714,7 @@ - keys = [] - values = [] - objectsToWrite = [] -- for key, value in iteritems(obj): -+ for key, value in sorted(iteritems(obj)): - keys.append(key) - values.append(value) - for key in keys: diff --git a/depends/patches/native_cdrkit/cdrkit-deterministic.patch b/depends/patches/native_cdrkit/cdrkit-deterministic.patch deleted file mode 100644 index 8ab0993dc4..0000000000 --- a/depends/patches/native_cdrkit/cdrkit-deterministic.patch +++ /dev/null @@ -1,86 +0,0 @@ ---- cdrkit-1.1.11.old/genisoimage/tree.c 2008-10-21 19:57:47.000000000 -0400 -+++ cdrkit-1.1.11/genisoimage/tree.c 2013-12-06 00:23:18.489622668 -0500 -@@ -1139,8 +1139,9 @@ - scan_directory_tree(struct directory *this_dir, char *path, - struct directory_entry *de) - { -- DIR *current_dir; -+ int current_file; - char whole_path[PATH_MAX]; -+ struct dirent **d_list; - struct dirent *d_entry; - struct directory *parent; - int dflag; -@@ -1164,7 +1165,8 @@ - this_dir->dir_flags |= DIR_WAS_SCANNED; - - errno = 0; /* Paranoia */ -- current_dir = opendir(path); -+ //current_dir = opendir(path); -+ current_file = scandir(path, &d_list, NULL, alphasort); - d_entry = NULL; - - /* -@@ -1173,12 +1175,12 @@ - */ - old_path = path; - -- if (current_dir) { -+ if (current_file >= 0) { - errno = 0; -- d_entry = readdir(current_dir); -+ d_entry = d_list[0]; - } - -- if (!current_dir || !d_entry) { -+ if (current_file < 0 || !d_entry) { - int ret = 1; - - #ifdef USE_LIBSCHILY -@@ -1191,8 +1193,8 @@ - de->isorec.flags[0] &= ~ISO_DIRECTORY; - ret = 0; - } -- if (current_dir) -- closedir(current_dir); -+ if(d_list) -+ free(d_list); - return (ret); - } - #ifdef ABORT_DEEP_ISO_ONLY -@@ -1208,7 +1210,7 @@ - errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n"); - errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n"); - } -- closedir(current_dir); -+ free(d_list); - return (1); - } - #endif -@@ -1250,13 +1252,13 @@ - * The first time through, skip this, since we already asked - * for the first entry when we opened the directory. - */ -- if (dflag) -- d_entry = readdir(current_dir); -+ if (dflag && current_file >= 0) -+ d_entry = d_list[current_file]; - dflag++; - -- if (!d_entry) -+ if (current_file < 0) - break; -- -+ current_file--; - /* OK, got a valid entry */ - - /* If we do not want all files, then pitch the backups. */ -@@ -1348,7 +1350,7 @@ - insert_file_entry(this_dir, whole_path, d_entry->d_name); - #endif /* APPLE_HYB */ - } -- closedir(current_dir); -+ free(d_list); - - #ifdef APPLE_HYB - /* diff --git a/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch b/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch new file mode 100644 index 0000000000..f346c8f2cf --- /dev/null +++ b/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch @@ -0,0 +1,45 @@ +From 3e5fd3fb56bc9ff03beb535979e33dcf83fe1f70 Mon Sep 17 00:00:00 2001 +From: Cory Fields +Date: Thu, 8 May 2014 12:39:42 -0400 +Subject: [PATCH] dmg: remove libcrypto dependency + +--- + dmg/CMakeLists.txt | 16 ---------------- + 1 file changed, 16 deletions(-) + +diff --git a/dmg/CMakeLists.txt b/dmg/CMakeLists.txt +index eec62d6..3969f64 100644 +--- a/dmg/CMakeLists.txt ++++ b/dmg/CMakeLists.txt +@@ -1,12 +1,5 @@ +-INCLUDE(FindOpenSSL) + INCLUDE(FindZLIB) + +-FIND_LIBRARY(CRYPTO_LIBRARIES crypto +- PATHS +- /usr/lib +- /usr/local/lib +- ) +- + IF(NOT ZLIB_FOUND) + message(FATAL_ERROR "zlib is required for dmg!") + ENDIF(NOT ZLIB_FOUND) +@@ -18,15 +11,6 @@ link_directories(${PROJECT_BINARY_DIR}/common ${PROJECT_BINARY_DIR}/hfs) + + add_library(dmg adc.c base64.c checksum.c dmgfile.c dmglib.c filevault.c io.c partition.c resources.c udif.c) + +-IF(OPENSSL_FOUND) +- add_definitions(-DHAVE_CRYPT) +- include_directories(${OPENSSL_INCLUDE_DIR}) +- target_link_libraries(dmg ${CRYPTO_LIBRARIES}) +- IF(WIN32) +- TARGET_LINK_LIBRARIES(dmg gdi32) +- ENDIF(WIN32) +-ENDIF(OPENSSL_FOUND) +- + target_link_libraries(dmg common hfs z) + + add_executable(dmg-bin dmg.c) +-- +2.22.0 + diff --git a/depends/patches/qt/dont_hardcode_pwd.patch b/depends/patches/qt/dont_hardcode_pwd.patch new file mode 100644 index 0000000000..a74e9cb098 --- /dev/null +++ b/depends/patches/qt/dont_hardcode_pwd.patch @@ -0,0 +1,27 @@ +commit 0e953866fc4672486e29e1ba6d83b4207e7b2f0b +Author: fanquake +Date: Tue Aug 18 15:09:06 2020 +0800 + + Don't hardcode pwd path + + Let a man use his builtins if he wants to! Also, removes the unnecessary + assumption that pwd lives under /bin/pwd. + + See #15581. + +diff --git a/qtbase/configure b/qtbase/configure +index 08b49a8d..faea5b55 100755 +--- a/qtbase/configure ++++ b/qtbase/configure +@@ -36,9 +36,9 @@ + relconf=`basename $0` + # the directory of this script is the "source tree" + relpath=`dirname $0` +-relpath=`(cd "$relpath"; /bin/pwd)` ++relpath=`(cd "$relpath"; pwd)` + # the current directory is the "build tree" or "object tree" +-outpath=`/bin/pwd` ++outpath=`pwd` + + WHICH="which" + diff --git a/depends/patches/qt/dont_hardcode_x86_64.patch b/depends/patches/qt/dont_hardcode_x86_64.patch new file mode 100644 index 0000000000..0e1ca6acda --- /dev/null +++ b/depends/patches/qt/dont_hardcode_x86_64.patch @@ -0,0 +1,123 @@ +macOS: Don't hard-code x86_64 as the architecture when using qmake + +Upstream commit: + - Qt 6.1: 9082cc8e8d5a6441dabe5e7a95bc0cd9085b95fe + +For other Qt branches see +https://codereview.qt-project.org/q/I70db7e4c27f0d3da5d0af33cb491d72c312d3fa8 + + +--- old/qtbase/configure.json ++++ new/qtbase/configure.json +@@ -208,11 +208,18 @@ + + "testTypeDependencies": { + "linkerSupportsFlag": [ "use_gold_linker" ], +- "verifySpec": [ "shared", "use_gold_linker", "compiler-flags", "qmakeargs", "commit" ], ++ "verifySpec": [ ++ "shared", ++ "use_gold_linker", ++ "compiler-flags", "qmakeargs", ++ "simulator_and_device", ++ "thread", ++ "commit" ], + "compile": [ "verifyspec" ], + "detectPkgConfig": [ "cross_compile", "machineTuple" ], + "library": [ "pkg-config", "compiler-flags" ], +- "getPkgConfigVariable": [ "pkg-config" ] ++ "getPkgConfigVariable": [ "pkg-config" ], ++ "architecture" : [ "verifyspec" ] + }, + + "testTypeAliases": { +@@ -653,7 +660,7 @@ + }, + "architecture": { + "label": "Architecture", +- "output": [ "architecture" ] ++ "output": [ "architecture", "commitConfig" ] + }, + "pkg-config": { + "label": "Using pkg-config", +diff --git a/configure.pri b/configure.pri +index 33c90a8c2f..71767e29d6 100644 + +--- old/qtbase/configure.pri ++++ new/qtbase/configure.pri +@@ -642,6 +642,13 @@ defineTest(qtConfOutput_commitOptions) { + write_file($$QT_BUILD_TREE/mkspecs/qdevice.pri, $${currentConfig}.output.devicePro)|error() + } + ++# Output is written after configuring each Qt module, ++# but some tests within a module might depend on the ++# configuration output of previous tests. ++defineTest(qtConfOutput_commitConfig) { ++ qtConfProcessOutput() ++} ++ + # type (empty or 'host'), option name, default value + defineTest(processQtPath) { + out_var = config.rel_input.$${2} +diff --git a/mkspecs/common/macx.conf b/mkspecs/common/macx.conf +index 7d4a406134..de96c12fc9 100644 + +--- old/qtbase/mkspecs/common/macx.conf ++++ new/qtbase/mkspecs/common/macx.conf +@@ -6,7 +6,6 @@ QMAKE_PLATFORM += macos osx macx + QMAKE_MAC_SDK = macosx + + QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.12 +-QMAKE_APPLE_DEVICE_ARCHS = x86_64 + + QT_MAC_SDK_VERSION_MIN = 10.13 + QT_MAC_SDK_VERSION_MAX = 11.0 +diff --git a/mkspecs/features/mac/default_post.prf b/mkspecs/features/mac/default_post.prf +index d052808c14..0a89effe87 100644 + +--- old/qtbase/mkspecs/features/mac/default_post.prf ++++ new/qtbase/mkspecs/features/mac/default_post.prf +@@ -89,6 +89,11 @@ app_extension_api_only { + QMAKE_LFLAGS += $$QMAKE_CFLAGS_APPLICATION_EXTENSION + } + ++# Non-universal builds do not set QMAKE_APPLE_DEVICE_ARCHS, ++# so we pick it up from what the arch test resolved instead. ++isEmpty(QMAKE_APPLE_DEVICE_ARCHS): \ ++ QMAKE_APPLE_DEVICE_ARCHS = $$QT_ARCH ++ + macx-xcode { + qmake_pkginfo_typeinfo.name = QMAKE_PKGINFO_TYPEINFO + !isEmpty(QMAKE_PKGINFO_TYPEINFO): \ +@@ -144,9 +149,6 @@ macx-xcode { + simulator: VALID_SIMULATOR_ARCHS = $$QMAKE_APPLE_SIMULATOR_ARCHS + VALID_ARCHS = $$VALID_DEVICE_ARCHS $$VALID_SIMULATOR_ARCHS + +- isEmpty(VALID_ARCHS): \ +- error("QMAKE_APPLE_DEVICE_ARCHS or QMAKE_APPLE_SIMULATOR_ARCHS must contain at least one architecture") +- + single_arch: VALID_ARCHS = $$first(VALID_ARCHS) + + ACTIVE_ARCHS = $(filter $(EXPORT_VALID_ARCHS), $(ARCHS)) +diff --git a/mkspecs/features/toolchain.prf b/mkspecs/features/toolchain.prf +index 5003679bd0..c7c080cb07 100644 + +--- old/qtbase/mkspecs/features/toolchain.prf ++++ new/qtbase/mkspecs/features/toolchain.prf +@@ -182,9 +182,14 @@ isEmpty($${target_prefix}.INCDIRS) { + # UIKit simulator platforms will see the device SDK's sysroot in + # QMAKE_DEFAULT_*DIRS, because they're handled in a single build pass. + darwin { +- # Clang doesn't pick up the architecture from the sysroot, and will +- # default to the host architecture, so we need to manually set it. +- cxx_flags += -arch $$QMAKE_APPLE_DEVICE_ARCHS ++ uikit { ++ # Clang doesn't automatically pick up the architecture, just because ++ # we're passing the iOS sysroot below, and we will end up building the ++ # test for the host architecture, resulting in linker errors when ++ # linking against the iOS libraries. We work around this by passing ++ # the architecture explicitly. ++ cxx_flags += -arch $$first(QMAKE_APPLE_DEVICE_ARCHS) ++ } + + uikit:macx-xcode: \ + cxx_flags += -isysroot $$sdk_path_device.value diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index 2f6ff00f40..a186aeb8f6 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -890,6 +890,14 @@ +@@ -898,6 +898,14 @@ __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } diff --git a/depends/patches/qt/fix_android_pch.patch b/depends/patches/qt/fix_android_pch.patch new file mode 100644 index 0000000000..195e1c5e59 --- /dev/null +++ b/depends/patches/qt/fix_android_pch.patch @@ -0,0 +1,10 @@ +--- old/qtbase/mkspecs/common/android-base-head.conf ++++ new/qtbase/mkspecs/common/android-base-head.conf +@@ -72,6 +72,6 @@ CROSS_COMPILE = $$NDK_TOOLCHAIN_PATH/bin/$$NDK_TOOLS_PREFIX- + QMAKE_PCH_OUTPUT_EXT = .gch + + QMAKE_CFLAGS_PRECOMPILE = -x c-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} +-QMAKE_CFLAGS_USE_PRECOMPILE = -include ${QMAKE_PCH_OUTPUT_BASE} ++QMAKE_CFLAGS_USE_PRECOMPILE = -include-pch ${QMAKE_PCH_OUTPUT} + QMAKE_CXXFLAGS_PRECOMPILE = -x c++-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} + QMAKE_CXXFLAGS_USE_PRECOMPILE = $$QMAKE_CFLAGS_USE_PRECOMPILE diff --git a/depends/patches/qt/fix_android_qmake_conf.patch b/depends/patches/qt/fix_android_qmake_conf.patch deleted file mode 100644 index 13bfff9776..0000000000 --- a/depends/patches/qt/fix_android_qmake_conf.patch +++ /dev/null @@ -1,20 +0,0 @@ ---- old/qtbase/mkspecs/android-clang/qmake.conf -+++ new/qtbase/mkspecs/android-clang/qmake.conf -@@ -30,7 +30,7 @@ - QMAKE_CFLAGS += -target mips64el-none-linux-android - - QMAKE_CFLAGS += -gcc-toolchain $$NDK_TOOLCHAIN_PATH --QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a -+QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a -nostdlib++ - QMAKE_CFLAGS += -DANDROID_HAS_WSTRING --sysroot=$$NDK_ROOT/sysroot \ - -isystem $$NDK_ROOT/sysroot/usr/include/$$NDK_TOOLS_PREFIX \ - -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++/include \ -@@ -40,7 +40,7 @@ - ANDROID_SOURCES_CXX_STL_LIBDIR = $$NDK_ROOT/sources/cxx-stl/llvm-libc++/libs/$$ANDROID_TARGET_ARCH - - ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so --ANDROID_CXX_STL_LIBS = -lc++ -+ANDROID_CXX_STL_LIBS = -lc++_shared - - QMAKE_ARM_CFLAGS_RELEASE = -Oz - QMAKE_ARM_CFLAGS_RELEASE_WITH_DEBUGINFO = -g -Oz diff --git a/depends/patches/qt/fix_configure_mac.patch b/depends/patches/qt/fix_configure_mac.patch deleted file mode 100644 index 0d7dd647de..0000000000 --- a/depends/patches/qt/fix_configure_mac.patch +++ /dev/null @@ -1,50 +0,0 @@ ---- old/qtbase/mkspecs/features/mac/sdk.prf 2018-02-08 10:24:48.000000000 -0800 -+++ new/qtbase/mkspecs/features/mac/sdk.prf 2018-03-23 10:38:56.000000000 -0700 -@@ -8,21 +8,21 @@ - defineReplace(xcodeSDKInfo) { - info = $$1 - equals(info, "Path"): \ -- info = --show-sdk-path -+ infoarg = --show-sdk-path - equals(info, "PlatformPath"): \ -- info = --show-sdk-platform-path -+ infoarg = --show-sdk-platform-path - equals(info, "SDKVersion"): \ -- info = --show-sdk-version -+ infoarg = --show-sdk-version - sdk = $$2 - isEmpty(sdk): \ - sdk = $$QMAKE_MAC_SDK - - isEmpty(QMAKE_MAC_SDK.$${sdk}.$${info}) { -- QMAKE_MAC_SDK.$${sdk}.$${info} = $$system("/usr/bin/xcrun --sdk $$sdk $$info 2>/dev/null") -+ QMAKE_MAC_SDK.$${sdk}.$${info} = $$system("/usr/bin/xcrun --sdk $$sdk $$infoarg 2>/dev/null") - # --show-sdk-platform-path won't work for Command Line Tools; this is fine - # only used by the XCTest backend to testlib -- isEmpty(QMAKE_MAC_SDK.$${sdk}.$${info}):if(!isEmpty(QMAKE_XCODEBUILD_PATH)|!equals(info, "--show-sdk-platform-path")): \ -- error("Could not resolve SDK $$info for \'$$sdk\'") -+ isEmpty(QMAKE_MAC_SDK.$${sdk}.$${info}):if(!isEmpty(QMAKE_XCODEBUILD_PATH)|!equals(infoarg, "--show-sdk-platform-path")): \ -+ error("Could not resolve SDK $$info for \'$$sdk\' using $$infoarg") - cache(QMAKE_MAC_SDK.$${sdk}.$${info}, set stash, QMAKE_MAC_SDK.$${sdk}.$${info}) - } - ---- old/qtbase/configure 2018-02-08 10:24:48.000000000 -0800 -+++ new/qtbase/configure 2018-03-23 05:42:29.000000000 -0700 -@@ -232,8 +232,13 @@ - - sdk=$(getSingleQMakeVariable "QMAKE_MAC_SDK" "$1") - if [ -z "$sdk" ]; then echo "QMAKE_MAC_SDK must be set when building on Mac" >&2; exit 1; fi -- sysroot=$(/usr/bin/xcrun --sdk $sdk --show-sdk-path 2>/dev/null) -- if [ -z "$sysroot" ]; then echo "Failed to resolve SDK path for '$sdk'" >&2; exit 1; fi -+ sysroot=$(getSingleQMakeVariable "QMAKE_MAC_SDK_PATH" "$1") -+ -+ echo "sysroot pre-configured as $sysroot"; -+ if [ -z "$sysroot" ]; then -+ sysroot=$(/usr/bin/xcrun --sdk $sdk --show-sdk-path 2>/dev/null) -+ if [ -z "$sysroot" ]; then echo "Failed to resolve SDK path for '$sdk'" >&2; exit 1; fi -+ fi - - case "$sdk" in - macosx*) - - diff --git a/depends/patches/qt/fix_lib_paths.patch b/depends/patches/qt/fix_lib_paths.patch new file mode 100644 index 0000000000..d1a15373f4 --- /dev/null +++ b/depends/patches/qt/fix_lib_paths.patch @@ -0,0 +1,193 @@ +--- old/qtbase/mkspecs/common/mac.conf ++++ new/qtbase/mkspecs/common/mac.conf +@@ -14,7 +14,6 @@ + + QMAKE_RESOURCE = /Developer/Tools/Rez + QMAKE_EXTENSION_SHLIB = dylib +-QMAKE_EXTENSIONS_AUX_SHLIB = tbd + QMAKE_LIBDIR = + + # sdk.prf will prefix the proper SDK sysroot + +--- old/qtbase/mkspecs/features/qmake_use.prf ++++ new/qtbase/mkspecs/features/qmake_use.prf +@@ -22,6 +22,8 @@ + !defined(QMAKE_LIBS_$$nu, var): \ + error("Library '$$lower($$replace(nu, _, -))' is not defined.") + ++ QMAKE_LIBDIR += $$eval(QMAKE_LIBDIR_$$nu) ++ + debug: \ + LIBS$${suffix} += $$eval(QMAKE_LIBS_$${nu}_DEBUG) $$eval(QMAKE_LIBS_$$nu) + else: \ + +--- old/qtbase/mkspecs/features/qt_configure.prf ++++ new/qtbase/mkspecs/features/qt_configure.prf +@@ -526,98 +526,23 @@ + return($$sysrootified) + } + +-# libs-var, libs, in-paths, out-paths-var ++# libs-var, libs, in-paths + defineTest(qtConfResolveLibs) { +- ret = true +- paths = $$3 +- out = +- copy = false +- for (l, 2) { +- $$copy { +- copy = false +- out += $$l +- } else: equals(l, "-s") { +- # em++ flag to link libraries from emscripten-ports; passed on literally. +- copy = true +- out += $$l +- } else: contains(l, "^-L.*") { +- lp = $$replace(l, "^-L", ) +- gcc: lp = $$qtGccSysrootifiedPath($$lp) +- !exists($$lp/.) { +- qtLog("Library path $$val_escape(lp) is invalid.") +- ret = false +- } else { +- paths += $$lp +- } +- } else: contains(l, "^-l.*") { +- lib = $$replace(l, "^-l", ) +- lcan = +- integrity:contains(lib, "^.*\\.a") { +- # INTEGRITY compiler searches for exact filename +- # if -l argument has .a suffix +- lcan += $${lib} +- } else: contains(lib, "^:.*") { +- # Use exact filename when -l:filename syntax is used. +- lib ~= s/^:// +- lcan += $${lib} +- } else: unix { +- # Under UNIX, we look for actual shared libraries, in addition +- # to static ones. +- shexts = $$QMAKE_EXTENSION_SHLIB $$QMAKE_EXTENSIONS_AUX_SHLIB +- for (ext, shexts) { +- lcan += $${QMAKE_PREFIX_SHLIB}$${lib}.$${ext} +- } +- lcan += \ +- $${QMAKE_PREFIX_STATICLIB}$${lib}.$${QMAKE_EXTENSION_STATICLIB} +- } else { +- # Under Windows, we look only for static libraries, as even for DLLs +- # one actually links against a static import library. +- mingw { +- lcan += \ +- # MinGW supports UNIX-style library naming in addition to +- # the MSVC style. +- lib$${lib}.dll.a lib$${lib}.a \ +- # Fun fact: prefix-less libraries are also supported. +- $${lib}.dll.a $${lib}.a +- } +- lcan += $${lib}.lib +- } +- l = $$qtConfFindInPathList($$lcan, $$paths $$EXTRA_LIBDIR $$QMAKE_DEFAULT_LIBDIRS) +- isEmpty(l) { +- qtLog("None of [$$val_escape(lcan)] found in [$$val_escape(paths)] and global paths.") +- ret = false +- } else { +- out += $$l +- } +- } else { +- out += $$l +- } +- } +- $$1 = $$out ++ for (path, 3): \ ++ pre_lflags += -L$$path ++ $$1 = $$pre_lflags $$2 + export($$1) +- !isEmpty(4) { +- $$4 = $$paths +- export($$4) +- } +- return($$ret) +-} +- +-# source-var +-defineTest(qtConfResolveAllLibs) { +- ret = true +- !qtConfResolveLibs($${1}.libs, $$eval($${1}.libs), , $${1}.libdirs): \ +- ret = false +- for (b, $${1}.builds._KEYS_): \ +- !qtConfResolveLibs($${1}.builds.$${b}, $$eval($${1}.builds.$${b}), $$eval($${1}.libdirs), ): \ +- ret = false +- return($$ret) ++ return(true) + } + + # libs-var, in-paths, libs + defineTest(qtConfResolvePathLibs) { + ret = true +- gcc: 2 = $$qtGccSysrootifiedPaths($$2) +- for (libdir, 2) { ++ gcc: \ ++ local_paths = $$qtGccSysrootifiedPaths($$2) ++ else: \ ++ local_paths = $$2 ++ for (libdir, local_paths) { + !exists($$libdir/.) { + qtLog("Library path $$val_escape(libdir) is invalid.") + ret = false +@@ -667,8 +592,11 @@ + # includes-var, in-paths, test-object-var + defineTest(qtConfResolvePathIncs) { + ret = true +- gcc: 2 = $$qtGccSysrootifiedPaths($$2) +- for (incdir, 2) { ++ gcc: \ ++ local_paths = $$qtGccSysrootifiedPaths($$2) ++ else: \ ++ local_paths = $$2 ++ for (incdir, local_paths) { + !exists($$incdir/.) { + qtLog("Include path $$val_escape(incdir) is invalid.") + ret = false +@@ -727,6 +655,7 @@ + vars += $$eval(config.commandline.rev_assignments.$${iv}) + defined(config.input.$${iv}, var) { + eval($${1}.builds.$${b} = $$eval(config.input.$${iv})) ++ export($${1}.builds.$${b}) + $${1}.builds._KEYS_ *= $${b} + any = true + } else { +@@ -741,11 +670,14 @@ + export($${1}.builds._KEYS_) + # we also reset the generic libs, to avoid surprises. + $${1}.libs = ++ export($${1}.libs) + } + + # direct libs. overwrites inline libs. +- defined(config.input.$${input}.libs, var): \ ++ defined(config.input.$${input}.libs, var) { + eval($${1}.libs = $$eval(config.input.$${input}.libs)) ++ export($${1}.libs) ++ } + + includes = $$eval(config.input.$${input}.incdir) + +@@ -754,6 +686,7 @@ + !isEmpty(prefix) { + includes += $$prefix/include + $${1}.libs = -L$$prefix/lib $$eval($${1}.libs) ++ export($${1}.libs) + } + + libdir = $$eval(config.input.$${input}.libdir) +@@ -762,11 +695,9 @@ + for (ld, libdir): \ + libs += -L$$ld + $${1}.libs = $$libs $$eval($${1}.libs) ++ export($${1}.libs) + } + +- !qtConfResolveAllLibs($$1): \ +- return(false) +- + !qtConfResolvePathIncs($${1}.includedir, $$includes, $$2): \ + return(false) + diff --git a/depends/patches/qt/fix_limits_header.patch b/depends/patches/qt/fix_limits_header.patch new file mode 100644 index 0000000000..e4313770e5 --- /dev/null +++ b/depends/patches/qt/fix_limits_header.patch @@ -0,0 +1,44 @@ +Fix compiling with GCC 11 + +See: https://bugreports.qt.io/browse/QTBUG-90395. + +Upstream commits: + - Qt 5.15 -- unavailable as open source + - Qt 6.0: b2af6332ea37e45ab230a7a5d2d278f86d961b83 + - Qt 6.1: 9c56d4da2ff631a8c1c30475bd792f6c86bda53c + +--- old/qtbase/src/corelib/global/qendian.h ++++ new/qtbase/src/corelib/global/qendian.h +@@ -44,6 +44,8 @@ + #include + #include + ++#include ++ + // include stdlib.h and hope that it defines __GLIBC__ for glibc-based systems + #include + #include + +--- old/qtbase/src/corelib/tools/qbytearraymatcher.h ++++ new/qtbase/src/corelib/tools/qbytearraymatcher.h +@@ -42,6 +42,8 @@ + + #include + ++#include ++ + QT_BEGIN_NAMESPACE + + + +--- old/qtbase/src/tools/moc/generator.cpp ++++ new/qtbase/src/tools/moc/generator.cpp +@@ -40,6 +40,8 @@ + #include + #include + ++#include ++ + #include + #include + diff --git a/depends/patches/qt/fix_montery_include.patch b/depends/patches/qt/fix_montery_include.patch new file mode 100644 index 0000000000..38b700addf --- /dev/null +++ b/depends/patches/qt/fix_montery_include.patch @@ -0,0 +1,21 @@ +From dece6f5840463ae2ddf927d65eb1b3680e34a547 +From: Øystein Heskestad +Date: Wed, 27 Oct 2021 13:07:46 +0200 +Subject: [PATCH] Add missing macOS header file that was indirectly included before + +See: https://bugreports.qt.io/browse/QTBUG-97855 + +Upstream Commits: + - Qt 6.2: c884bf138a21dd7320e35cef34d24e22e74d7ce0 + +diff --git a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +index e070ba97..07c75b04 100644 +--- a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h ++++ b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +@@ -40,6 +40,7 @@ + #ifndef QIOSURFACEGRAPHICSBUFFER_H + #define QIOSURFACEGRAPHICSBUFFER_H + ++#include + #include + #include diff --git a/depends/patches/qt/fix_no_printer.patch b/depends/patches/qt/fix_no_printer.patch index f868ca2577..1372356138 100644 --- a/depends/patches/qt/fix_no_printer.patch +++ b/depends/patches/qt/fix_no_printer.patch @@ -10,10 +10,10 @@ --- x/qtbase/src/plugins/plugins.pro +++ y/qtbase/src/plugins/plugins.pro -@@ -8,6 +8,3 @@ qtHaveModule(gui) { - qtConfig(imageformatplugin): SUBDIRS *= imageformats +@@ -9,6 +9,3 @@ qtHaveModule(gui) { !android:qtConfig(library): SUBDIRS *= generic } + qtHaveModule(widgets): SUBDIRS += styles - -!winrt:qtHaveModule(printsupport): \ - SUBDIRS += printsupport diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch index 34302a9f2d..a5de2b4b9e 100644 --- a/depends/patches/qt/fix_qt_pkgconfig.patch +++ b/depends/patches/qt/fix_qt_pkgconfig.patch @@ -1,11 +1,23 @@ --- old/qtbase/mkspecs/features/qt_module.prf +++ new/qtbase/mkspecs/features/qt_module.prf -@@ -245,7 +245,7 @@ +@@ -269,7 +269,7 @@ load(qt_installs) load(qt_targets) # this builds on top of qt_common --!internal_module:!lib_bundle:if(unix|mingw) { -+unix|mingw { +-!internal_module:if(unix|mingw) { ++if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { CONFIG += create_pc QMAKE_PKGCONFIG_DESTDIR = pkgconfig host_build: \ +@@ -284,9 +284,9 @@ load(qt_targets) + QMAKE_PKGCONFIG_CFLAGS = -D$$MODULE_DEFINE -I${includedir}/$$MODULE_INCNAME + } + QMAKE_PKGCONFIG_NAME = $$replace(TARGET, ^Qt, "Qt$$QT_MAJOR_VERSION ") +- QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION) ++ QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION)$$qtPlatformTargetSuffix() + for(i, MODULE_DEPENDS): \ +- QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0)) ++ QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0))$$qtPlatformTargetSuffix() + isEmpty(QMAKE_PKGCONFIG_DESCRIPTION): \ + QMAKE_PKGCONFIG_DESCRIPTION = $$replace(TARGET, ^Qt, "Qt ") module + !isEmpty(lib_replace0.match) { diff --git a/depends/patches/qt/fix_rcc_determinism.patch b/depends/patches/qt/fix_rcc_determinism.patch deleted file mode 100644 index c1b07fe23a..0000000000 --- a/depends/patches/qt/fix_rcc_determinism.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- old/qtbase/src/tools/rcc/rcc.cpp -+++ new/qtbase/src/tools/rcc/rcc.cpp -@@ -207,7 +207,11 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib) - if (lib.formatVersion() >= 2) { - // last modified time stamp - const QDateTime lastModified = m_fileInfo.lastModified(); -- lib.writeNumber8(quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0)); -+ quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0); -+ static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong(); -+ if (sourceDate != 0) -+ lastmod = sourceDate; -+ lib.writeNumber8(lastmod); - if (text || pass1) - lib.writeChar('\n'); - } diff --git a/depends/patches/qt/fix_riscv64_arch.patch b/depends/patches/qt/fix_riscv64_arch.patch deleted file mode 100644 index e7f29f01f9..0000000000 --- a/depends/patches/qt/fix_riscv64_arch.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/qtbase/src/3rdparty/double-conversion/include/double-conversion/utils.h b/qtbase/src/3rdparty/double-conversion/include/double-conversion/utils.h -index 20bfd36..93729fa 100644 ---- a/qtbase/src/3rdparty/double-conversion/include/double-conversion/utils.h -+++ b/qtbase/src/3rdparty/double-conversion/include/double-conversion/utils.h -@@ -65,7 +65,8 @@ - defined(__sparc__) || defined(__sparc) || defined(__s390__) || \ - defined(__SH4__) || defined(__alpha__) || \ - defined(_MIPS_ARCH_MIPS32R2) || \ -- defined(__AARCH64EL__) -+ defined(__AARCH64EL__) || defined(__aarch64__) || \ -+ defined(__riscv) - #define DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS 1 - #elif defined(_M_IX86) || defined(__i386__) || defined(__i386) - #if defined(_WIN32) diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 4cd96df29f..e4bfaa1463 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -1,20 +1,18 @@ MAKEFILE_GENERATOR = UNIX -CONFIG += app_bundle incremental global_init_link_order lib_version_first plugin_no_soname absolute_library_soname +CONFIG += app_bundle incremental lib_version_first absolute_library_soname QMAKE_INCREMENTAL_STYLE = sublib include(../common/macx.conf) include(../common/gcc-base-mac.conf) include(../common/clang.conf) include(../common/clang-mac.conf) QMAKE_MAC_SDK_PATH=$${MAC_SDK_PATH} -QMAKE_XCODE_VERSION=4.3 +QMAKE_XCODE_VERSION = $${XCODE_VERSION} QMAKE_XCODE_DEVELOPER_PATH=/Developer -QMAKE_MACOSX_DEPLOYMENT_TARGET = $${MAC_MIN_VERSION} QMAKE_MAC_SDK=macosx QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} QMAKE_MAC_SDK.macosx.PlatformPath = /phony -QMAKE_APPLE_DEVICE_ARCHS=x86_64 !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS !host_build: QMAKE_CXXFLAGS += $$QMAKE_CFLAGS diff --git a/depends/patches/qt/no-xlib.patch b/depends/patches/qt/no-xlib.patch index fe82c2c73c..f4a6f09ee4 100644 --- a/depends/patches/qt/no-xlib.patch +++ b/depends/patches/qt/no-xlib.patch @@ -22,15 +22,15 @@ index 7c62c2e2b3..c05c6c0a07 100644 #include #include -@@ -384,6 +386,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *widget) - w->setCursor(c, isBitmapCursor); +@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) + xcb_flush(xcb_connection()); } +#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) static int cursorIdForShape(int cshape) { int cursorId = 0; -@@ -437,6 +440,7 @@ static int cursorIdForShape(int cshape) +@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape) } return cursorId; } @@ -38,7 +38,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape) { -@@ -558,7 +562,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) +@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) xcb_cursor_t QXcbCursor::createFontCursor(int cshape) { xcb_connection_t *conn = xcb_connection(); @@ -48,22 +48,23 @@ index 7c62c2e2b3..c05c6c0a07 100644 xcb_cursor_t cursor = XCB_NONE; // Try Xcursor first -@@ -589,6 +595,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) +@@ -585,7 +591,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) + // Non-standard X11 cursors are created from bitmaps cursor = createNonStandardCursor(cshape); - +- +#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) // Create a glpyh cursor if everything else failed if (!cursor && cursorId) { cursor = xcb_generate_id(conn); -@@ -596,6 +603,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) +@@ -593,6 +599,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) cursorId, cursorId + 1, 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0); } +#endif if (cursor && cshape >= 0 && cshape < Qt::LastCursor && connection()->hasXFixes()) { - const char *name = cursorNames[cshape]; + const char *name = cursorNames[cshape].front(); -- 2.22.0 diff --git a/depends/patches/qt/no_sdk_version_check.patch b/depends/patches/qt/no_sdk_version_check.patch new file mode 100644 index 0000000000..b16635b572 --- /dev/null +++ b/depends/patches/qt/no_sdk_version_check.patch @@ -0,0 +1,20 @@ +commit f5eb142cd04be2bc4ca610ed3b5b7e8ce3520ee3 +Author: fanquake +Date: Tue Jan 5 16:08:49 2021 +0800 + + Don't invoke macOS SDK version checking + + This tries to use xcrun which is not available when cross-compiling. + +diff --git a/qtbase/mkspecs/features/mac/default_post.prf b/qtbase/mkspecs/features/mac/default_post.prf +index 92a9112bca6..447e186eb26 100644 +--- a/qtbase/mkspecs/features/mac/default_post.prf ++++ b/qtbase/mkspecs/features/mac/default_post.prf +@@ -8,7 +8,6 @@ contains(TEMPLATE, .*app) { + !macx-xcode:if(isEmpty(BUILDS)|build_pass) { + # Detect changes to the platform SDK + QMAKE_EXTRA_VARIABLES += QMAKE_MAC_SDK QMAKE_MAC_SDK_VERSION QMAKE_XCODE_DEVELOPER_PATH +- QMAKE_EXTRA_INCLUDES += $$shell_quote($$PWD/sdk.mk) + } + + # Detect incompatible SDK versions diff --git a/depends/patches/qt/qt.pro b/depends/patches/qt/qt.pro new file mode 100644 index 0000000000..8f2e900a84 --- /dev/null +++ b/depends/patches/qt/qt.pro @@ -0,0 +1,16 @@ +# Create the super cache so modules will add themselves to it. +cache(, super) + +!QTDIR_build: cache(CONFIG, add, $$list(QTDIR_build)) + +prl = no_install_prl +CONFIG += $$prl +cache(CONFIG, add stash, prl) + +TEMPLATE = subdirs +SUBDIRS = qtbase qttools qttranslations + +qttools.depends = qtbase +qttranslations.depends = qttools + +load(qt_configure) diff --git a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch new file mode 100644 index 0000000000..0358bea6e9 --- /dev/null +++ b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch @@ -0,0 +1,17 @@ +The moc executable loops through headers on CPLUS_INCLUDE_PATH and stumbles +on the GCC internal _GLIBCXX_VISIBILITY macro. Tell it to ignore it as it is +not supposed to be looking there to begin with. + +Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 + +diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp +--- a/qtbase/src/tools/moc/main.cpp ++++ b/qtbase/src/tools/moc/main.cpp +@@ -188,6 +188,7 @@ int runMoc(int argc, char **argv) + dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); + pp.macros["__attribute__"] = dummyVariadicFunctionMacro; + pp.macros["__declspec"] = dummyVariadicFunctionMacro; ++ pp.macros["_GLIBCXX_VISIBILITY"] = dummyVariadicFunctionMacro; + + QString filename; + QString output; diff --git a/depends/patches/qt/qttools_src.pro b/depends/patches/qt/qttools_src.pro new file mode 100644 index 0000000000..6ef71a0942 --- /dev/null +++ b/depends/patches/qt/qttools_src.pro @@ -0,0 +1,6 @@ +TEMPLATE = subdirs +SUBDIRS = linguist + +fb = force_bootstrap +CONFIG += $$fb +cache(CONFIG, add, fb) diff --git a/depends/patches/qt/support_new_android_ndks.patch b/depends/patches/qt/support_new_android_ndks.patch new file mode 100644 index 0000000000..85c8ae2132 --- /dev/null +++ b/depends/patches/qt/support_new_android_ndks.patch @@ -0,0 +1,122 @@ +Follow Google's BuildSystemMaintainers doc to support future NDK releases. + +Upstream commit: + - Qt 5.14: 9b14950ff600a4ce5a8698b67ab38907c50417f1 + +--- old/qtbase/mkspecs/android-clang/qmake.conf ++++ new/qtbase/mkspecs/android-clang/qmake.conf +@@ -14,43 +14,29 @@ + QMAKE_CC = $$NDK_LLVM_PATH/bin/clang + QMAKE_CXX = $$NDK_LLVM_PATH/bin/clang++ + ++# Follow https://android.googlesource.com/platform/ndk/+/ndk-release-r20/docs/BuildSystemMaintainers.md ++ + equals(ANDROID_TARGET_ARCH, armeabi-v7a): \ +- QMAKE_CFLAGS += -target armv7-none-linux-androideabi +-else: equals(ANDROID_TARGET_ARCH, armeabi): \ +- QMAKE_CFLAGS += -target armv5te-none-linux-androideabi ++ QMAKE_CFLAGS = -target armv7a-linux-androideabi$$replace(ANDROID_PLATFORM, "android-", "") + else: equals(ANDROID_TARGET_ARCH, arm64-v8a): \ +- QMAKE_CFLAGS += -target aarch64-none-linux-android ++ QMAKE_CFLAGS = -target aarch64-linux-android$$replace(ANDROID_PLATFORM, "android-", "") + else: equals(ANDROID_TARGET_ARCH, x86): \ +- QMAKE_CFLAGS += -target i686-none-linux-android -mstackrealign ++ QMAKE_CFLAGS = -target i686-linux-android$$replace(ANDROID_PLATFORM, "android-", "") -mstackrealign + else: equals(ANDROID_TARGET_ARCH, x86_64): \ +- QMAKE_CFLAGS += -target x86_64-none-linux-android +-else: equals(ANDROID_TARGET_ARCH, mips): \ +- QMAKE_CFLAGS += -target mipsel-none-linux-android +-else: equals(ANDROID_TARGET_ARCH, mips64): \ +- QMAKE_CFLAGS += -target mips64el-none-linux-android +- +-QMAKE_CFLAGS += -gcc-toolchain $$NDK_TOOLCHAIN_PATH -fno-limit-debug-info +- +-QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libatomic.a -nostdlib++ +-equals(ANDROID_TARGET_ARCH, armeabi-v7a): QMAKE_LINK += -Wl,--exclude-libs,libunwind.a +- +-QMAKE_CFLAGS += -DANDROID_HAS_WSTRING --sysroot=$$NDK_ROOT/sysroot \ +- -isystem $$NDK_ROOT/sysroot/usr/include/$$NDK_TOOLS_PREFIX \ +- -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++/include \ +- -isystem $$NDK_ROOT/sources/android/support/include \ +- -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++abi/include ++ QMAKE_CFLAGS = -target x86_64-linux-android$$replace(ANDROID_PLATFORM, "android-", "") + +-ANDROID_SOURCES_CXX_STL_LIBDIR = $$NDK_ROOT/sources/cxx-stl/llvm-libc++/libs/$$ANDROID_TARGET_ARCH ++QMAKE_CFLAGS += -fno-limit-debug-info + +-ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so ++QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS + +-ANDROID_USE_LLVM = true ++ANDROID_STDCPP_PATH = $$NDK_LLVM_PATH/sysroot/usr/lib/$$NDK_TOOLS_PREFIX/libc++_shared.so + +-exists($$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so): \ +- ANDROID_CXX_STL_LIBS = -lc++ +-else: \ +- ANDROID_CXX_STL_LIBS = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so.$$replace(ANDROID_PLATFORM, "android-", "") ++ANDROID_USE_LLVM = true + +-QMAKE_CFLAGS_OPTIMIZE_SIZE = -Oz ++QMAKE_CFLAGS_OPTIMIZE_SIZE = -Oz ++QMAKE_LIBDIR_POST = ++QMAKE_LFLAGS = ++QMAKE_LIBS_PRIVATE = ++ANDROID_CXX_STL_LIBS = + + include(../common/android-base-tail.conf) + +--- old/qtbase/mkspecs/common/android-base-head.conf ++++ new/qtbase/mkspecs/common/android-base-head.conf +@@ -64,7 +58,6 @@ + } + + CONFIG += $$ANDROID_PLATFORM +-QMAKE_CFLAGS = -D__ANDROID_API__=$$replace(ANDROID_PLATFORM, "android-", "") + + ANDROID_PLATFORM_ROOT_PATH = $$NDK_ROOT/platforms/$$ANDROID_PLATFORM/arch-$$ANDROID_ARCHITECTURE/ + +--- old/qtbase/mkspecs/common/android-base-tail.conf ++++ new/qtbase/mkspecs/common/android-base-tail.conf +@@ -6,22 +6,17 @@ + QMAKE_CFLAGS += -fstack-protector-strong -DANDROID + + equals(ANDROID_TARGET_ARCH, armeabi-v7a): \ +- QMAKE_CFLAGS += -march=armv7-a -mfloat-abi=softfp -mfpu=vfp -fno-builtin-memmove ++ QMAKE_CFLAGS += -march=armv7-a -mfloat-abi=softfp -mfpu=vfp + else: equals(ANDROID_TARGET_ARCH, armeabi): \ +- QMAKE_CFLAGS += -march=armv5te -mtune=xscale -msoft-float -fno-builtin-memmove +-# -fno-builtin-memmove is used to workaround https://code.google.com/p/android/issues/detail?id=81692 ++ QMAKE_CFLAGS += -march=armv5te -mtune=xscale -msoft-float + + QMAKE_CFLAGS_WARN_ON = -Wall -W + QMAKE_CFLAGS_WARN_OFF = + equals(ANDROID_TARGET_ARCH, armeabi-v7a) | equals(ANDROID_TARGET_ARCH, armeabi) { + CONFIG += optimize_size + QMAKE_CFLAGS_DEBUG = -g -marm -O0 +- equals(ANDROID_TARGET_ARCH, armeabi):if(equals(NDK_TOOLCHAIN_VERSION, 4.8)|equals(NDK_TOOLCHAIN_VERSION, 4.9)) { +- DEFINES += QT_OS_ANDROID_GCC_48_WORKAROUND +- } else { +- QMAKE_CFLAGS_RELEASE += -mthumb +- QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO += -mthumb +- } ++ QMAKE_CFLAGS_RELEASE += -mthumb ++ QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO += -mthumb + } + + QMAKE_CFLAGS_SHLIB = -fPIC +@@ -61,15 +56,12 @@ + QMAKE_RANLIB = $${CROSS_COMPILE}ranlib + + QMAKE_INCDIR_POST = +-QMAKE_LIBDIR_POST = $$ANDROID_SOURCES_CXX_STL_LIBDIR + QMAKE_INCDIR_X11 = + QMAKE_LIBDIR_X11 = + QMAKE_INCDIR_OPENGL = + QMAKE_LIBDIR_OPENGL = + + QMAKE_LINK_SHLIB = $$QMAKE_LINK +-QMAKE_LFLAGS = --sysroot=$$ANDROID_PLATFORM_ROOT_PATH +-equals(ANDROID_TARGET_ARCH, x86_64) QMAKE_LFLAGS += -L$$ANDROID_PLATFORM_ROOT_PATH/usr/lib64 + QMAKE_LFLAGS_APP = -Wl,--no-undefined -Wl,-z,noexecstack -shared + QMAKE_LFLAGS_SHLIB = -Wl,--no-undefined -Wl,-z,noexecstack -shared + QMAKE_LFLAGS_PLUGIN = $$QMAKE_LFLAGS_SHLIB diff --git a/depends/patches/qt/xkb-default.patch b/depends/patches/qt/xkb-default.patch deleted file mode 100644 index 165abf3e2e..0000000000 --- a/depends/patches/qt/xkb-default.patch +++ /dev/null @@ -1,26 +0,0 @@ ---- old/qtbase/src/gui/configure.pri 2018-06-06 17:28:10.000000000 -0400 -+++ new/qtbase/src/gui/configure.pri 2018-08-17 18:43:01.589384567 -0400 -@@ -43,18 +43,11 @@ - } - - defineTest(qtConfTest_xkbConfigRoot) { -- qtConfTest_getPkgConfigVariable($${1}): return(true) -- -- for (dir, $$list("/usr/share/X11/xkb", "/usr/local/share/X11/xkb")) { -- exists($$dir) { -- $${1}.value = $$dir -- export($${1}.value) -- $${1}.cache += value -- export($${1}.cache) -- return(true) -- } -- } -- return(false) -+ $${1}.value = "/usr/share/X11/xkb" -+ export($${1}.value) -+ $${1}.cache += value -+ export($${1}.cache) -+ return(true) - } - - defineTest(qtConfTest_qpaDefaultPlatform) { diff --git a/depends/patches/zeromq/remove_libstd_link.patch b/depends/patches/zeromq/remove_libstd_link.patch new file mode 100644 index 0000000000..ddf91e6abf --- /dev/null +++ b/depends/patches/zeromq/remove_libstd_link.patch @@ -0,0 +1,25 @@ +commit 47d4cd12a2c051815ddda78adebdb3923b260d8a +Author: fanquake +Date: Tue Aug 18 14:45:40 2020 +0800 + + Remove needless linking against libstdc++ + + This is broken for a number of reasons, including: + - g++ understands "static-libstdc++ -lstdc++" to mean "link against + whatever libstdc++ exists, probably shared", which in itself is buggy. + - another stdlib (libc++ for example) may be in use + + See #11981. + +diff --git a/src/libzmq.pc.in b/src/libzmq.pc.in +index 233bc3a..3c2bf0d 100644 +--- a/src/libzmq.pc.in ++++ b/src/libzmq.pc.in +@@ -7,6 +7,6 @@ Name: libzmq + Description: 0MQ c++ library + Version: @VERSION@ + Libs: -L${libdir} -lzmq +-Libs.private: -lstdc++ @pkg_config_libs_private@ ++Libs.private: @pkg_config_libs_private@ + Requires.private: @pkg_config_names_private@ + Cflags: -I${includedir} @pkg_config_defines@ diff --git a/share/setup.nsi.in b/share/setup.nsi.in index bd5a0fe2e1..a67c969a6b 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -2,6 +2,7 @@ Name "@PACKAGE_NAME@ (@WINDOWS_BITS@-bit)" RequestExecutionLevel highest SetCompressor /SOLID lzma +SetDateSave off # General Symbol Definitions !define REGKEY "SOFTWARE\$(^Name)" diff --git a/src/Makefile.am b/src/Makefile.am index 101bdccd54..a7ca9cdbe5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -95,7 +95,7 @@ $(TOR_LIBS): tor.timestamp fi LIBBLSSIG_INCLUDES=-I./bls-signatures/build/contrib/relic/include -I./bls-signatures/contrib/relic/include -LIBBLSSIG_LIBS=bls-signatures/build/libchiabls.a +LIBBLSSIG_LIBS=bls-signatures/build/libchiabls.la LIBBLSSIG_DEPENDS=-lgmp $(LIBBLSSIG_LIBS): $(wildcard bls-signatures/src/*.cpp) $(wildcard bls-signatures/src/*.hpp) @@ -512,6 +512,7 @@ libbitcoin_wallet_a_SOURCES = \ # crypto primitives library crypto_libbitcoin_crypto_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_CONFIG_INCLUDES) crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +crypto_libbitcoin_crypto_a_CFLAGS = $(AM_CFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ @@ -848,11 +849,10 @@ if GLIBC_BACK_COMPAT libbitcoinconsensus_la_SOURCES += compat/glibc_compat.cpp endif -libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(LIBBLSSIG_LIBS) $(LIBBLSSIG_DEPENDS) -libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) +libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) +libbitcoinconsensus_la_LIBADD = $(LIBBLSSIG_LIBS) $(LIBSECP256K1) libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include $(LIBBLSSIG_INCLUDES) -DBUILD_BITCOIN_INTERNAL libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -EXTRA_libbitcoinconsensus_la_DEPENDENCIES = $(LIBBLSSIG_LIBS) endif # @@ -882,6 +882,7 @@ CLEANFILES += tor.timestamp tor.timestamp.tmp DISTCLEANFILES = obj/build.h EXTRA_DIST = $(CTAES_DIST) +EXTRA_DIST += tor/GNUmakefile config/bitcoin-config.h: config/stamp-h1 @@ -909,10 +910,6 @@ distclean-local: ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ -.mm.o: - $(AM_V_CXX) $(OBJCXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ - $(CPPFLAGS) $(AM_CXXFLAGS) $(QT_INCLUDES) $(AM_CXXFLAGS) $(PIE_FLAGS) $(CXXFLAGS) -c -o $@ $< - check-symbols: $(bin_PROGRAMS) if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 55a86e6183..9ac869e2e8 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -404,17 +404,24 @@ RES_ICONS = \ qt/res/images/checkbox_partly_checked_light.png \ qt/res/images/checkbox_partly_checked_hover_light.png \ qt/res/images/checkbox_partly_checked_disabled_light.png \ - qt/res/images/firo_logo_white.png \ + qt/res/images/firo_logo.png \ qt/res/images/radio_normal_light.png \ qt/res/images/radio_normal_hover_light.png \ qt/res/images/radio_checked_light.png \ qt/res/images/radio_checked_hover_light.png \ qt/res/images/radio_normal_disabled_light.png \ - qt/res/images/radio_checked_disabled_light.png + qt/res/images/radio_checked_disabled_light.png \ + qt/res/icons/ext_add.png \ + qt/res/icons/ext_add_light.png RES_CSS = \ qt/res/css/firo.css +RES_FONTS = \ + qt/res/fonts/Saira_SemiCondensed-Bold.ttf \ + qt/res/fonts/SourceSansPro-Regular.ttf \ + qt/res/fonts/SourceSansPro-Bold.ttf + BITCOIN_QT_BASE_CPP = \ qt/bantablemodel.cpp \ qt/bitcoinaddressvalidator.cpp \ @@ -522,7 +529,7 @@ qt_libfiroqt_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUD qt_libfiroqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libfiroqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(PROTOBUF_PROTO) $(RES_ICONS) $(RES_IMAGES) $(RES_CSS) $(RES_MOVIES) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(PROTOBUF_PROTO) $(RES_ICONS) $(RES_IMAGES) $(RES_CSS) $(RES_MOVIES) $(RES_FONTS) nodist_qt_libfiroqt_a_SOURCES = $(QT_MOC_CPP) $(QT_MOC) $(PROTOBUF_CC) \ $(PROTOBUF_H) $(QT_QRC_CPP) $(QT_QRC_LOCALE_CPP) @@ -630,14 +637,12 @@ translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCO $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) @test -f $(RCC) @cp -f $< $(@D)/temp_$( $@ + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin_locale --format-version 1 $(@D)/temp_$( $@ @rm $(@D)/temp_$( $@ + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin --format-version 1 $< > $@ CLEAN_QT = $(nodist_qt_libfiroqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno diff --git a/src/bls-signatures/CMakeLists.txt b/src/bls-signatures/CMakeLists.txt index 5a4201c335..306b96edfd 100644 --- a/src/bls-signatures/CMakeLists.txt +++ b/src/bls-signatures/CMakeLists.txt @@ -1,5 +1,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.1.0 FATAL_ERROR) -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_POSITION_INDEPENDENT_CODE ON) IF(NOT CMAKE_BUILD_TYPE) diff --git a/src/bls-signatures/Makefile b/src/bls-signatures/Makefile index 379095679a..e390b612ee 100644 --- a/src/bls-signatures/Makefile +++ b/src/bls-signatures/Makefile @@ -1,7 +1,7 @@ -all: build/libchiabls.a +all: build/libchiabls.la -build/libchiabls.a: +build/libchiabls.la: $(AM_V_at)$(MAKE) -C ./build/ .PHONY distdir: @@ -10,13 +10,12 @@ build/libchiabls.a: clean: $(AM_V_at)$(MAKE) -C ./build/ clean - rm -f ./build/libchiabls.a + rm -f ./build/libchiabls.la ./build/.libs/libchiabls.a distclean: rm -rf ./build/ check: all - ./build/src/runtest install: - $(AM_V_at)$(MAKE) -C ./build/ install +# do nothing diff --git a/src/bls-signatures/configure-cmake.sh b/src/bls-signatures/configure-cmake.sh index 71910f2832..5d41cf9c25 100755 --- a/src/bls-signatures/configure-cmake.sh +++ b/src/bls-signatures/configure-cmake.sh @@ -14,7 +14,7 @@ esac if test "x$enable_tests" == "xyes" then - cmake_osflags="$cmake_osflags -DENABLE_TESTS=1" + cmake_osflags="$cmake_osflags -DENABLE_TESTS=0" fi cmake -Bbuild -H. -DCMAKE_INSTALL_PREFIX=$depends_prefix -DWSIZE=64 -DMULTI=PTHREAD -DARITH=gmp $cmake_osflags diff --git a/src/bls-signatures/contrib/catch/catch.hpp b/src/bls-signatures/contrib/catch/catch.hpp index aacc5601f3..0384171ae4 100644 --- a/src/bls-signatures/contrib/catch/catch.hpp +++ b/src/bls-signatures/contrib/catch/catch.hpp @@ -1,9 +1,9 @@ /* - * Catch v2.2.3 - * Generated: 2018-06-06 23:11:57.601416 + * Catch v2.13.4 + * Generated: 2020-12-29 14:48:00.116107 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly - * Copyright (c) 2018 Two Blue Cubes Ltd. All rights reserved. + * Copyright (c) 2020 Two Blue Cubes Ltd. All rights reserved. * * Distributed under the Boost Software License, Version 1.0. (See accompanying * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -14,8 +14,8 @@ #define CATCH_VERSION_MAJOR 2 -#define CATCH_VERSION_MINOR 2 -#define CATCH_VERSION_PATCH 3 +#define CATCH_VERSION_MINOR 13 +#define CATCH_VERSION_PATCH 4 #ifdef __clang__ # pragma clang system_header @@ -30,14 +30,17 @@ # pragma warning(push) # pragma warning(disable: 161 1682) # else // __ICC -# pragma clang diagnostic ignored "-Wunused-variable" # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wpadded" # pragma clang diagnostic ignored "-Wswitch-enum" # pragma clang diagnostic ignored "-Wcovered-switch-default" # endif #elif defined __GNUC__ -# pragma GCC diagnostic ignored "-Wparentheses" + // Because REQUIREs trigger GCC's -Wparentheses, and because still + // supported version of g++ have only buggy support for _Pragmas, + // Wparentheses have to be suppressed globally. +# pragma GCC diagnostic ignored "-Wparentheses" // See #674 for details + # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wunused-variable" # pragma GCC diagnostic ignored "-Wpadded" @@ -55,7 +58,9 @@ # if defined(CATCH_CONFIG_DISABLE_MATCHERS) # undef CATCH_CONFIG_DISABLE_MATCHERS # endif -# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER) +# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +# endif #endif #if !defined(CATCH_CONFIG_IMPL_ONLY) @@ -104,6 +109,7 @@ namespace Catch { // CATCH_CONFIG_COUNTER : is the __COUNTER__ macro supported? // CATCH_CONFIG_WINDOWS_SEH : is Windows SEH supported? // CATCH_CONFIG_POSIX_SIGNALS : are POSIX signals supported? +// CATCH_CONFIG_DISABLE_EXCEPTIONS : Are exceptions enabled? // **************** // Note to maintainers: if new toggles are added please document them // in configuration.md, too @@ -116,34 +122,61 @@ namespace Catch { #ifdef __cplusplus -# if __cplusplus >= 201402L +# if (__cplusplus >= 201402L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201402L) # define CATCH_CPP14_OR_GREATER # endif -# if __cplusplus >= 201703L +# if (__cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) # define CATCH_CPP17_OR_GREATER # endif #endif -#if defined(CATCH_CPP17_OR_GREATER) -# define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS +// We have to avoid both ICC and Clang, because they try to mask themselves +// as gcc, and we want only GCC in this block +#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) && !defined(__CUDACC__) +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic push" ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic pop" ) + +# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) (void)__builtin_constant_p(__VA_ARGS__) + #endif -#ifdef __clang__ +#if defined(__clang__) + +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic push" ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic pop" ) + +// As of this writing, IBM XL's implementation of __builtin_constant_p has a bug +// which results in calls to destructors being emitted for each temporary, +// without a matching initialization. In practice, this can result in something +// like `std::string::~string` being called on an uninitialized value. +// +// For example, this code will likely segfault under IBM XL: +// ``` +// REQUIRE(std::string("12") + "34" == "1234") +// ``` +// +// Therefore, `CATCH_INTERNAL_IGNORE_BUT_WARN` is not implemented. +# if !defined(__ibmxl__) && !defined(__CUDACC__) +# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) (void)__builtin_constant_p(__VA_ARGS__) /* NOLINT(cppcoreguidelines-pro-type-vararg, hicpp-vararg) */ +# endif + +# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) \ + _Pragma( "clang diagnostic ignored \"-Wglobal-constructors\"") + +# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) + +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wunused-variable\"" ) -# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ - _Pragma( "clang diagnostic push" ) \ - _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) \ - _Pragma( "clang diagnostic ignored \"-Wglobal-constructors\"") -# define CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ - _Pragma( "clang diagnostic pop" ) +# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"" ) -# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ - _Pragma( "clang diagnostic push" ) \ - _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) -# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS \ - _Pragma( "clang diagnostic pop" ) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS \ + _Pragma( "clang diagnostic ignored \"-Wunused-template\"" ) #endif // __clang__ @@ -168,6 +201,7 @@ namespace Catch { // Android somehow still does not support std::to_string #if defined(__ANDROID__) # define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING +# define CATCH_INTERNAL_CONFIG_ANDROID_LOGWRITE #endif //////////////////////////////////////////////////////////////////////////////// @@ -176,6 +210,12 @@ namespace Catch { # define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH #endif +//////////////////////////////////////////////////////////////////////////////// +// PS4 +#if defined(__ORBIS__) +# define CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE +#endif + //////////////////////////////////////////////////////////////////////////////// // Cygwin #ifdef __CYGWIN__ @@ -183,16 +223,22 @@ namespace Catch { // Required for some versions of Cygwin to declare gettimeofday // see: http://stackoverflow.com/questions/36901803/gettimeofday-not-declared-in-this-scope-cygwin # define _BSD_SOURCE +// some versions of cygwin (most) do not support std::to_string. Use the libstd check. +// https://gcc.gnu.org/onlinedocs/gcc-4.8.2/libstdc++/api/a01053_source.html line 2812-2813 +# if !((__cplusplus >= 201103L) && defined(_GLIBCXX_USE_C99) \ + && !defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)) + +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING +# endif #endif // __CYGWIN__ //////////////////////////////////////////////////////////////////////////////// // Visual C++ -#ifdef _MSC_VER +#if defined(_MSC_VER) -# if _MSC_VER >= 1900 // Visual Studio 2015 or newer -# define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS -# endif +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION __pragma( warning(push) ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION __pragma( warning(pop) ) // Universal Windows platform does not support SEH // Or console colours (or console at all...) @@ -202,15 +248,40 @@ namespace Catch { # define CATCH_INTERNAL_CONFIG_WINDOWS_SEH # endif +// MSVC traditional preprocessor needs some workaround for __VA_ARGS__ +// _MSVC_TRADITIONAL == 0 means new conformant preprocessor +// _MSVC_TRADITIONAL == 1 means old traditional non-conformant preprocessor +# if !defined(__clang__) // Handle Clang masquerading for msvc +# if !defined(_MSVC_TRADITIONAL) || (defined(_MSVC_TRADITIONAL) && _MSVC_TRADITIONAL) +# define CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +# endif // MSVC_TRADITIONAL +# endif // __clang__ + +#endif // _MSC_VER + +#if defined(_REENTRANT) || defined(_MSC_VER) +// Enable async processing, as -pthread is specified or no additional linking is required +# define CATCH_INTERNAL_CONFIG_USE_ASYNC #endif // _MSC_VER //////////////////////////////////////////////////////////////////////////////// +// Check if we are compiled with -fno-exceptions or equivalent +#if defined(__EXCEPTIONS) || defined(__cpp_exceptions) || defined(_CPPUNWIND) +# define CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED +#endif +//////////////////////////////////////////////////////////////////////////////// // DJGPP #ifdef __DJGPP__ # define CATCH_INTERNAL_CONFIG_NO_WCHAR #endif // __DJGPP__ +//////////////////////////////////////////////////////////////////////////////// +// Embarcadero C++Build +#if defined(__BORLANDC__) + #define CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN +#endif + //////////////////////////////////////////////////////////////////////////////// // Use of __COUNTER__ is suppressed during code analysis in @@ -222,6 +293,58 @@ namespace Catch { #define CATCH_INTERNAL_CONFIG_COUNTER #endif +//////////////////////////////////////////////////////////////////////////////// + +// RTX is a special version of Windows that is real time. +// This means that it is detected as Windows, but does not provide +// the same set of capabilities as real Windows does. +#if defined(UNDER_RTSS) || defined(RTX64_BUILD) + #define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH + #define CATCH_INTERNAL_CONFIG_NO_ASYNC + #define CATCH_CONFIG_COLOUR_NONE +#endif + +#if !defined(_GLIBCXX_USE_C99_MATH_TR1) +#define CATCH_INTERNAL_CONFIG_GLOBAL_NEXTAFTER +#endif + +// Various stdlib support checks that require __has_include +#if defined(__has_include) + // Check if string_view is available and usable + #if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # define CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW + #endif + + // Check if optional is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # define CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) + + // Check if byte is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # include + # if __cpp_lib_byte > 0 + # define CATCH_INTERNAL_CONFIG_CPP17_BYTE + # endif + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) + + // Check if variant is available and usable + # if __has_include() && defined(CATCH_CPP17_OR_GREATER) + # if defined(__clang__) && (__clang_major__ < 8) + // work around clang bug with libstdc++ https://bugs.llvm.org/show_bug.cgi?id=31852 + // fix should be in clang 8, workaround in libstdc++ 8.2 + # include + # if defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) + # define CATCH_CONFIG_NO_CPP17_VARIANT + # else + # define CATCH_INTERNAL_CONFIG_CPP17_VARIANT + # endif // defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) + # else + # define CATCH_INTERNAL_CONFIG_CPP17_VARIANT + # endif // defined(__clang__) && (__clang_major__ < 8) + # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) +#endif // defined(__has_include) + #if defined(CATCH_INTERNAL_CONFIG_COUNTER) && !defined(CATCH_CONFIG_NO_COUNTER) && !defined(CATCH_CONFIG_COUNTER) # define CATCH_CONFIG_COUNTER #endif @@ -241,17 +364,99 @@ namespace Catch { # define CATCH_CONFIG_CPP11_TO_STRING #endif -#if defined(CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) && !defined(CATCH_CONFIG_NO_CPP17_UNCAUGHT_EXCEPTIONS) && !defined(CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) -# define CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS +#if defined(CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_NO_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_CPP17_OPTIONAL) +# define CATCH_CONFIG_CPP17_OPTIONAL +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_NO_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_CPP17_STRING_VIEW) +# define CATCH_CONFIG_CPP17_STRING_VIEW +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_VARIANT) && !defined(CATCH_CONFIG_NO_CPP17_VARIANT) && !defined(CATCH_CONFIG_CPP17_VARIANT) +# define CATCH_CONFIG_CPP17_VARIANT +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_BYTE) && !defined(CATCH_CONFIG_NO_CPP17_BYTE) && !defined(CATCH_CONFIG_CPP17_BYTE) +# define CATCH_CONFIG_CPP17_BYTE +#endif + +#if defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) +# define CATCH_INTERNAL_CONFIG_NEW_CAPTURE +#endif + +#if defined(CATCH_INTERNAL_CONFIG_NEW_CAPTURE) && !defined(CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NEW_CAPTURE) +# define CATCH_CONFIG_NEW_CAPTURE +#endif + +#if !defined(CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED) && !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) +# define CATCH_CONFIG_DISABLE_EXCEPTIONS +#endif + +#if defined(CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_NO_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_POLYFILL_ISNAN) +# define CATCH_CONFIG_POLYFILL_ISNAN +#endif + +#if defined(CATCH_INTERNAL_CONFIG_USE_ASYNC) && !defined(CATCH_INTERNAL_CONFIG_NO_ASYNC) && !defined(CATCH_CONFIG_NO_USE_ASYNC) && !defined(CATCH_CONFIG_USE_ASYNC) +# define CATCH_CONFIG_USE_ASYNC +#endif + +#if defined(CATCH_INTERNAL_CONFIG_ANDROID_LOGWRITE) && !defined(CATCH_CONFIG_NO_ANDROID_LOGWRITE) && !defined(CATCH_CONFIG_ANDROID_LOGWRITE) +# define CATCH_CONFIG_ANDROID_LOGWRITE +#endif + +#if defined(CATCH_INTERNAL_CONFIG_GLOBAL_NEXTAFTER) && !defined(CATCH_CONFIG_NO_GLOBAL_NEXTAFTER) && !defined(CATCH_CONFIG_GLOBAL_NEXTAFTER) +# define CATCH_CONFIG_GLOBAL_NEXTAFTER #endif +// Even if we do not think the compiler has that warning, we still have +// to provide a macro that can be used by the code. +#if !defined(CATCH_INTERNAL_START_WARNINGS_SUPPRESSION) +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION +#endif +#if !defined(CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION +#endif #if !defined(CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS) # define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS -# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS #endif #if !defined(CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS) # define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS -# define CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS +#endif + +// The goal of this macro is to avoid evaluation of the arguments, but +// still have the compiler warn on problems inside... +#if !defined(CATCH_INTERNAL_IGNORE_BUT_WARN) +# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) +#endif + +#if defined(__APPLE__) && defined(__apple_build_version__) && (__clang_major__ < 10) +# undef CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS +#elif defined(__clang__) && (__clang_major__ < 5) +# undef CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS +#endif + +#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS +#endif + +#if defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) +#define CATCH_TRY if ((true)) +#define CATCH_CATCH_ALL if ((false)) +#define CATCH_CATCH_ANON(type) if ((false)) +#else +#define CATCH_TRY try +#define CATCH_CATCH_ALL catch (...) +#define CATCH_CATCH_ANON(type) catch (type) +#endif + +#if defined(CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_NO_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) +#define CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #endif // end catch_compiler_capabilities.h @@ -267,6 +472,10 @@ namespace Catch { #include #include +// We need a dummy global operator<< so we can bring it into Catch namespace later +struct Catch_global_namespace_dummy {}; +std::ostream& operator<<(std::ostream&, Catch_global_namespace_dummy); + namespace Catch { struct CaseSensitive { enum Choice { @@ -293,12 +502,12 @@ namespace Catch { line( _line ) {} - SourceLineInfo( SourceLineInfo const& other ) = default; - SourceLineInfo( SourceLineInfo && ) = default; - SourceLineInfo& operator = ( SourceLineInfo const& ) = default; - SourceLineInfo& operator = ( SourceLineInfo && ) = default; + SourceLineInfo( SourceLineInfo const& other ) = default; + SourceLineInfo& operator = ( SourceLineInfo const& ) = default; + SourceLineInfo( SourceLineInfo&& ) noexcept = default; + SourceLineInfo& operator = ( SourceLineInfo&& ) noexcept = default; - bool empty() const noexcept; + bool empty() const noexcept { return file[0] == '\0'; } bool operator == ( SourceLineInfo const& other ) const noexcept; bool operator < ( SourceLineInfo const& other ) const noexcept; @@ -308,6 +517,11 @@ namespace Catch { std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ); + // Bring in operator<< from global namespace into Catch namespace + // This is necessary because the overload of operator<< above makes + // lookup stop at namespace Catch + using ::operator<<; + // Use this in variadic streaming macros to allow // >> +StreamEndStop // as well as @@ -334,9 +548,10 @@ namespace Catch { } // end namespace Catch #define CATCH_REGISTER_TAG_ALIAS( alias, spec ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ namespace{ Catch::RegistrarForTagAliases INTERNAL_CATCH_UNIQUE_NAME( AutoRegisterTagAlias )( alias, spec, CATCH_INTERNAL_LINEINFO ); } \ - CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION // end catch_tag_alias_autoregistrar.h // start catch_test_registry.h @@ -344,7 +559,6 @@ namespace Catch { // start catch_interfaces_testcase.h #include -#include namespace Catch { @@ -355,8 +569,6 @@ namespace Catch { virtual ~ITestInvoker(); }; - using ITestCasePtr = std::shared_ptr; - class TestCase; struct IConfig; @@ -366,6 +578,7 @@ namespace Catch { virtual std::vector const& getAllTestsSorted( IConfig const& config ) const = 0; }; + bool isThrowSafe( TestCase const& testCase, IConfig const& config ); bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ); std::vector const& getAllTestCasesSorted( IConfig const& config ); @@ -378,55 +591,30 @@ namespace Catch { #include #include #include +#include namespace Catch { - class StringData; - /// A non-owning string class (similar to the forthcoming std::string_view) /// Note that, because a StringRef may be a substring of another string, - /// it may not be null terminated. c_str() must return a null terminated - /// string, however, and so the StringRef will internally take ownership - /// (taking a copy), if necessary. In theory this ownership is not externally - /// visible - but it does mean (substring) StringRefs should not be shared between - /// threads. + /// it may not be null terminated. class StringRef { public: using size_type = std::size_t; + using const_iterator = const char*; private: - friend struct StringRefTestAccess; - - char const* m_start; - size_type m_size; - - char* m_data = nullptr; - - void takeOwnership(); - static constexpr char const* const s_empty = ""; - public: // construction/ assignment - StringRef() noexcept - : StringRef( s_empty, 0 ) - {} - - StringRef( StringRef const& other ) noexcept - : m_start( other.m_start ), - m_size( other.m_size ) - {} + char const* m_start = s_empty; + size_type m_size = 0; - StringRef( StringRef&& other ) noexcept - : m_start( other.m_start ), - m_size( other.m_size ), - m_data( other.m_data ) - { - other.m_data = nullptr; - } + public: // construction + constexpr StringRef() noexcept = default; StringRef( char const* rawChars ) noexcept; - StringRef( char const* rawChars, size_type size ) noexcept + constexpr StringRef( char const* rawChars, size_type size ) noexcept : m_start( rawChars ), m_size( size ) {} @@ -436,65 +624,333 @@ namespace Catch { m_size( stdString.size() ) {} - ~StringRef() noexcept { - delete[] m_data; - } - - auto operator = ( StringRef const &other ) noexcept -> StringRef& { - delete[] m_data; - m_data = nullptr; - m_start = other.m_start; - m_size = other.m_size; - return *this; + explicit operator std::string() const { + return std::string(m_start, m_size); } - operator std::string() const; - - void swap( StringRef& other ) noexcept; - public: // operators auto operator == ( StringRef const& other ) const noexcept -> bool; - auto operator != ( StringRef const& other ) const noexcept -> bool; + auto operator != (StringRef const& other) const noexcept -> bool { + return !(*this == other); + } - auto operator[] ( size_type index ) const noexcept -> char; + auto operator[] ( size_type index ) const noexcept -> char { + assert(index < m_size); + return m_start[index]; + } public: // named queries - auto empty() const noexcept -> bool { + constexpr auto empty() const noexcept -> bool { return m_size == 0; } - auto size() const noexcept -> size_type { + constexpr auto size() const noexcept -> size_type { return m_size; } - auto numberOfCharacters() const noexcept -> size_type; + // Returns the current start pointer. If the StringRef is not + // null-terminated, throws std::domain_exception auto c_str() const -> char const*; public: // substrings and searches - auto substr( size_type start, size_type size ) const noexcept -> StringRef; + // Returns a substring of [start, start + length). + // If start + length > size(), then the substring is [start, size()). + // If start > size(), then the substring is empty. + auto substr( size_type start, size_type length ) const noexcept -> StringRef; - // Returns the current start pointer. - // Note that the pointer can change when if the StringRef is a substring - auto currentData() const noexcept -> char const*; + // Returns the current start pointer. May not be null-terminated. + auto data() const noexcept -> char const*; - private: // ownership queries - may not be consistent between calls - auto isOwned() const noexcept -> bool; - auto isSubstring() const noexcept -> bool; - }; + constexpr auto isNullTerminated() const noexcept -> bool { + return m_start[m_size] == '\0'; + } - auto operator + ( StringRef const& lhs, StringRef const& rhs ) -> std::string; - auto operator + ( StringRef const& lhs, char const* rhs ) -> std::string; - auto operator + ( char const* lhs, StringRef const& rhs ) -> std::string; + public: // iterators + constexpr const_iterator begin() const { return m_start; } + constexpr const_iterator end() const { return m_start + m_size; } + }; auto operator += ( std::string& lhs, StringRef const& sr ) -> std::string&; auto operator << ( std::ostream& os, StringRef const& sr ) -> std::ostream&; - inline auto operator "" _sr( char const* rawChars, std::size_t size ) noexcept -> StringRef { + constexpr auto operator "" _sr( char const* rawChars, std::size_t size ) noexcept -> StringRef { return StringRef( rawChars, size ); } - } // namespace Catch +constexpr auto operator "" _catch_sr( char const* rawChars, std::size_t size ) noexcept -> Catch::StringRef { + return Catch::StringRef( rawChars, size ); +} + // end catch_stringref.h +// start catch_preprocessor.hpp + + +#define CATCH_RECURSION_LEVEL0(...) __VA_ARGS__ +#define CATCH_RECURSION_LEVEL1(...) CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL2(...) CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL3(...) CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL4(...) CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL5(...) CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(__VA_ARGS__))) + +#ifdef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_EXPAND_VARGS(...) __VA_ARGS__ +// MSVC needs more evaluations +#define CATCH_RECURSION_LEVEL6(...) CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(__VA_ARGS__))) +#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL6(CATCH_RECURSION_LEVEL6(__VA_ARGS__)) +#else +#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL5(__VA_ARGS__) +#endif + +#define CATCH_REC_END(...) +#define CATCH_REC_OUT + +#define CATCH_EMPTY() +#define CATCH_DEFER(id) id CATCH_EMPTY() + +#define CATCH_REC_GET_END2() 0, CATCH_REC_END +#define CATCH_REC_GET_END1(...) CATCH_REC_GET_END2 +#define CATCH_REC_GET_END(...) CATCH_REC_GET_END1 +#define CATCH_REC_NEXT0(test, next, ...) next CATCH_REC_OUT +#define CATCH_REC_NEXT1(test, next) CATCH_DEFER ( CATCH_REC_NEXT0 ) ( test, next, 0) +#define CATCH_REC_NEXT(test, next) CATCH_REC_NEXT1(CATCH_REC_GET_END test, next) + +#define CATCH_REC_LIST0(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST1(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0) ) ( f, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST2(f, x, peek, ...) f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) + +#define CATCH_REC_LIST0_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST1_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0_UD) ) ( f, userdata, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST2_UD(f, userdata, x, peek, ...) f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) + +// Applies the function macro `f` to each of the remaining parameters, inserts commas between the results, +// and passes userdata as the first parameter to each invocation, +// e.g. CATCH_REC_LIST_UD(f, x, a, b, c) evaluates to f(x, a), f(x, b), f(x, c) +#define CATCH_REC_LIST_UD(f, userdata, ...) CATCH_RECURSE(CATCH_REC_LIST2_UD(f, userdata, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) + +#define CATCH_REC_LIST(f, ...) CATCH_RECURSE(CATCH_REC_LIST2(f, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) + +#define INTERNAL_CATCH_EXPAND1(param) INTERNAL_CATCH_EXPAND2(param) +#define INTERNAL_CATCH_EXPAND2(...) INTERNAL_CATCH_NO## __VA_ARGS__ +#define INTERNAL_CATCH_DEF(...) INTERNAL_CATCH_DEF __VA_ARGS__ +#define INTERNAL_CATCH_NOINTERNAL_CATCH_DEF +#define INTERNAL_CATCH_STRINGIZE(...) INTERNAL_CATCH_STRINGIZE2(__VA_ARGS__) +#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_STRINGIZE2(...) #__VA_ARGS__ +#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) +#else +// MSVC is adding extra space and needs another indirection to expand INTERNAL_CATCH_NOINTERNAL_CATCH_DEF +#define INTERNAL_CATCH_STRINGIZE2(...) INTERNAL_CATCH_STRINGIZE3(__VA_ARGS__) +#define INTERNAL_CATCH_STRINGIZE3(...) #__VA_ARGS__ +#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) (INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) + 1) +#endif + +#define INTERNAL_CATCH_MAKE_NAMESPACE2(...) ns_##__VA_ARGS__ +#define INTERNAL_CATCH_MAKE_NAMESPACE(name) INTERNAL_CATCH_MAKE_NAMESPACE2(name) + +#define INTERNAL_CATCH_REMOVE_PARENS(...) INTERNAL_CATCH_EXPAND1(INTERNAL_CATCH_DEF __VA_ARGS__) + +#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) decltype(get_wrapper()) +#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__)) +#else +#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) INTERNAL_CATCH_EXPAND_VARGS(decltype(get_wrapper())) +#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_EXPAND_VARGS(INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__))) +#endif + +#define INTERNAL_CATCH_MAKE_TYPE_LISTS_FROM_TYPES(...)\ + CATCH_REC_LIST(INTERNAL_CATCH_MAKE_TYPE_LIST,__VA_ARGS__) + +#define INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_0) INTERNAL_CATCH_REMOVE_PARENS(_0) +#define INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_0, _1) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_1) +#define INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_0, _1, _2) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_1, _2) +#define INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_0, _1, _2, _3) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_1, _2, _3) +#define INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_0, _1, _2, _3, _4) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_1, _2, _3, _4) +#define INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_0, _1, _2, _3, _4, _5) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_1, _2, _3, _4, _5) +#define INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_0, _1, _2, _3, _4, _5, _6) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_1, _2, _3, _4, _5, _6) +#define INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_0, _1, _2, _3, _4, _5, _6, _7) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_1, _2, _3, _4, _5, _6, _7) +#define INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_1, _2, _3, _4, _5, _6, _7, _8) +#define INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9) +#define INTERNAL_CATCH_REMOVE_PARENS_11_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10) + +#define INTERNAL_CATCH_VA_NARGS_IMPL(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N + +#define INTERNAL_CATCH_TYPE_GEN\ + template struct TypeList {};\ + template\ + constexpr auto get_wrapper() noexcept -> TypeList { return {}; }\ + template class...> struct TemplateTypeList{};\ + template class...Cs>\ + constexpr auto get_wrapper() noexcept -> TemplateTypeList { return {}; }\ + template\ + struct append;\ + template\ + struct rewrap;\ + template class, typename...>\ + struct create;\ + template class, typename>\ + struct convert;\ + \ + template \ + struct append { using type = T; };\ + template< template class L1, typename...E1, template class L2, typename...E2, typename...Rest>\ + struct append, L2, Rest...> { using type = typename append, Rest...>::type; };\ + template< template class L1, typename...E1, typename...Rest>\ + struct append, TypeList, Rest...> { using type = L1; };\ + \ + template< template class Container, template class List, typename...elems>\ + struct rewrap, List> { using type = TypeList>; };\ + template< template class Container, template class List, class...Elems, typename...Elements>\ + struct rewrap, List, Elements...> { using type = typename append>, typename rewrap, Elements...>::type>::type; };\ + \ + template