Merge branch 'master' into payjoin_tests

MANIFEST.in

@@ -3,11 +3,14 @@ include README.rst
 include electrum.desktop
 include *.py
 include run_electrum
-include contrib/requirements/requirements.txt
-include contrib/requirements/requirements-hw.txt
 recursive-include packages *.py
 recursive-include packages cacert.pem
 
+include contrib/requirements/requirements*.txt
+include contrib/deterministic-build/requirements*.txt
+include contrib/make_libsecp256k1.sh
+include contrib/build_tools_util.sh
+
 graft electrum
 prune electrum/tests
 graft contrib/udev

RELEASE-NOTES

@@ -1,3 +1,34 @@
+# Release 4.0.4 - (Oct 15, 2020)
+ * PSBT: fix regression in 4.0.3 where UTXO data was not included in
+   QR codes (#6600)
+ * new feature: "Cancel tx" (#6641). The Qt/kivy GUI allows cancelling
+   an unconfirmed RBF tx by double-spending its inputs to self.
+ * Windows binary:
+   - fix some issues with QR scanning by building zbar ourselves (#6593)
+   - when using setup exe, also install a debug binary (#6603)
+ * Ledger: fix "The derivation path is unusual" warnings (#6512)
+   (needs Bitcoin app 1.4.8+ installed on device)
+ * A few other minor bugfixes and usability improvements.
+
+# Release 4.0.3 - (Sep 11, 2020)
+ * PSBT: restore compatibility with Bitcoin Core following CVE-2020-14199:
+   we now allow a PSBT input to have both UTXO and WITNESS_UTXO (#6429).
+   (PSBTs created since 4.0.1 already contained UTXO for segwit inputs)
+ * Hardware wallets:
+   - bitbox02: better multisig UX: implement get_soft_device_id (#6386)
+   - coldcard: fix "show address" for multisig (#6517)
+   - all: run all device communication on a dedicated thread (#6561).
+     This should resolve some threading issues.
+ * new feature: "Automated BIP39 recovery" (#6219, #6155)
+   When restoring from a BIP39 seed, add option to scan many known
+   derivation paths for history, and show them to user to choose from.
+ * show derivation path of keystores in Qt GUI Wallet>Information (#4700)
+ * fix "signtransaction" RPC command (#6502)
+ * Dependencies: pyaes is no longer needed (#6563)
+ * The tar.gz source dist now bundles make_libsecp256k1.sh, to help
+   users getting libsecp256k1 (#6323).
+ * A few other minor bugfixes and usability improvements.
+
 # Release 4.0.2 - (July 8, 2020)
  - rm old corrupted non-bip70 invoices (#6345)
  - other minor fixes

contrib/android/Dockerfile

@@ -92,29 +92,38 @@ ENV HOME_DIR="/home/${USER}"
 ENV WORK_DIR="${HOME_DIR}/wspace" \
     PATH="${HOME_DIR}/.local/bin:${PATH}"
 
-# install system dependencies
+# install system/build dependencies
+# https://github.com/kivy/buildozer/blob/master/docs/source/installation.rst#android-on-ubuntu-2004-64bit
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        python3 python3-pip python3-setuptools git wget lbzip2 patch sudo \
-        software-properties-common libssl-dev \
-    && apt -y autoremove
-
-# build dependencies
-# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
-RUN dpkg --add-architecture i386 \
-    && apt -y update -qq \
-    && apt -y install -qq --no-install-recommends \
-        build-essential ccache git python3 python3-dev \
-        libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
-        libidn11:i386 \
-        zip zlib1g-dev zlib1g:i386 \
-    && apt -y autoremove \
-    && apt -y clean
-
-# specific recipes dependencies (e.g. libffi requires autoreconf binary)
-RUN apt -y update -qq \
-    && apt -y install -qq --no-install-recommends \
-        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        wget \
+        lbzip2 \
+        patch \
+        sudo \
+        software-properties-common \
+        git \
+        zip \
+        unzip \
+        build-essential \
+        ccache \
+        openjdk-13-jdk \
+        autoconf \
+        libtool \
+        pkg-config \
+        zlib1g-dev \
+        libncurses5-dev \
+        libncursesw5-dev \
+        libtinfo5 \
+        cmake \
+        libffi-dev \
+        libssl-dev \
+        automake \
+        gettext \
+        libltdl-dev \
     && apt -y autoremove \
     && apt -y clean
 

contrib/android/buildozer.spec

@@ -38,8 +38,8 @@ version.filename = %(source.dir)s/electrum/version.py
 # (list) Application requirements
 requirements =
     # note: re python3.8, see #6147
-    hostpython3==3.7.7,
-    python3==3.7.7,
+    hostpython3==3.7.9,
+    python3==3.7.9,
     android,
     openssl,
     plyer,

contrib/build-linux/appimage/Dockerfile

@@ -11,15 +11,15 @@ RUN apt-get update -q && \
         autoconf=2.69-9 \
         libtool=2.4.6-0.1 \
         xz-utils=5.1.1alpha+20120614-2ubuntu2 \
-        libssl-dev=1.0.2g-1ubuntu4.16 \
-        libssl1.0.0=1.0.2g-1ubuntu4.16 \
-        openssl=1.0.2g-1ubuntu4.16 \
+        libssl-dev=1.0.2g-1ubuntu4.17 \
+        libssl1.0.0=1.0.2g-1ubuntu4.17 \
+        openssl=1.0.2g-1ubuntu4.17 \
         zlib1g-dev=1:1.2.8.dfsg-2ubuntu4.3 \
         libffi-dev=3.2.1-4 \
         libncurses5-dev=6.0+20160213-1ubuntu1 \
         libsqlite3-dev=3.11.0-1ubuntu1.5 \
         libusb-1.0-0-dev=2:1.0.20-1 \
-        libudev-dev=229-4ubuntu21.28 \
+        libudev-dev=229-4ubuntu21.29 \
         gettext=0.19.7-2ubuntu3.1 \
         libzbar0=0.10+doc-10ubuntu1  \
         libdbus-1-3=1.10.6-1ubuntu3.6 \

contrib/build-linux/appimage/build.sh

@@ -13,7 +13,7 @@ CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage"
 export GCC_STRIP_BINARIES="1"
 
 # pinned versions
-PYTHON_VERSION=3.7.7
+PYTHON_VERSION=3.7.9
 PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15"
 SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386"
 
@@ -38,7 +38,7 @@ download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/AppImage/AppI
 verify_hash "$CACHEDIR/appimagetool" "d918b4df547b388ef253f3c9e7f6529ca81a885395c31f619d9aaf7030499a13"
 
 download_if_not_exist "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz"
-verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "06a0a9f1bf0d8cd1e4121194d666c4e28ddae4dd54346de6c343206599f02136"
+verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "91923007b05005b5f9bd46f3b9172248aea5abc1543e8a636d59e629c3331b01"
 
 
 
@@ -113,8 +113,11 @@ info "preparing electrum-locale."
 )
 
 
-info "installing electrum and its dependencies."
+info "Installing build dependencies."
 mkdir -p "$CACHEDIR/pip_cache"
+"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-appimage-build.txt"
+
+info "installing electrum and its dependencies."
 "$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
 "$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
 "$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"

contrib/build-wine/Dockerfile

@@ -6,8 +6,8 @@ RUN dpkg --add-architecture i386 && \
     apt-get update -q && \
     apt-get install -qy \
         wget=1.19.4-1ubuntu2.2 \
-        gnupg2=2.2.4-1ubuntu1.2 \
-        dirmngr=2.2.4-1ubuntu1.2 \
+        gnupg2=2.2.4-1ubuntu1.3 \
+        dirmngr=2.2.4-1ubuntu1.3 \
         python3-software-properties=0.96.24.32.1 \
         software-properties-common=0.96.24.32.1
 
@@ -17,8 +17,11 @@ RUN apt-get update -q && \
         p7zip-full=16.02+dfsg-6 \
         make=4.1-9.1ubuntu1 \
         mingw-w64=5.0.3-1 \
+        mingw-w64-tools=5.0.3-1 \
+        win-iconv-mingw-w64-dev=0.0.8-2 \
         autotools-dev=20180224.1 \
         autoconf=2.69-11 \
+        autopoint=0.19.8.1-6ubuntu0.3 \
         libtool=2.4.6-2 \
         gettext=0.19.8.1-6
 

contrib/build-wine/build.sh

@@ -11,7 +11,7 @@ export CACHEDIR="$here/.cache"
 export PIP_CACHE_DIR="$CACHEDIR/pip_cache"
 
 export BUILD_TYPE="wine"
-export GCC_TRIPLET_HOST="i686-w64-mingw32"
+export GCC_TRIPLET_HOST="i686-w64-mingw32"  # make sure to clear caches if changing this
 export GCC_TRIPLET_BUILD="x86_64-pc-linux-gnu"
 export GCC_STRIP_BINARIES="1"
 
@@ -29,6 +29,12 @@ else
     "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
 fi
 
+if [ -f "$PROJECT_ROOT/electrum/libzbar-0.dll" ]; then
+    info "libzbar already built, skipping"
+else
+    "$CONTRIB"/make_zbar.sh || fail "Could not build zbar"
+fi
+
 $here/prepare-wine.sh || fail "prepare-wine failed"
 
 info "Resetting modification time in C:\Python..."

contrib/build-wine/deterministic.spec

@@ -34,14 +34,14 @@ binaries += [b for b in collect_dynamic_libs('PyQt5') if 'qwindowsvista' in b[0]
 
 binaries += [('C:/tmp/libsecp256k1-0.dll', '.')]
 binaries += [('C:/tmp/libusb-1.0.dll', '.')]
+binaries += [('C:/tmp/libzbar-0.dll', '.')]
 
 datas = [
     (home+'electrum/*.json', 'electrum'),
     (home+'electrum/lnwire/*.csv', 'electrum/lnwire'),
     (home+'electrum/wordlist/english.txt', 'electrum/wordlist'),
     (home+'electrum/locale', 'electrum/locale'),
     (home+'electrum/plugins', 'electrum/plugins'),
-    ('C:\\Program Files (x86)\\ZBar\\bin\\', '.'),
     (home+'electrum/gui/icons', 'electrum/gui/icons'),
 ]
 datas += collect_data_files('trezorlib')
@@ -139,7 +139,7 @@ exe_portable = EXE(
 #####
 # exe and separate files that NSIS uses to build installer "setup" exe
 
-exe_dependent = EXE(
+exe_inside_setup_noconsole = EXE(
     pyz,
     a.scripts,
     exclude_binaries=True,
@@ -150,8 +150,20 @@ exe_dependent = EXE(
     icon=home+'electrum/gui/icons/electrum.ico',
     console=False)
 
+exe_inside_setup_console = EXE(
+    pyz,
+    a.scripts,
+    exclude_binaries=True,
+    name=os.path.join('build\\pyi.win32\\electrum', cmdline_name+"-debug"),
+    debug=False,
+    strip=None,
+    upx=False,
+    icon=home+'electrum/gui/icons/electrum.ico',
+    console=True)
+
 coll = COLLECT(
-    exe_dependent,
+    exe_inside_setup_noconsole,
+    exe_inside_setup_console,
     a.binaries,
     a.zipfiles,
     a.datas,

contrib/build-wine/prepare-wine.sh

@@ -2,22 +2,18 @@
 
 # Please update these carefully, some versions won't work under Wine
 NSIS_FILENAME=nsis-3.05-setup.exe
-NSIS_URL=https://prdownloads.sourceforge.net/nsis/$NSIS_FILENAME?download
+NSIS_URL=https://downloads.sourceforge.net/project/nsis/NSIS%203/3.05/$NSIS_FILENAME
 NSIS_SHA256=1a3cc9401667547b9b9327a177b13485f7c59c2303d4b6183e7bc9e6c8d6bfdb
 
-ZBAR_FILENAME=zbarw-20121031-setup.exe
-ZBAR_URL=https://sourceforge.net/projects/zbarw/files/$ZBAR_FILENAME/download
-ZBAR_SHA256=177e32b272fa76528a3af486b74e9cb356707be1c5ace4ed3fcee9723e2c2c02
-
 LIBUSB_REPO="https://github.com/libusb/libusb.git"
 LIBUSB_COMMIT="e782eeb2514266f6738e242cdcb18e3ae1ed06fa"
 # ^ tag v1.0.23
 
 PYINSTALLER_REPO="https://github.com/SomberNight/pyinstaller.git"
-PYINSTALLER_COMMIT="e934539374e30d1500fcdbe8e4eb0860413935b2"
-# ^ tag 3.6, plus a custom commit that fixes cross-compilation with MinGW
+PYINSTALLER_COMMIT="6e455b2c1208465742484436009bfb1e1baf2e01"
+# ^ tag 4.0, plus a custom commit that fixes cross-compilation with MinGW
 
-PYTHON_VERSION=3.7.7
+PYTHON_VERSION=3.7.9
 
 ## These settings probably don't need change
 export WINEPREFIX=/opt/wine64
@@ -47,12 +43,19 @@ info "Installing Python."
 # keys from https://www.python.org/downloads/#pubkeys
 KEYRING_PYTHON_DEV="keyring-electrum-build-python-dev.gpg"
 gpg --no-default-keyring --keyring $KEYRING_PYTHON_DEV --import "$here"/gpg_keys/7ED10B6531D7C8E1BC296021FC624643487034E5.asc
-PYTHON_DOWNLOADS="$CACHEDIR/python$PYTHON_VERSION"
+if [ "$GCC_TRIPLET_HOST" = "i686-w64-mingw32" ] ; then
+    ARCH="win32"
+elif [ "$GCC_TRIPLET_HOST" = "x86_64-w64-mingw32" ] ; then
+    ARCH="amd64"
+else
+    fail "unexpected GCC_TRIPLET_HOST: $GCC_TRIPLET_HOST"
+fi
+PYTHON_DOWNLOADS="$CACHEDIR/python$PYTHON_VERSION-$ARCH"
 mkdir -p "$PYTHON_DOWNLOADS"
 for msifile in core dev exe lib pip tools; do
     echo "Installing $msifile..."
-    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi" "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi"
-    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi.asc" "https://www.python.org/ftp/python/$PYTHON_VERSION/win32/${msifile}.msi.asc"
+    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi" "https://www.python.org/ftp/python/$PYTHON_VERSION/$ARCH/${msifile}.msi"
+    download_if_not_exist "$PYTHON_DOWNLOADS/${msifile}.msi.asc" "https://www.python.org/ftp/python/$PYTHON_VERSION/$ARCH/${msifile}.msi.asc"
     verify_signature "$PYTHON_DOWNLOADS/${msifile}.msi.asc" $KEYRING_PYTHON_DEV
     wine msiexec /i "$PYTHON_DOWNLOADS/${msifile}.msi" /qb TARGETDIR=$PYHOME
 done
@@ -63,11 +66,6 @@ $PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"
 info "Installing dependencies specific to binaries."
 $PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
 
-info "Installing ZBar."
-download_if_not_exist "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_URL"
-verify_hash "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_SHA256"
-wine "$CACHEDIR/$ZBAR_FILENAME" /S
-
 info "Installing NSIS."
 download_if_not_exist "$CACHEDIR/$NSIS_FILENAME" "$NSIS_URL"
 verify_hash "$CACHEDIR/$NSIS_FILENAME" "$NSIS_SHA256"
@@ -91,18 +89,19 @@ info "Compiling libusb..."
     git checkout -b pinned "${LIBUSB_COMMIT}^{commit}"
     echo "libusb_1_0_la_LDFLAGS += -Wc,-static" >> libusb/Makefile.am
     ./bootstrap.sh || fail "Could not bootstrap libusb"
-    host="i686-w64-mingw32"
+    host="$GCC_TRIPLET_HOST"
     LDFLAGS="-Wl,--no-insert-timestamp" ./configure \
         --host=$host \
-        --build=x86_64-pc-linux-gnu || fail "Could not run ./configure for libusb"
+        --build=$GCC_TRIPLET_BUILD || fail "Could not run ./configure for libusb"
     make -j4 || fail "Could not build libusb"
     ${host}-strip libusb/.libs/libusb-1.0.dll
 ) || fail "libusb build failed"
 cp "$CACHEDIR/libusb/libusb/.libs/libusb-1.0.dll" $WINEPREFIX/drive_c/tmp/  || fail "Could not copy libusb to its destination"
 
 
-# copy libsecp dll (already built)
+# copy already built DLLs
 cp "$PROJECT_ROOT/electrum/libsecp256k1-0.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libsecp to its destination"
+cp "$PROJECT_ROOT/electrum/libzbar-0.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libzbar to its destination"
 
 
 info "Building PyInstaller."
@@ -126,10 +125,21 @@ info "Building PyInstaller."
     echo "const char *electrum_tag = \"tagged by Electrum@$ELECTRUM_COMMIT_HASH\";" >> ./bootloader/src/pyi_main.c
     pushd bootloader
     # cross-compile to Windows using host python
-    python3 ./waf all CC=i686-w64-mingw32-gcc CFLAGS="-static -Wno-dangling-else -Wno-error=unused-value"
+    python3 ./waf all CC="${GCC_TRIPLET_HOST}-gcc" \
+                      CFLAGS="-static \
+                              -Wno-dangling-else \
+                              -Wno-error=unused-value \
+                              -Wno-error=implicit-function-declaration \
+                              -Wno-error=int-to-pointer-cast"
     popd
     # sanity check bootloader is there:
-    [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir!"
+    if [ "$GCC_TRIPLET_HOST" = "i686-w64-mingw32" ] ; then
+        [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir! (32bit)"
+    elif [ "$GCC_TRIPLET_HOST" = "x86_64-w64-mingw32" ] ; then
+        [[ -e PyInstaller/bootloader/Windows-64bit/runw.exe ]] || fail "Could not find runw.exe in target dir! (64bit)"
+    else
+        fail "unexpected GCC_TRIPLET_HOST: $GCC_TRIPLET_HOST"
+    fi
 ) || fail "PyInstaller build failed"
 info "Installing PyInstaller."
 $PYTHON -m pip install --no-dependencies --no-warn-script-location ./pyinstaller

contrib/build_tools_util.sh

@@ -119,8 +119,6 @@ export SOURCE_DATE_EPOCH=1530212462
 export PYTHONHASHSEED=22
 # Set the build type, overridden by wine build
 export BUILD_TYPE="${BUILD_TYPE:-$(uname | tr '[:upper:]' '[:lower:]')}"
-# No additional autoconf flags by default
-export AUTOCONF_FLAGS=""
 # Add host / build flags if the triplets are set
 if [ -n "$GCC_TRIPLET_HOST" ] ; then
     export AUTOCONF_FLAGS="$AUTOCONF_FLAGS --host=$GCC_TRIPLET_HOST"