Add support for Manylinux2014 (#750)

Author: jopohl

.gitignore

@@ -48,6 +48,9 @@ src/urh/dev/native/lib/*.cpp
 
 /tests/show_gui
 
-
+src/urh/*.txt
+src/urh/URHProject.xml
+.coverage*
+src/urh/dev/gr/scripts/top_block.py
 
 misc/*

README.md

@@ -67,22 +67,37 @@ On Windows, URH can be installed with its [Installer](https://github.com/jopohl/
 If you get an error about missing ```api-ms-win-crt-runtime-l1-1-0.dll```, run Windows Update or directly install [KB2999226](https://support.microsoft.com/en-us/help/2999226/update-for-universal-c-runtime-in-windows).
 
 ### Linux
-#### Install via Package Manager
-URH is included in the repositories of many linux distributions such as __Arch Linux__, __Gentoo__, __Fedora__, __openSUSE__ or __NixOS__. There is also a package for __FreeBSD__. If available, simply use your package manager to install URH. 
-
-#### Generic Installation with pip (Ubuntu/Debian)
-URH is available on [PyPi](https://pypi.org/project/urh/) so you can install it with ``` pip ``` . For Ubuntu/Debian use the following commands to install URH including extensions for native SDR support.
+#### Generic Installation with pip (recommended)
+URH is available on [PyPi](https://pypi.org/project/urh/) so you can install it with 
+```bash 
+# IMPORTANT: Make sure your pip is up to date
+sudo python3 -m pip install --upgrade pip  # Update your pip installation
+sudo python3 -m pip install urh            # Install URH
+``` 
+This is the recommended way to install URH on Linux because it comes with __all native extensions__ precompiled.
+
+##### udev rules
+In order to access your SDR as non-root user, install the according __udev rules__. 
+For example, you can install the HackRF udev rules with the following commands.
 
 ```bash
-sudo apt update
-sudo apt install python3-numpy python3-psutil python3-pyqt5 g++ libpython3-dev python3-pip cython3
-# Install following packages for native support of corresponding SDR (before installing URH)
-sudo apt install libhackrf-dev liblimesuite-dev libbladerf-dev librtlsdr-dev libairspy-dev libuhd-dev libiio-dev
-sudo pip3 install urh
+sudo tee -a /etc/udev/rules.d/52-hackrf.rules >/dev/null <<-EOF
+ATTR{idVendor}=="1d50", ATTR{idProduct}=="604b", SYMLINK+="hackrf-jawbreaker-%k", MODE="660", GROUP="plugdev"
+ATTR{idVendor}=="1d50", ATTR{idProduct}=="6089", SYMLINK+="hackrf-one-%k", MODE="660", GROUP="plugdev"
+ATTR{idVendor}=="1fc9", ATTR{idProduct}=="000c", SYMLINK+="hackrf-dfu-%k", MODE="660", GROUP="plugdev"
+EOF
+sudo udevadm control --reload-rules
 ```
+Make sure your current user is in the ```plugdev``` group to make these rules work.
+You find rules for other SDRs by searching for "```<SDR name>``` udev rules" in your favorite search engine.
+
+#### Install via Package Manager
+URH is included in the repositories of many linux distributions such as __Arch Linux__, __Gentoo__, __Fedora__, __openSUSE__ or __NixOS__. There is also a package for __FreeBSD__.  If available, simply use your package manager to install URH.
+
+__Note__: For native support, you must install the according ```-dev``` package(s) of your SDR(s) such as ```hackrf-dev``` __before__ installing URH.
 
 #### Docker Image
-The official URH docker image is available [here](https://hub.docker.com/r/jopohl/urh/).
+The official URH docker image is available [here](https://hub.docker.com/r/jopohl/urh/). It has all native backends included and ready to operate. 
 
 ### MacOS
 #### Using DMG

data/azure-pipelines.yml

@@ -27,30 +27,41 @@ jobs:
       architecture: 'x64'
 
   - script: |
-      python -m pip install --upgrade pip
+      python -m pip install --upgrade pip twine
       pip install -r data/requirements.txt
       sudo apt-get install libhackrf-dev librtlsdr-dev xvfb libxkbcommon-x11-0 x11-utils
       pip install twine setuptools wheel pytest pytest-xvfb pytest-cov pytest-faulthandler
       python -c "import tempfile, os; open(os.path.join(tempfile.gettempdir(), 'urh_releasing'), 'w').close()"
     displayName: 'Install dependencies'
 
-  - script: python setup.py build_ext --inplace
+  - script: python src/urh/cythonext/build.py
     displayName: "Build extensions"
 
   - script: python setup.py sdist
     displayName: "Build sdist"
     condition: eq(variables['python.version'], '3.7')
 
-  - script: twine upload --skip-existing dist/*.gz
-    displayName: "Upload sdist to PyPi"
-    condition: and(eq(variables['python.version'], '3.7'), contains(variables['Build.SourceBranch'], 'refs/tags/'))
+  - script: |
+      docker run --rm \
+      -e PYVER=$(python -c "import sys; print('%s%s' % (sys.version_info.major, sys.version_info.minor))") \
+      -v `pwd`:/io jopohl/urh_manylinux2014 /io/data/make_manylinux2014_wheels.sh
+    displayName: "Build wheel"
+
+  - script: |
+      pip install dist/*.whl
+      urh --version
+      xvfb-run urh autoclose
+    displayName: 'Testrun'
+
+  - script: twine upload --skip-existing dist/*
+    displayName: "PyPi Upload"
+    condition: contains(variables['Build.SourceBranch'], 'refs/tags/')
     env:
       TWINE_USERNAME: $(twine.username)
       TWINE_PASSWORD: $(twine.password)
       TWINE_REPOSITORY_URL: "https://upload.pypi.org/legacy/"
 
   - task: PublishBuildArtifacts@1
-    condition: eq(variables['python.version'], '3.7')
     inputs:
       pathtoPublish: dist/
       artifactName: 'dist'
@@ -97,12 +108,6 @@ jobs:
       coverageType: lines
       baseBranchRef: refs/heads/master
 
-  - script: |
-      pip install .
-      urh --version
-      xvfb-run urh autoclose
-    displayName: 'Testrun'
-
 - job: 'Windows'
   pool:
     vmImage: 'vs2017-win2016'
@@ -144,7 +149,7 @@ jobs:
       destinationFolder: src\urh\dev\native\lib\shared
       #cleanDestinationFolder: true
 
-  - script: python setup.py build_ext --inplace
+  - script: python src/urh/cythonext/build.py
     displayName: "Build extensions"
 
   - script: |
@@ -253,7 +258,7 @@ jobs:
       sudo make install
     displayName: "Install libusb and boost"
 
-  - script: python setup.py build_ext --inplace
+  - script: python src/urh/cythonext/build.py
     displayName: "Build extensions"
 
   - script: python setup.py bdist_wheel

data/make_manylinux2014_wheels.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# for adapted jopohl/urh_manylinux
+
+touch /tmp/urh_releasing
+for PYBIN in /opt/python/*$PYVER*/bin; do   # for all if PYVER not set
+    echo -e "\033[1mInstalling requirements for $PYBIN\033[0m"
+    "${PYBIN}/pip" install -r /io/data/requirements.txt
+
+    cd /io || return
+    echo -e "\033[1mBuilding extentions for $PYBIN\033[0m"
+    "${PYBIN}/python3" setup.py build_ext "-j$(nproc)"
+
+    echo -e "\033[1mBuilding wheel for $PYBIN\033[0m"
+    "${PYBIN}/pip" wheel --no-deps /io/ -w /wheelhouse/
+done
+
+# Bundle external libs into wheels
+echo -e '\033[92mRepairing wheels...\033[0m'
+for whl in /wheelhouse/*.whl; do
+    auditwheel repair "$whl" -w /io/dist/
+done

data/make_manylinux2014_wheels_full.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# for plain quay.io/pypa/manylinux2014_x86_64
+
+AIRSPY_VERSION="1.0.9"
+BLADERF_VERSION="2018.08"
+LIMESUITE_VERSION="20.01.0"
+SDRPLAY_VERSION="2.13"
+RTLSDR_VERSION="0.6.0"
+
+echo -e '\033[92mInstalling dependencies...\033[0m'
+yum -y -q install wget cmake3 hackrf-devel uhd-devel\
+       https://github.com/analogdevicesinc/libiio/releases/download/v0.19/libiio-0.19.g5f5af2e-centos-7-x86_64.rpm
+
+build_airspy() {
+   &> /dev/null
+  tar xf /tmp/airspy.tar.gz -C /tmp
+  cmake3 -Wno-dev -S /tmp/airspyone_host-$AIRSPY_VERSION -B /tmp/build_airspy > /dev/null
+  make --silent -C /tmp/build_airspy > /dev/null
+  make --silent -C /tmp/build_airspy install > /dev/null
+}
+
+build_bladerf() {
+  wget https://github.com/Nuand/bladeRF/archive/$BLADERF_VERSION.tar.gz -O /tmp/bladeRF.tar.gz &> /dev/null
+  tar xf /tmp/bladeRF.tar.gz -C /tmp
+  cmake3 -Wno-dev -S /tmp/bladeRF-$BLADERF_VERSION/host -B /tmp/build_blade > /dev/null
+  make --silent -C /tmp/build_blade > /dev/null
+  make --silent -C /tmp/build_blade install > /dev/null
+}
+
+build_limesdr() {
+  wget https://github.com/myriadrf/LimeSuite/archive/v$LIMESUITE_VERSION.tar.gz -O /tmp/lime.tar.gz &> /dev/null
+  tar xf /tmp/lime.tar.gz -C /tmp
+  cmake3 -Wno-dev -S /tmp/LimeSuite-$LIMESUITE_VERSION -B /tmp/build_lime > /dev/null
+  make --silent -C /tmp/build_lime > /dev/null
+  make --silent -C /tmp/build_lime install > /dev/null
+}
+
+build_rtlsdr() {
+  wget https://github.com/osmocom/rtl-sdr/archive/$RTLSDR_VERSION.tar.gz -O /tmp/rtlsdr.tar.gz &> /dev/null
+  tar xf /tmp/rtlsdr.tar.gz -C /tmp
+  cmake3 -Wno-dev -DDETACH_KERNEL_DRIVER=ON -S /tmp/rtl-sdr-$RTLSDR_VERSION -B /tmp/build_rtlsdr > /dev/null
+  make --silent -C /tmp/build_rtlsdr > /dev/null
+  make --silent -C /tmp/build_rtlsdr install > /dev/null
+}
+
+build_sdrplay() {
+  wget http://www.sdrplay.com/software/SDRplay_RSP_API-Linux-$SDRPLAY_VERSION.1.run -O /tmp/sdrplay.run &> /dev/null
+  bash /tmp/sdrplay.run --tar xf -C /tmp
+  mv /tmp/mirsdrapi-rsp.h /usr/include
+  mv /tmp/x86_64/* /usr/lib64
+  ln -s /usr/lib64/libmirsdrapi-rsp.so.$SDRPLAY_VERSION /usr/lib64/libmirsdrapi-rsp.so
+}
+
+echo -e '\033[92mCompiling SDR libs...\033[0m'
+build_airspy &
+build_bladerf &
+build_limesdr &
+build_rtlsdr &
+build_sdrplay &
+wait
+
+touch /tmp/urh_releasing
+for PYBIN in /opt/python/*/bin; do
+    echo -e "\033[1mInstalling requirements for $PYBIN\033[0m"
+    "${PYBIN}/pip" install -r /io/data/requirements.txt > /dev/null
+
+    cd /io || return
+    echo -e "\033[1mBuilding extentions for $PYBIN\033[0m"
+    "${PYBIN}/python3" setup.py build_ext "-j$(nproc)" | grep --color=always "Skipping"
+
+    echo -e "\033[1mBuilding wheel for $PYBIN\033[0m"
+    "${PYBIN}/pip" wheel --no-deps /io/ -w /wheelhouse/ > /dev/null
+done
+
+# Bundle external libs into wheels
+echo -e '\033[92mRepairing wheels...\033[0m'
+for whl in /wheelhouse/*.whl; do
+    auditwheel repair "$whl" -w /io/dist/
+done

data/manylinux2014.Dockerfile

@@ -0,0 +1,35 @@
+FROM quay.io/pypa/manylinux2014_x86_64
+
+RUN export AIRSPY_VERSION="1.0.9" \
+ && export BLADERF_VERSION="2018.08" \
+ && export LIMESUITE_VERSION="20.01.0" \
+ && export SDRPLAY_VERSION="2.13" \
+ && export RTLSDR_VERSION="0.6.0" \
+ && yum -y install wget cmake3 hackrf-devel uhd-devel https://github.com/analogdevicesinc/libiio/releases/download/v0.19/libiio-0.19.g5f5af2e-centos-7-x86_64.rpm \
+ && wget https://github.com/airspy/airspyone_host/archive/v$AIRSPY_VERSION.tar.gz -O /tmp/airspy.tar.gz \
+ && tar xf /tmp/airspy.tar.gz -C /tmp \
+ && cmake3 -Wno-dev -S /tmp/airspyone_host-$AIRSPY_VERSION -B /tmp/build_airspy \
+ && make -j$(nproc) -C /tmp/build_airspy  \
+ && make -C /tmp/build_airspy install \
+ && wget https://github.com/Nuand/bladeRF/archive/$BLADERF_VERSION.tar.gz -O /tmp/bladeRF.tar.gz \
+ && tar xf /tmp/bladeRF.tar.gz -C /tmp \
+ && cmake3 -Wno-dev -S /tmp/bladeRF-$BLADERF_VERSION/host -B /tmp/build_blade \
+ && make -j$(nproc) -C /tmp/build_blade \
+ && make -C /tmp/build_blade install \
+ && wget https://github.com/myriadrf/LimeSuite/archive/v$LIMESUITE_VERSION.tar.gz -O /tmp/lime.tar.gz \
+ && tar xf /tmp/lime.tar.gz -C /tmp \
+ && cmake3 -S /tmp/LimeSuite-$LIMESUITE_VERSION -B /tmp/build_lime \
+ && make -j$(nproc) -C /tmp/build_lime \
+ && make -C /tmp/build_lime install \
+ && wget https://github.com/osmocom/rtl-sdr/archive/$RTLSDR_VERSION.tar.gz -O /tmp/rtlsdr.tar.gz \
+ && tar xf /tmp/rtlsdr.tar.gz -C /tmp \
+ && cmake3 -DDETACH_KERNEL_DRIVER=ON -S /tmp/rtl-sdr-$RTLSDR_VERSION -B /tmp/build_rtlsdr \
+ && make -j$(nproc) -C /tmp/build_rtlsdr \
+ && make -C /tmp/build_rtlsdr install \
+ && wget http://www.sdrplay.com/software/SDRplay_RSP_API-Linux-$SDRPLAY_VERSION.1.run -O /tmp/sdrplay.run \
+ && bash /tmp/sdrplay.run --tar xf -C /tmp \
+ && mv /tmp/mirsdrapi-rsp.h /usr/include \
+ && mv /tmp/x86_64/* /usr/lib64 \
+ && ln -s /usr/lib64/libmirsdrapi-rsp.so.$SDRPLAY_VERSION /usr/lib64/libmirsdrapi-rsp.so \
+ && rm -rf /tmp/* \
+ && yum clean all

setup.py

@@ -1,5 +1,4 @@
 import os
-import shutil
 import sys
 import tempfile
 
@@ -31,7 +30,6 @@
     OPEN_MP_FLAG = "-fopenmp"
     NO_NUMPY_WARNINGS_FLAG = "-Wno-cpp"
 
-
 UI_SUBDIRS = ("actions", "delegates", "views")
 PLUGINS = [path for path in os.listdir("src/urh/plugins") if os.path.isdir(os.path.join("src/urh/plugins", path))]
 URH_DIR = "urh"
@@ -96,7 +94,8 @@ def get_extensions():
         for extension in extensions:
             extension.extra_compile_args.append(NO_NUMPY_WARNINGS_FLAG)
 
-    extensions = cythonize(extensions, compiler_directives=COMPILER_DIRECTIVES, quiet=True, compile_time_env=device_extras)
+    extensions = cythonize(extensions, compiler_directives=COMPILER_DIRECTIVES, compile_time_env=device_extras,
+                           nthreads=0 if sys.platform == "win32" else os.cpu_count())
     return extensions
 
 
@@ -108,6 +107,7 @@ def read_long_description():
     except:
         return ""
 
+
 install_requires = ["numpy", "psutil", "cython"]
 if IS_RELEASE:
     install_requires.append("pyqt5")
@@ -120,7 +120,6 @@ def read_long_description():
 if sys.version_info < (3, 4):
     install_requires.append('enum34')
 
-
 setup(
     name="urh",
     version=version.VERSION,

src/urh/cythonext/build.py

@@ -9,8 +9,8 @@
 def main():
     cur_dir = os.path.realpath(__file__)
     os.chdir(os.path.realpath(os.path.join(cur_dir, "..", "..", "..", "..")))
-    #call([sys.executable, "setup.py", "clean", "--all"])
-    call([sys.executable, "setup.py", "build_ext", "--inplace"])
+    # call([sys.executable, "setup.py", "clean", "--all"])
+    call([sys.executable, "setup.py", "build_ext", "--inplace", "-j{}".format(os.cpu_count())])
 
 
 if __name__ == "__main__":

src/urh/dev/native/ExtensionHelper.py

@@ -18,7 +18,6 @@
                        'initializedcheck': False,
                        }
 
-
 DEVICES = {
     "airspy": {"lib": "airspy", "test_function": "open"},
     "bladerf": {"lib": "bladeRF", "test_function": "bladerf_open",
@@ -44,11 +43,11 @@
     "sdrplay": {"lib": "mir_sdr_api" if sys.platform == "win32" else "mirsdrapi-rsp",
                 "test_function": "mir_sdr_ApiVersion",
                 "api_version_check_code":
-                """
-                #include<stdio.h>
-                int main(void) {
-                float version=0.0; mir_sdr_ApiVersion(&version); printf("%f", version); return 0;}
-                """}
+                    """
+                    #include<stdio.h>
+                    int main(void) {
+                    float version=0.0; mir_sdr_ApiVersion(&version); printf("%f", version); return 0;}
+                    """}
 }
 
 
@@ -193,7 +192,7 @@ def get_device_extensions_and_extras(library_dirs=None, include_dirs=None):
                 print("    Environment variable {} is unset, try to automatically detect API version".format(env_name))
 
             if ver is None:
-                ver = check_api_version(compiler, params["api_version_check_code"], (params["lib"], ),
+                ver = check_api_version(compiler, params["api_version_check_code"], (params["lib"],),
                                         library_dirs, include_dirs)
             device_extras[env_name] = ver
             print("    Using {}={}".format(env_name, ver))
@@ -267,6 +266,7 @@ def perform_health_check() -> str:
         include_directories = []
 
     import numpy as np
+
     include_directories.append(np.get_include())
 
     cur_dir = os.path.dirname(os.path.realpath(__file__))
@@ -282,8 +282,11 @@ def perform_health_check() -> str:
 
     dev_extensions, dev_extras = get_device_extensions_and_extras(library_dirs=library_directories,
                                                                   include_dirs=include_directories)
+    sys.argv.append("-j{}".format(os.cpu_count()))
+
     setup(
         name="urh",
         ext_modules=cythonize(dev_extensions, force=True,
-                              compile_time_env=dev_extras, compiler_directives=COMPILER_DIRECTIVES),
+                              compile_time_env=dev_extras, compiler_directives=COMPILER_DIRECTIVES,
+                              nthreads=0 if sys.platform == "win32" else os.cpu_count()),
     )