feat: asdbexchange support (#80)

Co-authored-by: Thomas MAUGIN <maugin.thomas@gmail.com>

Author: wiedehopf

Dockerfile

@@ -68,6 +68,74 @@ RUN ./sensible-build.sh ${DEBIAN_VERSION} && \
     cd package-${DEBIAN_VERSION} && \
     dpkg-buildpackage -b
 
+#ADSBEXCHANGE
+# pinned commits, feel free to update to most recent commit, no major versions usually
+
+FROM debian:buster as adsbexchange_packages
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+WORKDIR /tmp
+RUN set -x && \
+    apt-get update && \
+    apt-get install -y --no-install-suggests --no-install-recommends \
+    jq \
+    uuid-runtime \
+    wget \
+    make \
+    gcc \
+    ncurses-dev \
+    ncurses-bin \
+    zlib1g-dev \
+    zlib1g \
+    python3-venv \
+    python3-dev
+
+FROM adsbexchange_packages as adsbexchange
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+WORKDIR /tmp
+RUN set -x && \
+    mkdir -p /usr/local/share/adsbexchange/ && \
+    SRCTMP=/srctmp && \
+    # readsb as a feed client
+    URL=https://github.com/adsbxchange/readsb && \
+    COMMIT=da57fe341a2485fbeb645147f3738c565c38c312 && \
+    mkdir -p $SRCTMP && wget -O ${SRCTMP}.tar.gz ${URL}/archive/${COMMIT}.tar.gz && tar xf ${SRCTMP}.tar.gz -C ${SRCTMP} --strip-components=1 && \
+    pushd ${SRCTMP} && \
+    echo "$COMMIT" > READSB_VERSION && \
+    cat READSB_VERSION && \
+    make -j "$(nproc)" AIRCRAFT_HASH_BITS=12 && \
+    cp -v -T readsb /usr/local/share/adsbexchange/readsb && \
+    popd && \
+    rm -rf ${SRCTMP} ${SRCTMP}.tar.gz && \
+    # mlat-client
+    URL=https://github.com/adsbxchange/mlat-client &&\
+    COMMIT=8d8b5a784727526620d5608c6d581fe3082deb79 && \
+    mkdir -p $SRCTMP && wget -O ${SRCTMP}.tar.gz ${URL}/archive/${COMMIT}.tar.gz && tar xf ${SRCTMP}.tar.gz -C ${SRCTMP} --strip-components=1 && \
+    pushd ${SRCTMP} && \
+    VENV="/usr/local/share/adsbexchange/venv" && \
+    python3 -m venv "${VENV}" && \
+    source "${VENV}/bin/activate" && \
+    ./setup.py build && \
+    ./setup.py install && \
+    deactivate && \
+    popd && \
+    ldconfig && \
+    rm -rf ${SRCTMP} ${SRCTMP}.tar.gz && \
+    # adsbexchange-stats
+    URL=https://github.com/adsbxchange/adsbexchange-stats && \
+    COMMIT=6b9fe07ba728de5e732fe87fa3ae0afdbb390709 && \
+    mkdir -p $SRCTMP && wget -O ${SRCTMP}.tar.gz ${URL}/archive/${COMMIT}.tar.gz && tar xf ${SRCTMP}.tar.gz -C ${SRCTMP} --strip-components=1 && \
+    cp -v -T ${SRCTMP}/json-status /usr/local/share/adsbexchange/json-status && \
+    rm -rf ${SRCTMP} ${SRCTMP}.tar.gz && \
+    # Clean-up
+    apt-get remove -y ${TEMP_PACKAGES[@]} && \
+    apt-get autoremove -y && \
+    rm -rf /tmp/* /var/lib/apt/lists/* && \
+    # readsb: simple tests
+    /usr/local/share/adsbexchange/readsb --version && \
+    # mlat-client: simple test
+    /usr/local/share/adsbexchange/venv/bin/python3 -c 'import mlat.client'
+
 # THTTPD
 FROM alpine:3.13.2 AS thttpd
 
@@ -101,6 +169,7 @@ FROM debian:buster-slim as copyall
 COPY --from=dump1090 /tmp/dump1090/dump1090 /copy_root/usr/lib/fr24/
 COPY --from=dump1090 /tmp/dump1090/public_html_merged /copy_root/usr/lib/fr24/public_html
 COPY --from=piaware /tmp/piaware_builder /copy_root/piaware_builder
+COPY --from=adsbexchange  /usr/local/share/adsbexchange /copy_root/usr/local/share/adsbexchange
 RUN mv /copy_root/piaware_builder/piaware_*_*.deb /copy_root/piaware.deb && \
     rm -rf /copy_root/piaware_builder
 COPY --from=thttpd /thttpd/thttpd /copy_root/
@@ -149,7 +218,15 @@ RUN apt-get update && \
     librtlsdr-dev \
     pkg-config \
     libncurses5-dev \
-    libbladerf-dev && \
+    libbladerf-dev \
+    # adsbexchange
+    jq \
+    ncurses-bin \
+    zlib1g \
+    python3-venv \
+    curl \
+    gzip \
+    && \
     # RTL-SDR
     cd /tmp && \
     mkdir -p /etc/modprobe.d && \

README.md

@@ -36,6 +36,12 @@ docker run -d -p 8080:8080 -p 8754:8754 \
 	-e "HTML_SITE_NAME=MY_SITE_NAME" \
 	-e "PANORAMA_ID=MY_PANORAMA_ID" \
 	-e "LAYERS_OWM_API_KEY=MY_OWM_API_KEY" \
+	-e "SERVICE_ENABLE_ADSBEXCHANGE=true" \
+	-e "ADSBEXCHANGE_UUID=MY_UUID" \
+	-e "ADSBEXCHANGE_STATION_NAME=MY_STATION_NAME" \
+	-e "MLAT_EXACT_LAT=MY_EXACT_SITE_LAT" \
+	-e "MLAT_EXACT_LON=MY_EXACT_SITE_LON" \
+	-e "MLAT_ALTITUDE_MSL_METERS=MY_SITE_ALT_MSL_METERS" \
 	thomx/fr24feed-piaware
 ```
 
@@ -58,6 +64,7 @@ To disable starting a service you can add an environement variable :
 | `SERVICE_ENABLE_FR24FEED`             | `false`                  | Disable fr24feed service  |
 | `SERVICE_ENABLE_HTTP`                 | `false`                  | Disable http service      |
 | `SERVICE_ENABLE_IMPORT_OVER_NETCAT`   | `false`                  | Disable import over netcat|
+| `SERVICE_ENABLE_ADSBEXCHANGE`         | `false`                  | Disable adsbexchange feed |
 
 
 Ex : `-e "SERVICE_ENABLE_HTTP=false"`
@@ -73,6 +80,7 @@ docker run -it --rm \
 	-e "SERVICE_ENABLE_DUMP1090=false" \
 	-e "SERVICE_ENABLE_HTTP=false" \
 	-e "SERVICE_ENABLE_FR24FEED=false" \
+	-e "SERVICE_ENABLE_ADSBEXCHANGE=false" \
 	thomx/fr24feed-piaware /bin/bash
 ```
 When the container starts you should see the feeder id, note it. Wait 5 minutes and you should see a new receiver at https://fr.flightaware.com/adsb/piaware/claim (use the same IP as your docker host), claim it and exit the container.
@@ -98,6 +106,7 @@ docker run -it --rm \
 	-e "SERVICE_ENABLE_HTTP=false" \
 	-e "SERVICE_ENABLE_PIAWARE=false" \
 	-e "SERVICE_ENABLE_FR24FEED=false" \
+	-e "SERVICE_ENABLE_ADSBEXCHANGE=false" \
 	thomx/fr24feed-piaware /bin/bash
 ```
 
@@ -122,6 +131,45 @@ Add the environment variable `FR24FEED_FR24KEY` with your sharing key.
 
 Ex : `-e "FR24FEED_FR24KEY=0123456789"`
 
+## ADS-B Exchange
+
+
+Add the environment variable `ADSBEXCHANGE_UUID` with a UUID generated by <https://www.adsbexchange.com/myip/gen-uuid/>.
+In case of multiple receivers, please use a different UUID for each receiver.
+
+Add the environment variable `SERVICE_ENABLE_ADSBEXCHANGE` and set it to `true`.
+
+
+| Environment Variable                  | Configuration property   | Default value     |
+|---------------------------------------|--------------------------|-------------------|
+| `SERVICE_ENABLE_ADSBEXCHANGE`         | `enable this feed client`| `false`           |
+| `ADSBEXCHANGE_UUID`                   | `uuid (required)`        | `none`            |
+| `ADSBEXCHANGE_STATION_NAME`           | `station name`           | `none`            |
+| `ADSBEXCHANGE_MLAT`                   | `mlat`                   | `true`            |
+
+Configure the MLAT coordinates so that adsbexchange MLAT can work. (see its own section below)
+If you don't want to supply your exact coordinates, please set the `ADSBEXCHANGE_MLAT` environment variable to `false`. (you won't get MLAT results and won't contribute to MLAT)
+
+Add the environment variable `ADSBEXCHANGE_STATION_NAME`, it will be used for the mlat map / sync status.
+You can check that your MLAT is working correctly by searching for your station name here: <https://map.adsbexchange.com/mlat-map/>
+(MLAT map marker is snapped to a 5 mile grid and then offset randomly to avoid stacking markers, the precise lat / lon for MLAT are not publicly accessible at adsbexchange)
+
+The ADS-B Exchange Anywhere map will be available at: <https://www.adsbexchange.com/api/feeders/?feed=MY_UUID>
+
+## Exact coordinates for MLAT
+
+Get your exact coordinates and altitude above sealevel in meters from one these websites:
+ - <https://www.freemaptools.com/elevation-finder.htm>
+ - <https://www.mapcoordinates.net/en>
+
+It's important for MLAT accuracy that these aren't off by more than about 10 m / 30 ft.
+
+| Environment Variable                  | Configuration property   | Default value     |
+|---------------------------------------|--------------------------|-------------------|
+| `MLAT_EXACT_LAT`                      | `decimal latitude`       | `none`            |
+| `MLAT_EXACT_LON`                      | `decimal longitude`      | `none`            |
+| `MLAT_ALTITUDE_MSL_METERS`            | `altitude above MSL in m`| `none`            |
+
 ## Add custom properties
 
 **Note** : you can add any property to either fr24feed or piaware configuration file by adding an environment variable starting with `PIAWARE_...` or `FR24FEED_...`.

docker-compose.yml

@@ -27,3 +27,6 @@ services:
     logging:
       options:
         max-size: "30m"
+    tmpfs:
+      - /run:exec,size=32M
+      - /usr/lib/fr24/public_html/data:size=32M

root/etc/s6-overlay/s6-rc.d/adsbexchange-bundle/contents.d/adsbexchange-feed

@@ -0,0 +1 @@
+bundle

root/etc/s6-overlay/s6-rc.d/adsbexchange-bundle/contents.d/adsbexchange-mlat

@@ -0,0 +1,4 @@
+#!/command/with-contenv bash
+
+# exit container stop s6
+kill 1

root/etc/s6-overlay/s6-rc.d/adsbexchange-bundle/contents.d/adsbexchange-stats

@@ -0,0 +1,16 @@
+#!/command/with-contenv bash
+
+set -eo pipefail
+
+/usr/local/share/adsbexchange/readsb --net-only --debug=n --quiet \
+    --net-beast-reduce-interval 0.5 --net-connector feed.adsbexchange.com,30004,beast_reduce_out,feed.adsbexchange.com,64004 \
+    --write-json /run/adsbexchange-feed \
+    --write-json-every 1 \
+    --write-prom /run/adsbexchange-feed/stats.prom \
+    --net-heartbeat 60 --net-ro-size 1280 --net-ro-interval 0.2 \
+    --net-bi-port 30154 \
+    --net-connector 127.0.0.1,30005,beast_in \
+    --lat "${HTML_SITE_LAT}" --lon "${HTML_SITE_LON}" \
+    2>&1 | mawk -W interactive '{printf "%c[32m[adsbexchange-feed]%c[0m %s\n", 27, 27, $0}'
+
+# awk -W interactive ...  (prefix log messages with color and "[adsbexchange-feed]")

root/etc/s6-overlay/s6-rc.d/adsbexchange-bundle/type

@@ -0,0 +1 @@
+longrun

root/etc/s6-overlay/s6-rc.d/adsbexchange-feed/dependencies.d/conf-adsbexchange

@@ -0,0 +1,4 @@
+#!/command/with-contenv bash
+
+# exit container stop s6
+kill 1

root/etc/s6-overlay/s6-rc.d/adsbexchange-feed/dependencies.d/dump1090

@@ -0,0 +1,36 @@
+#!/command/with-contenv bash
+
+if [ "$ADSBEXCHANGE_MLAT" == "no" ] || [ "$ADSBEXCHANGE_MLAT" == "false" ]; then
+    tail -f /dev/null
+    exit 0
+fi
+
+set -eo pipefail
+
+MLAT_ALTITUDE_MSL_METERS="$( echo "${MLAT_ALTITUDE_MSL_METERS}" | tr --delete ' m')"
+
+if [ -z "$MLAT_EXACT_LAT" ] || [ -z "$MLAT_EXACT_LON" ] || [ -z "$MLAT_ALTITUDE_MSL_METERS" ] || [ -z "$ADSBEXCHANGE_STATION_NAME" ]; then
+    for i in {1..5}; do
+        echo "FATAL: MLAT_EXACT_LAT or MLAT_EXACT_LON or MLAT_ALTITUDE_MSL_METERS or ADSBEXCHANGE_STATION_NAME not set!" | mawk -W interactive '{printf "%c[32m[adsbexchange-conf]%c31m %s\n", 27, 27, $0}'
+    done
+    kill 1
+    exit 1
+fi
+# pushes MLAT results into dump1090 via port 30104
+# listens on 31003 to provide MLAT results as SBS / basestation
+# listens on 30157 to provide MLAT results as beast
+# pushes MLAT results into adsbexchange-feed so adsbexchange-stats can push a json with MLAT results for the anywhere map
+/usr/local/share/adsbexchange/venv/bin/mlat-client \
+    --input-type dump1090 --no-udp \
+    --input-connect 127.0.0.1:30005 \
+    --server feed.adsbexchange.com:31090 \
+    --lat "${MLAT_EXACT_LAT}" --lon "${MLAT_EXACT_LON}" \
+    --alt "${MLAT_ALTITUDE_MSL_METERS}" \
+    --user "${ADSBEXCHANGE_STATION_NAME}" \
+    --results beast,connect,127.0.0.1:30104 \
+    --results basestation,listen,31003 \
+    --results beast,listen,30157 \
+    --results beast,connect,127.0.0.1:30154 \
+    2>&1 | mawk -W interactive '{printf "%c[32m[adsbexchange-mlat]%c[0m %s\n", 27, 27, $0}'
+
+# awk -W interactive ...  (prefix log messages with color and "[adsbexchange-mlat]")

root/etc/s6-overlay/s6-rc.d/adsbexchange-feed/finish

@@ -0,0 +1 @@
+longrun

root/etc/s6-overlay/s6-rc.d/adsbexchange-feed/run

@@ -0,0 +1,4 @@
+#!/command/with-contenv bash
+
+# exit container stop s6
+kill 1

root/etc/s6-overlay/s6-rc.d/adsbexchange-feed/type

@@ -0,0 +1,7 @@
+#!/command/with-contenv bash
+
+set -eo pipefail
+
+/usr/local/share/adsbexchange/json-status 2>&1 | mawk -W interactive '{printf "%c[32m[adsbexchange-stats]%c[0m %s\n", 27, 27, $0}'
+
+# awk -W interactive ...  (prefix log messages with color and "[adsbexchange-stats]")

root/etc/s6-overlay/s6-rc.d/adsbexchange-mlat/dependencies.d/adsbexchange-feed

@@ -0,0 +1 @@
+longrun

root/etc/s6-overlay/s6-rc.d/adsbexchange-mlat/dependencies.d/conf-adsbexchange

@@ -0,0 +1,24 @@
+#!/command/with-contenv bash
+
+SERVICE_ENABLE_ADSBEXCHANGE=${SERVICE_ENABLE_ADSBEXCHANGE:-false}
+
+if [ "$SERVICE_ENABLE_ADSBEXCHANGE" == "false" ]; then
+    exit 1
+fi
+
+if (( ${#ADSBEXCHANGE_UUID} == 36 )); then
+    set -eo pipefail
+
+    echo "${ADSBEXCHANGE_UUID}" > /usr/local/share/adsbexchange/adsbx-uuid
+    mkdir -p /run/adsbexchange-stats
+    mkdir -p /run/adsbexchange-feed
+
+    # delay other adsbexchange services a little bit to produce less error messages on startup
+    sleep 1.5
+else
+    for i in {1..5}; do
+        echo "FATAL: ADSB_EXCHANGE_UUID not set or invalid!" | mawk -W interactive '{printf "%c[32m[adsbexchange-conf]%c[31m %s\n", 27, 27, $0}'
+    done
+    kill 1
+    exit 1
+fi

root/etc/s6-overlay/s6-rc.d/adsbexchange-mlat/dependencies.d/dump1090

@@ -0,0 +1 @@
+oneshot

root/etc/s6-overlay/s6-rc.d/adsbexchange-mlat/finish

@@ -0,0 +1 @@
+/etc/s6-overlay/s6-rc.d/conf-adsbexchange/script