Feat: OpenSky Master Database File channel

Include/Constants.h

@@ -301,6 +301,8 @@ constexpr long HTTP_NOT_AVAIL =     503;        //
 constexpr long HTTP_GATEWAY_TIMEOUT=504;        //                               Gateway Timeout
 constexpr long HTTP_TIMEOUT =       524;        //                               Connection Timeout
 constexpr long HTTP_NO_JSON =       601;        ///< private definition: cannot be parsed as JSON
+constexpr long HTTP_FLAG_SENDING =   -1;        ///< used only internal to logging: sending data
+constexpr long HTTP_FLAG_UDP =       -2;        ///< used only internal to logging: received UDP data
 constexpr int CH_MAC_ERR_CNT =      5;          // max number of tolerated errors, afterwards invalid channel
 constexpr int SERR_LEN = 100;                   // size of buffer for IO error texts (strerror_s)
 #define ERR_XPLANE_ONLY         "LiveTraffic works in X-Plane only, version 10 or higher"

Include/DataRefs.h

@@ -426,10 +426,11 @@ enum dataRefsLT {
     DR_CHANNEL_FORE_FLIGHT_SENDER,
     DR_CHANNEL_FSCHARTER,
     DR_CHANNEL_OPEN_GLIDER_NET,
-    DR_CHANNEL_ADSB_EXCHANGE_ONLINE,
     DR_CHANNEL_ADSB_HUB,
     DR_CHANNEL_OPEN_SKY_ONLINE,
     DR_CHANNEL_OPEN_SKY_AC_MASTERDATA,
+    DR_CHANNEL_OPEN_SKY_AC_MASTERFILE,
+    DR_CHANNEL_ADSB_EXCHANGE_ONLINE,
     DR_CHANNEL_REAL_TRAFFIC_ONLINE,     // currently highest-prio channel
     // always last, number of elements:
     CNT_DATAREFS_LT

Include/LTChannel.h

@@ -164,8 +164,11 @@ class LTOnlineChannel : public LTChannel
     virtual void CleanupCurl ();
     // CURL callback
     static size_t ReceiveData ( const char *ptr, size_t size, size_t nmemb, void *userdata );
-    // logs raw data to a text file
-    void DebugLogRaw (const char* data, bool bHeader = true);
+    /// @brief logs raw data to a text file
+    /// @param data The data to print, assumed to be zero-terminated text
+    /// @param httpCode `-1` for SENDing data, any other code is a received HTTP response code
+    /// @param bHeader Shall the header with timestamp be printed?
+    void DebugLogRaw (const char* data, long httpCode, bool bHeader = true);
 
 public:
     bool FetchAllData (const positionTy& pos) override;
@@ -196,8 +199,8 @@ class LTFlightDataChannel : public LTOnlineChannel {
 //MARK: LTACMasterdata
 //
 
-/// list of a/c for which static data is yet missing
-/// Note: Either a key is set (then we ask for a/c master data), or a call sign (then we ask for route information)
+/// @brief list of a/c for which static data is yet missing
+/// @note If no call sign is set then we ask for a/c master data, otherwise for route information
 struct acStatUpdateTy {
 public:
     LTFlightData::FDKeyTy acKey;    ///< a/c key to find a/c master data
@@ -206,6 +209,9 @@ struct acStatUpdateTy {
 
     DatRequTy type = DATREQU_NONE;   ///< type of this master data request
 
+    /// Request Attempt count, allows to route request to services of different priority
+    unsigned nRequCount = 0;
+
 public:
     /// @brief Constructor for both master data or route lookup
     /// @param k Key to aircraft, is always required to be able to update the aircraft after having fetched data
@@ -235,31 +241,64 @@ typedef std::set<acStatUpdateTy> setAcStatUpdateTy;
 typedef std::set<LTFlightData::FDKeyTy> setFdKeyTy;
 typedef std::set<std::string> setStringTy;
 
-/// Parent class for master data channels
+/// @brief Parent class for master data channels, handles queue for master data requests
+/// @details Static functions of LTACMasterdataChannel handle the queue of
+///          requests for master data. Implementations of this class register
+///          themselves and this way form a queue of channels.
+///          Requests received through RequestMasterData()
+///          and RequestRouteInfo() are passed on to the channels in the list.
+///          Each channel implementation can accept a request, add it to a local
+///          queue and (try to) process it, or reject it right away,
+///          so that it is offered to the next channel in the list.
 class LTACMasterdataChannel : public LTOnlineChannel
 {
-private:
-    /// Lock controlling multi-threaded access to all the 3 sets
-    static std::mutex mtxSets;
-    /// global list of static data requests
-    static setAcStatUpdateTy setAcStatUpdate;
+protected:
+    /// Lock synchronizing any thread access to the request lists
+    static std::recursive_mutex mtxMaster;
+    /// List of register master data services, in order of priority
+    static std::list<LTACMasterdataChannel*> lstChn;
+    /// list of static data requests for the current channel
+    setAcStatUpdateTy setAcStatRequ;
+    /// Last time the above list of requests got maintained, ie. cleared from outdated stuff
+    float tSetRequCleared = 0.0f;
     /// List of a/c to ignore, as we know we don't get data online
-    static setFdKeyTy setIgnoreAc;
+    setFdKeyTy setIgnoreAc;
     /// List of call signs to ignore, as we know we don't get route info online
-    static setStringTy setIgnoreCallSign;
-
-protected:
+    setStringTy setIgnoreCallSign;
     /// The request currently being processed
-    acStatUpdateTy requ;
+    acStatUpdateTy currRequ;
 
 public:
-	LTACMasterdataChannel (dataRefsLT ch, const char* chName) :
-        LTOnlineChannel(ch, CHT_MASTER_DATA, chName) {}
+    // Constructor
+    LTACMasterdataChannel (dataRefsLT ch, const char* chName);
 
-    virtual bool UpdateStaticData (const LTFlightData::FDKeyTy& keyAc,
-                                   const LTFlightData::FDStaticData& dat);
     int GetNumAcServed () const override { return 0; }  ///< how many a/c do we feed?
-
+
+protected:
+    /// @brief Called from static functions to receive a request for processing
+    /// @returns if request has been accepted
+    virtual bool AcceptRequest (const acStatUpdateTy& requ) = 0;
+    /// Add the request to the set if not duplicate
+    bool InsertRequest (const acStatUpdateTy& requ);
+    /// Is any request waiting?;
+    bool HaveAnyRequest () const { return !setAcStatRequ.empty(); }
+    /// @brief Fetch next master data request from our set into `currRequ`
+    /// @returns `true` if a request has been passed, `false` if no request was waiting
+    bool FetchNextRequest ();
+    /// Called regularly to keep the request queue updated
+    void MaintainMasterDataRequests ();
+
+    /// Perform the update to flight's static data
+    bool UpdateStaticData (const LTFlightData::FDKeyTy& keyAc,
+                           const LTFlightData::FDStaticData& dat);
+
+    /// Add the current request `currRequ` to the ignore list
+    void AddIgnore ();
+    /// Is the request already in one of the ignore lists?
+    bool ShallIgnore (const acStatUpdateTy& requ) const;
+
+    // *** Static function coordinating requests between channel objects ***
+public:
     /// Add request to fetch master data (returns `true` if added, `false` if duplicate)
     static bool RequestMasterData (const LTFlightData::FDKeyTy& keyAc,
                                    double distance)
@@ -270,21 +309,20 @@ class LTACMasterdataChannel : public LTOnlineChannel
                                    double distance)
     { return callSign.empty() ? false : RequestMasterData (keyAc, callSign, distance); }
 
-    /// Called regularly to keep the priority list updated
-    static void MaintainMasterDataRequests ();
-
 protected:
+    /// @brief Register a master data channel, that will be called to process requests
+    /// @note The order, in which registration happens, serves as a priority
+    static void RegisterMasterDataChn (LTACMasterdataChannel* pChn);
     /// Generically, uniquely add request to fetch data (returns `true` if added, `false` if duplicate)
     static bool RequestMasterData (const LTFlightData::FDKeyTy& keyAc,
                                    const std::string& callSign,
                                    double distance);
+    /// @brief Pass on a message to the next channel
+    /// @param pChn The calling channel, or `nullptr` if to process the channels from the beginning
+    /// @param requ The request to be passed on
+    /// @returns `true` if any channel accepted the request
+    static bool PassOnRequest (LTACMasterdataChannel* pChn, const acStatUpdateTy& requ);
 
-    /// Fetch next master data request from our set (`acKey` will be empty if there is no waiting request)
-    static acStatUpdateTy FetchNextRequest ();
-    /// Add an aircraft to the ignore list
-    static void AddIgnoreAc (const LTFlightData::FDKeyTy& keyAc);
-    /// Add a callsign to the ignore list
-    static void AddIgnoreCallSign (const std::string& cs);
 };
 
 //

Include/LTOpenSky.h

@@ -84,7 +84,8 @@ class OpenSkyConnection : public LTFlightDataChannel
 #define OPSKY_MD_CHECK_URL      "https://opensky-network.org/aircraft-database"
 #define OPSKY_MD_CHECK_POPUP    "Search and update OpenSky's databse of airframes"
 
-constexpr std::chrono::duration OPSKY_WAIT_BETWEEN = std::chrono::milliseconds(500);
+constexpr std::chrono::duration OPSKY_WAIT_BETWEEN = std::chrono::milliseconds( 300);   ///< Wait between immediate requests to OpenSky Master
+constexpr std::chrono::duration OPSKY_WAIT_NOQUEUE = std::chrono::milliseconds(3000);   ///< Wait if there is no request in the queue
 #define OPSKY_MD_NAME           "OpenSky Masterdata Online"
 #define OPSKY_MD_URL            "https://opensky-network.org/api/metadata/aircraft/icao/"
 #define OPSKY_MD_TRANSP_ICAO    "icao24"
@@ -100,6 +101,9 @@ constexpr std::chrono::duration OPSKY_WAIT_BETWEEN = std::chrono::milliseconds(5
 constexpr size_t OPSKY_MD_TEXT_VEHICLE_LEN = 20;    ///< length after which category description might contain useful text in case of a Surface Vehicle
 #define OPSKY_MD_TEXT_NO_CAT    "No ADS-B Emitter Category Information"
 
+#define OPSKY_MD_DB_URL         "https://opensky-network.org/datasets/metadata/"
+#define OPSKY_MD_DB_FILE        "aircraft-database-complete-%04d-%02d.csv"
+
 #define OPSKY_ROUTE_URL         "https://opensky-network.org/api/routes?callsign="
 #define OPSKY_ROUTE_CALLSIGN    "callsign"
 #define OPSKY_ROUTE_ROUTE       "route"
@@ -119,10 +123,59 @@ class OpenSkyAcMasterdata : public LTACMasterdataChannel
     std::string GetURL (const positionTy& pos) override;            ///< Returns the master data or route URL to query
     bool ProcessFetchedData () override;                            ///< Process received master or route data
 protected:
+    bool AcceptRequest (const acStatUpdateTy& requ) override;       ///< accept requests that aren't in the ignore lists
     void Main () override;                                          ///< virtual thread main function
     bool ProcessMasterData (JSON_Object* pJAc);                     ///< Process received aircraft master data
     bool ProcessRouteInfo (JSON_Object* pJRoute);                   ///< Process received route info
 };
 
+//
+//MARK: OpenSkyAcMasterFile
+//
+
+// Every how many lines to we save file position information?
+constexpr unsigned long OPSKY_NUM_LN_PER_POS = 250;
+
+// Index into the fields of each line of the database file
+enum AcMasterFileFieldsTy : size_t {
+    ACMFF_hexId  = 0,
+    ACMFF_reg,
+    ACMFF_manIcao,
+    ACMFF_man,
+    ACMFF_mdl,
+    ACMFF_designator,
+    ACMFF_serialNum,
+    ACMFF_lineNum,
+    ACMFF_icaoAircraftClass,
+    ACMFF_operator,
+    ACMFF_operatorCallsign,
+    ACMFF_opIcao,
+    ACMFF_opIata,
+    ACMFF_owner,
+    ACMFF_catDescr,
+    ACMFF_NUM_FIELDS
+};
+
+/// Represents downloading and reading from the OpenSky Master data file `aircraft-database-complete-YYYY-MM.csv`
+class OpenSkyAcMasterFile : public LTACMasterdataChannel
+{
+protected:
+    std::ifstream fAcDb;                                            ///< Aircraft Database file
+    typedef std::map<unsigned long,std::ifstream::pos_type> mapPosTy;///< map of a/c ids to file positions
+    mapPosTy mapPos;                                                ///< map of a/c ids to file positions
+    std::string ln;                                                 ///< a line in the database file
+public:
+    OpenSkyAcMasterFile ();                                         ///< Constructor sets channel, name, and URLs
+public:
+    std::string GetURL (const positionTy&) override { return ""; }  ///< No URL for the standard request processing
+    bool ProcessFetchedData () override;                            ///< Process looked up master data
+protected:
+    bool AcceptRequest (const acStatUpdateTy& requ) override;       ///< accept only master data requests
+    void Main () override;                                          ///< virtual thread main function
+    bool LookupData ();                                             ///< perform the file lookup
+    bool OpenDatabaseFile ();                                       ///< find an aircraft database file to open/download
+    bool TryOpenDbFile (int year, int month);                       ///< open/download the aircraft database file for the given month
+};
+
 
 #endif /* LTOpenSky_h */

Include/LiveTraffic.h

@@ -208,6 +208,11 @@ bool FileRecLookup (std::ifstream& f, size_t& n,
 void LTOpenURL  (const std::string& url, const std::string& addon = "");
 void LTOpenHelp (const std::string& path);
 
+// MARK: Remote File Download
+
+/// Download the given file, `false` if HTTP 404 not found, exceptions otherwise
+bool RemoteFileDownload (const std::string& url, const std::string& path);
+
 // MARK: String/Text Functions
 
 // change a std::string to uppercase
@@ -266,6 +271,10 @@ std::string str_concat (const std::vector<std::string>& vs, const std::string& s
 // returns first non-empty string, and "" in case all are empty
 std::string str_first_non_empty (const std::initializer_list<const std::string>& l);
 
+// separate string into fields with a multi-character delimiter
+std::vector<std::string> str_fields (const std::string& s,
+                                     const std::string& delim);
+
 /// Replaces personal information in the string, like email address
 std::string& str_replPers (std::string& s);