From 1ad746fdd68986bd81b80e79c0d394915b0d1189 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Thu, 30 Jan 2025 13:44:14 +0100 Subject: [PATCH] initial untested implementation for #1322 --- src/librssguard-standard/src/definitions.h | 4 ++ .../src/standardserviceroot.cpp | 47 +++++++++++++++++-- .../src/standardserviceroot.h | 5 ++ src/librssguard/definitions/definitions.h | 2 - .../network-web/networkfactory.cpp | 4 ++ src/librssguard/network-web/networkfactory.h | 2 + 6 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/librssguard-standard/src/definitions.h b/src/librssguard-standard/src/definitions.h index 484c58977..4e3dcc525 100644 --- a/src/librssguard-standard/src/definitions.h +++ b/src/librssguard-standard/src/definitions.h @@ -8,6 +8,10 @@ #define ADVANCED_FEED_ADD_DIALOG_CODE 64 +#define HTTP_CODE_NOT_MODIFIED 304 +#define HTTP_CODE_TOO_MANY_REQUESTS 429 +#define HTTP_CODE_UNAVAILABLE 503 + #define RSS_REGEX_MATCHER "]+type=\"application\\/(?:rss\\+xml)\"[^>]*>" #define RSS_HREF_REGEX_MATCHER "href=\"([^\"]+)\"" diff --git a/src/librssguard-standard/src/standardserviceroot.cpp b/src/librssguard-standard/src/standardserviceroot.cpp index 55a54cbb5..43b1da640 100644 --- a/src/librssguard-standard/src/standardserviceroot.cpp +++ b/src/librssguard-standard/src/standardserviceroot.cpp @@ -195,6 +195,10 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const { return ServiceRoot::additionalFlags() | Qt::ItemFlag::ItemIsDragEnabled | Qt::ItemFlag::ItemIsDropEnabled; } +void StandardServiceRoot::clearFeedOverload(StandardFeed* feed) { + m_overloadedHosts.remove(QUrl(feed->source()).host()); +} + QList StandardServiceRoot::obtainNewMessages(Feed* feed, const QHash& stated_messages, @@ -203,6 +207,13 @@ QList StandardServiceRoot::obtainNewMessages(Feed* feed, Q_UNUSED(tagged_messages) StandardFeed* f = static_cast(feed); + + if (checkIfFeedOverloaded(f)) { + qWarningNN << LOGSEC_CORE << "Feed with source" << QUOTE_W_SPACE(f->source()) + << "was signalled temporarily being down. Returning no articles for now."; + return {}; + } + QByteArray feed_contents; QString formatted_feed_contents; int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt(); @@ -233,12 +244,27 @@ QList StandardServiceRoot::obtainNewMessages(Feed* feed, f->http2Status()); if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) { - qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError) - << "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source()); - throw FeedFetchException(Feed::Status::NetworkError, - NetworkFactory::networkErrorText(network_result.m_networkError)); + if (network_result.m_httpCode == HTTP_CODE_TOO_MANY_REQUESTS || + network_result.m_httpCode == HTTP_CODE_UNAVAILABLE) { + + QDateTime safe_dt = NetworkFactory::extractRetryAfter(network_result.m_headers.value(QSL("retry-after"))); + + m_overloadedHosts.insert(QUrl(f->source()).host(), safe_dt); + + qWarningNN << LOGSEC_CORE << "Feed" << QUOTE_W_SPACE_DOT(feed->source()) + << "indicates that there is too many requests right now on the same host."; + return {}; + } + else { + qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError) + << "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source()); + throw FeedFetchException(Feed::Status::NetworkError, + NetworkFactory::networkErrorText(network_result.m_networkError)); + } } else { + clearFeedOverload(f); + f->setLastEtag(network_result.m_headers.value(QSL("etag"))); if (network_result.m_httpCode == HTTP_CODE_NOT_MODIFIED && feed_contents.trimmed().isEmpty()) { @@ -566,6 +592,19 @@ void StandardServiceRoot::exportFeeds() { form.data()->exec(); } +bool StandardServiceRoot::checkIfFeedOverloaded(StandardFeed* feed) const { + if (feed->sourceType() == StandardFeed::SourceType::Url || + feed->sourceType() == StandardFeed::SourceType::EmbeddedBrowser) { + QString hostname = QUrl(feed->source()).host(); + QDateTime retry_after = m_overloadedHosts.value(hostname); + + return retry_after.isValid() && retry_after > QDateTime::currentDateTimeUtc(); + } + else { + return false; + } +} + QList StandardServiceRoot::serviceMenu() { if (m_serviceMenu.isEmpty()) { ServiceRoot::serviceMenu(); diff --git a/src/librssguard-standard/src/standardserviceroot.h b/src/librssguard-standard/src/standardserviceroot.h index 5133d4e0c..482e309c9 100644 --- a/src/librssguard-standard/src/standardserviceroot.h +++ b/src/librssguard-standard/src/standardserviceroot.h @@ -54,6 +54,9 @@ class StandardServiceRoot : public ServiceRoot { void exportFeeds(); private: + void clearFeedOverload(StandardFeed* feed); + bool checkIfFeedOverloaded(StandardFeed* feed) const; + // Takes structure residing under given root item and adds feeds/categories from // it to active structure. // NOTE: This is used for import/export of the model. @@ -61,6 +64,8 @@ class StandardServiceRoot : public ServiceRoot { QPointer m_feedForMetadata = {}; QList m_feedContextMenu = {}; + + QHash m_overloadedHosts; }; #endif // STANDARDSERVICEROOT_H diff --git a/src/librssguard/definitions/definitions.h b/src/librssguard/definitions/definitions.h index af212dc7c..e454c14c8 100644 --- a/src/librssguard/definitions/definitions.h +++ b/src/librssguard/definitions/definitions.h @@ -154,8 +154,6 @@ #define CLI_THREADS "threads" -#define HTTP_CODE_NOT_MODIFIED 304 - #define HTTP_HEADERS_ACCEPT "Accept" #define HTTP_HEADERS_CONTENT_TYPE "Content-Type" #define HTTP_HEADERS_CONTENT_LENGTH "Content-Length" diff --git a/src/librssguard/network-web/networkfactory.cpp b/src/librssguard/network-web/networkfactory.cpp index 1a4b73e7b..9f6b3765f 100644 --- a/src/librssguard/network-web/networkfactory.cpp +++ b/src/librssguard/network-web/networkfactory.cpp @@ -13,6 +13,10 @@ #include #include +QDateTime NetworkFactory::extractRetryAfter(const QString& retry_after_value) { + return {}; +} + QStringList NetworkFactory::extractFeedLinksFromHtmlPage(const QUrl& url, const QString& html) { QStringList feeds; QRegularExpression rx(QSL(FEED_REGEX_MATCHER), QRegularExpression::PatternOption::CaseInsensitiveOption); diff --git a/src/librssguard/network-web/networkfactory.h b/src/librssguard/network-web/networkfactory.h index f3d0566ea..d0f5bdbca 100644 --- a/src/librssguard/network-web/networkfactory.h +++ b/src/librssguard/network-web/networkfactory.h @@ -48,6 +48,8 @@ class RSSGUARD_DLLSPEC NetworkFactory { Disabled = 2 }; + static QDateTime extractRetryAfter(const QString& retry_after_value); + static QStringList extractFeedLinksFromHtmlPage(const QUrl& url, const QString& html); static QPair generateBasicAuthHeader(NetworkAuthentication protection, const QString& username,