From 7a0c423357818b19eb431775452b1357bc7fd3eb Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Thu, 31 Jul 2025 09:17:42 +0200 Subject: Implement support for HTTP 429 Too Many Requests (#7760) * Implement support for HTTP 429 Too Many Requests Will obey the corresponding HTTP `Retry-After` header at domain level. * Implement 503 Service Unavailable * Sanitize Retry-After * Reduce default value when Retry-After is absent And make configuration parameter * Retry-After also for favicons --- app/Controllers/feedController.php | 6 ++-- app/Models/Feed.php | 29 ++++++++++----- app/Models/SimplePieResponse.php | 20 ++++++++++- app/Utils/httpUtil.php | 74 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 app/Utils/httpUtil.php (limited to 'app') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 2aae5a0a8..a080d5e67 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -83,7 +83,9 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController { switch ($kind) { case FreshRSS_Feed::KIND_RSS: case FreshRSS_Feed::KIND_RSS_FORCED: - $feed->load(true); //Throws FreshRSS_Feed_Exception, Minz_FileNotExistException + if ($feed->load(loadDetails: true) === null) { // Throws FreshRSS_Feed_Exception, Minz_FileNotExistException + throw new FreshRSS_FeedNotAdded_Exception($url); + } break; case FreshRSS_Feed::KIND_HTML_XPATH: case FreshRSS_Feed::KIND_XML_XPATH: @@ -345,7 +347,7 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController { $this->view->feed = new FreshRSS_Feed($url); try { // We try to get more information about the feed. - $this->view->feed->load(true); + $this->view->feed->load(loadDetails: true); $this->view->load_ok = true; } catch (Exception) { $this->view->load_ok = false; diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 171b054ca..81765d433 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -552,6 +552,10 @@ class FreshRSS_Feed extends Minz_Model { Minz_Exception::ERROR ); } else { + if (($retryAfter = FreshRSS_http_Util::getRetryAfter($this->url)) > 0) { + throw new FreshRSS_Feed_Exception('For that domain, will first retry after ' . date('c', $retryAfter) . + '. ' . $this->url(includeCredentials: false), code: 503); + } $simplePie = customSimplePie($this->attributes(), $this->curlOptions()); $url = htmlspecialchars_decode($this->url, ENT_QUOTES); if (str_ends_with($url, '#force_feed')) { @@ -571,15 +575,21 @@ class FreshRSS_Feed extends Minz_Model { Minz_ExtensionManager::callHook('simplepie_after_init', $simplePie, $this, $simplePieResult); if ($simplePieResult === false || $simplePie->get_hash() === '' || !empty($simplePie->error())) { - $errorMessage = $simplePie->error(); - if (empty($errorMessage)) { - $errorMessage = ''; - } elseif (is_array($errorMessage)) { - $errorMessage = json_encode($errorMessage, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_LINE_TERMINATORS) ?: ''; + if ($simplePie->status_code() === 429) { + $errorMessage = 'HTTP 429 Too Many Requests!'; + } elseif ($simplePie->status_code() === 503) { + $errorMessage = 'HTTP 503 Service Unavailable!'; + } else { + $errorMessage = $simplePie->error(); + if (empty($errorMessage)) { + $errorMessage = ''; + } elseif (is_array($errorMessage)) { + $errorMessage = json_encode($errorMessage, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_LINE_TERMINATORS) ?: ''; + } } throw new FreshRSS_Feed_Exception( ($errorMessage == '' ? 'Unknown error for feed' : $errorMessage) . - ' [' . \SimplePie\Misc::url_remove_credentials($this->url) . ']', + ' [' . $this->url(includeCredentials: false) . ']', $simplePie->status_code() ); } @@ -701,7 +711,7 @@ class FreshRSS_Feed extends Minz_Model { } if ($invalidGuids > 0) { - Minz_Log::warning("Feed has {$invalidGuids} invalid GUIDs: " . $this->url); + Minz_Log::warning("Feed has {$invalidGuids} invalid GUIDs: " . $this->url(includeCredentials: false)); if (!$this->attributeBoolean('unicityCriteriaForced') && $invalidGuids > round($invalidGuidsTolerance * count($items))) { $unicityCriteria = $this->attributeString('unicityCriteria'); if ($this->attributeBoolean('hasBadGuids')) { // Legacy @@ -719,7 +729,8 @@ class FreshRSS_Feed extends Minz_Model { if ($newUnicityCriteria !== $unicityCriteria) { $this->_attribute('hasBadGuids', null); // Remove legacy $this->_attribute('unicityCriteria', $newUnicityCriteria); - Minz_Log::warning('Feed unicity policy degraded (' . ($unicityCriteria ?: 'id') . ' → ' . $newUnicityCriteria . '): ' . $this->url); + Minz_Log::warning('Feed unicity policy degraded (' . ($unicityCriteria ?: 'id') . ' → ' . $newUnicityCriteria . '): ' . + $this->url(includeCredentials: false)); return $this->loadGuids($simplePie, $invalidGuidsTolerance); } } @@ -1167,7 +1178,7 @@ class FreshRSS_Feed extends Minz_Model { $affected = $feedDAO->markAsReadNotSeen($this->id(), $minLastSeen); } if ($affected > 0) { - Minz_Log::debug(__METHOD__ . " $affected items" . ($upstreamIsEmpty ? ' (all)' : '') . ' [' . $this->url(false) . ']'); + Minz_Log::debug(__METHOD__ . " $affected items" . ($upstreamIsEmpty ? ' (all)' : '') . ' [' . $this->url(includeCredentials: false) . ']'); } return $affected; } diff --git a/app/Models/SimplePieResponse.php b/app/Models/SimplePieResponse.php index 17c954e8c..6a444a86a 100644 --- a/app/Models/SimplePieResponse.php +++ b/app/Models/SimplePieResponse.php @@ -4,7 +4,25 @@ declare(strict_types=1); final class FreshRSS_SimplePieResponse extends \SimplePie\File { #[\Override] - protected function on_http_response(): void { + protected function on_http_response(string|false $response = ''): void { syslog(LOG_INFO, 'FreshRSS SimplePie GET ' . $this->get_status_code() . ' ' . \SimplePie\Misc::url_remove_credentials($this->get_final_requested_uri())); + + if (in_array($this->get_status_code(), [429, 503], true)) { + $parser = new \SimplePie\HTTP\Parser(is_string($response) ? $response : ''); + if ($parser->parse()) { + $headers = $parser->headers; + } else { + $headers = []; + } + + $retryAfter = FreshRSS_http_Util::setRetryAfter($this->get_final_requested_uri(), $headers['retry-after'] ?? ''); + if ($retryAfter > 0) { + $domain = parse_url($this->get_final_requested_uri(), PHP_URL_HOST); + if (is_string($domain) && $domain !== '') { + $errorMessage = 'Will retry after ' . date('c', $retryAfter) . ' for domain `' . $domain . '`'; + Minz_Log::notice($errorMessage); + } + } + } } } diff --git a/app/Utils/httpUtil.php b/app/Utils/httpUtil.php new file mode 100644 index 000000000..12ebc3d07 --- /dev/null +++ b/app/Utils/httpUtil.php @@ -0,0 +1,74 @@ +limits; + if (ctype_digit($retryAfter)) { + $retryAfter = time() + (int)$retryAfter; + } else { + $retryAfter = \SimplePie\Misc::parse_date($retryAfter) ?: + (time() + max(600, $limits['retry_after_default'] ?? 0)); + } + $retryAfter = min($retryAfter, time() + max(3600, $limits['retry_after_max'] ?? 0)); + + @mkdir(self::RETRY_AFTER_PATH); + if (!touch(self::RETRY_AFTER_PATH . $domain . '.txt', $retryAfter)) { + Minz_Log::error('Failed to set Retry-After for ' . $domain); + return 0; + } + return $retryAfter; + } +} -- cgit v1.2.3