Implement support for HTTP 429 Too Many Requests (#7760)

* Implement support for HTTP 429 Too Many Requests
Will obey the corresponding HTTP `Retry-After` header at domain level.

* Implement 503 Service Unavailable

* Sanitize Retry-After

* Reduce default value when Retry-After is absent
And make configuration parameter

* Retry-After also for favicons

4 files changed, 117 insertions, 12 deletions

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index 2aae5a0a8..a080d5e67 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -83,7 +83,9 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController {
 		switch ($kind) {
 			case FreshRSS_Feed::KIND_RSS:
 			case FreshRSS_Feed::KIND_RSS_FORCED:
-				$feed->load(true);	//Throws FreshRSS_Feed_Exception, Minz_FileNotExistException
+				if ($feed->load(loadDetails: true) === null) {	// Throws FreshRSS_Feed_Exception, Minz_FileNotExistException
+					throw new FreshRSS_FeedNotAdded_Exception($url);
+				}
 				break;
 			case FreshRSS_Feed::KIND_HTML_XPATH:
 			case FreshRSS_Feed::KIND_XML_XPATH:
@@ -345,7 +347,7 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController {
 			$this->view->feed = new FreshRSS_Feed($url);
 			try {
 				// We try to get more information about the feed.
-				$this->view->feed->load(true);
+				$this->view->feed->load(loadDetails: true);
 				$this->view->load_ok = true;
 			} catch (Exception) {
 				$this->view->load_ok = false;
diff --git a/app/Models/Feed.php b/app/Models/Feed.php
index 171b054ca..81765d433 100644
--- a/app/Models/Feed.php
+++ b/app/Models/Feed.php
@@ -552,6 +552,10 @@ class FreshRSS_Feed extends Minz_Model {
 					Minz_Exception::ERROR
 				);
 			} else {
+				if (($retryAfter = FreshRSS_http_Util::getRetryAfter($this->url)) > 0) {
+					throw new FreshRSS_Feed_Exception('For that domain, will first retry after ' . date('c', $retryAfter) .
+						'. ' . $this->url(includeCredentials: false), code: 503);
+				}
 				$simplePie = customSimplePie($this->attributes(), $this->curlOptions());
 				$url = htmlspecialchars_decode($this->url, ENT_QUOTES);
 				if (str_ends_with($url, '#force_feed')) {
@@ -571,15 +575,21 @@ class FreshRSS_Feed extends Minz_Model {
 				Minz_ExtensionManager::callHook('simplepie_after_init', $simplePie, $this, $simplePieResult);
 
 				if ($simplePieResult === false || $simplePie->get_hash() === '' || !empty($simplePie->error())) {
-					$errorMessage = $simplePie->error();
-					if (empty($errorMessage)) {
-						$errorMessage = '';
-					} elseif (is_array($errorMessage)) {
-						$errorMessage = json_encode($errorMessage, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_LINE_TERMINATORS) ?: '';
+					if ($simplePie->status_code() === 429) {
+						$errorMessage = 'HTTP 429 Too Many Requests!';
+					} elseif ($simplePie->status_code() === 503) {
+						$errorMessage = 'HTTP 503 Service Unavailable!';
+					} else {
+						$errorMessage = $simplePie->error();
+						if (empty($errorMessage)) {
+							$errorMessage = '';
+						} elseif (is_array($errorMessage)) {
+							$errorMessage = json_encode($errorMessage, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_LINE_TERMINATORS) ?: '';
+						}
 					}
 					throw new FreshRSS_Feed_Exception(
 						($errorMessage == '' ? 'Unknown error for feed' : $errorMessage) .
-							' [' . \SimplePie\Misc::url_remove_credentials($this->url) . ']',
+							' [' . $this->url(includeCredentials: false) . ']',
 						$simplePie->status_code()
 					);
 				}
@@ -701,7 +711,7 @@ class FreshRSS_Feed extends Minz_Model {
 		}
 
 		if ($invalidGuids > 0) {
-			Minz_Log::warning("Feed has {$invalidGuids} invalid GUIDs: " . $this->url);
+			Minz_Log::warning("Feed has {$invalidGuids} invalid GUIDs: " . $this->url(includeCredentials: false));
 			if (!$this->attributeBoolean('unicityCriteriaForced') && $invalidGuids > round($invalidGuidsTolerance * count($items))) {
 				$unicityCriteria = $this->attributeString('unicityCriteria');
 				if ($this->attributeBoolean('hasBadGuids')) {	// Legacy
@@ -719,7 +729,8 @@ class FreshRSS_Feed extends Minz_Model {
 				if ($newUnicityCriteria !== $unicityCriteria) {
 					$this->_attribute('hasBadGuids', null);	// Remove legacy
 					$this->_attribute('unicityCriteria', $newUnicityCriteria);
-					Minz_Log::warning('Feed unicity policy degraded (' . ($unicityCriteria ?: 'id') . ' → ' . $newUnicityCriteria . '): ' . $this->url);
+					Minz_Log::warning('Feed unicity policy degraded (' . ($unicityCriteria ?: 'id') . ' → ' . $newUnicityCriteria . '): ' .
+						$this->url(includeCredentials: false));
 					return $this->loadGuids($simplePie, $invalidGuidsTolerance);
 				}
 			}
@@ -1167,7 +1178,7 @@ class FreshRSS_Feed extends Minz_Model {
 			$affected = $feedDAO->markAsReadNotSeen($this->id(), $minLastSeen);
 		}
 		if ($affected > 0) {
-			Minz_Log::debug(__METHOD__ . " $affected items" . ($upstreamIsEmpty ? ' (all)' : '') . ' [' . $this->url(false) . ']');
+			Minz_Log::debug(__METHOD__ . " $affected items" . ($upstreamIsEmpty ? ' (all)' : '') . ' [' . $this->url(includeCredentials: false) . ']');
 		}
 		return $affected;
 	}
diff --git a/app/Models/SimplePieResponse.php b/app/Models/SimplePieResponse.php
index 17c954e8c..6a444a86a 100644
--- a/app/Models/SimplePieResponse.php
+++ b/app/Models/SimplePieResponse.php
@@ -4,7 +4,25 @@ declare(strict_types=1);
 final class FreshRSS_SimplePieResponse extends \SimplePie\File
 {
 	#[\Override]
-	protected function on_http_response(): void {
+	protected function on_http_response(string|false $response = ''): void {
 		syslog(LOG_INFO, 'FreshRSS SimplePie GET ' . $this->get_status_code() . ' ' . \SimplePie\Misc::url_remove_credentials($this->get_final_requested_uri()));
+
+		if (in_array($this->get_status_code(), [429, 503], true)) {
+			$parser = new \SimplePie\HTTP\Parser(is_string($response) ? $response : '');
+			if ($parser->parse()) {
+				$headers = $parser->headers;
+			} else {
+				$headers = [];
+			}
+
+			$retryAfter = FreshRSS_http_Util::setRetryAfter($this->get_final_requested_uri(), $headers['retry-after'] ?? '');
+			if ($retryAfter > 0) {
+				$domain = parse_url($this->get_final_requested_uri(), PHP_URL_HOST);
+				if (is_string($domain) && $domain !== '') {
+					$errorMessage = 'Will retry after ' . date('c', $retryAfter) . ' for domain `' . $domain . '`';
+					Minz_Log::notice($errorMessage);
+				}
+			}
+		}
 	}
 }
diff --git a/app/Utils/httpUtil.php b/app/Utils/httpUtil.php
new file mode 100644
index 000000000..12ebc3d07
--- /dev/null
+++ b/app/Utils/httpUtil.php
@@ -0,0 +1,74 @@
+<?php
+declare(strict_types=1);
+
+final class FreshRSS_http_Util {
+
+	private const RETRY_AFTER_PATH = DATA_PATH . '/Retry-After/';
+
+	/**
+	 * Clean up old Retry-After files
+	 */
+	private static function cleanRetryAfters(): void {
+		if (!is_dir(self::RETRY_AFTER_PATH)) {
+			return;
+		}
+		$files = glob(self::RETRY_AFTER_PATH . '*.txt', GLOB_NOSORT);
+		if ($files === false) {
+			return;
+		}
+		foreach ($files as $file) {
+			if (@filemtime($file) < time()) {
+				@unlink($file);
+			}
+		}
+	}
+
+	/**
+	 * Check whether the URL needs to wait for a Retry-After period.
+	 * @return int The timestamp of when the Retry-After expires, or 0 if not set.
+	 */
+	public static function getRetryAfter(string $url): int {
+		if (rand(0, 30) === 1) {	// Remove old files once in a while
+			self::cleanRetryAfters();
+		}
+		$domain = parse_url($url, PHP_URL_HOST);
+		if (!is_string($domain) || $domain === '') {
+			return 0;
+		}
+		$retryAfter = @filemtime(self::RETRY_AFTER_PATH . $domain . '.txt') ?: 0;
+		if ($retryAfter <= 0) {
+			return 0;
+		}
+		if ($retryAfter < time()) {
+			@unlink(self::RETRY_AFTER_PATH . $domain . '.txt');
+			return 0;
+		}
+		return $retryAfter;
+	}
+
+	/**
+	 * Store the HTTP Retry-After header value of an HTTP `429 Too Many Requests` or `503 Service Unavailable` response.
+	 */
+	public static function setRetryAfter(string $url, string $retryAfter): int {
+		$domain = parse_url($url, PHP_URL_HOST);
+		if (!is_string($domain) || $domain === '') {
+			return 0;
+		}
+
+		$limits = FreshRSS_Context::systemConf()->limits;
+		if (ctype_digit($retryAfter)) {
+			$retryAfter = time() + (int)$retryAfter;
+		} else {
+			$retryAfter = \SimplePie\Misc::parse_date($retryAfter) ?:
+				(time() + max(600, $limits['retry_after_default'] ?? 0));
+		}
+		$retryAfter = min($retryAfter, time() + max(3600, $limits['retry_after_max'] ?? 0));
+
+		@mkdir(self::RETRY_AFTER_PATH);
+		if (!touch(self::RETRY_AFTER_PATH . $domain . '.txt', $retryAfter)) {
+			Minz_Log::error('Failed to set Retry-After for ' . $domain);
+			return 0;
+		}
+		return $retryAfter;
+	}
+}