Stream instead of memory copy of SimplePie entries (#2972)

* Stream instead of memory copy of SimplePie entries

https://github.com/FreshRSS/FreshRSS/issues/2952

* Undo lines delete

* Typo

* Remove unaccessible code

https://github.com/FreshRSS/FreshRSS/pull/2972/files#r425624163

* Back-compatibility for Feed->entries

https://github.com/FreshRSS/FreshRSS/pull/2972/files#r425631913

1 files changed, 58 insertions, 51 deletions

diff --git a/app/Models/Feed.php b/app/Models/Feed.php
index 1b9bd6f9f..2974b8eb9 100644
--- a/app/Models/Feed.php
+++ b/app/Models/Feed.php
@@ -15,7 +15,6 @@ class FreshRSS_Feed extends Minz_Model {
 	private $category = 1;
 	private $nbEntries = -1;
 	private $nbNotRead = -1;
-	private $entries = null;
 	private $name = '';
 	private $website = '';
 	private $description = '';
@@ -72,7 +71,9 @@ class FreshRSS_Feed extends Minz_Model {
 		return $this->category;
 	}
 	public function entries() {
-		return $this->entries === null ? array() : $this->entries;
+		Minz_Log::warning(__method__ . ' is deprecated since FreshRSS 1.16.1!');
+		$simplePie = $this->load(false, true);
+		return $simplePie == null ? [] : iterator_to_array($this->loadEntries($simplePie));
 	}
 	public function name() {
 		return $this->name;
@@ -267,46 +268,47 @@ class FreshRSS_Feed extends Minz_Model {
 				if ($this->httpAuth != '') {
 					$url = preg_replace('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $url);
 				}
-				$feed = customSimplePie($this->attributes());
+				$simplePie = customSimplePie($this->attributes());
 				if (substr($url, -11) === '#force_feed') {
-					$feed->force_feed(true);
+					$simplePie->force_feed(true);
 					$url = substr($url, 0, -11);
 				}
-				$feed->set_feed_url($url);
+				$simplePie->set_feed_url($url);
 				if (!$loadDetails) {	//Only activates auto-discovery when adding a new feed
-					$feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE);
+					$simplePie->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE);
 				}
 				if ($this->attributes('clear_cache')) {
 					// Do not use `$simplePie->enable_cache(false);` as it would prevent caching in multiuser context
 					$this->clearCache();
 				}
-				Minz_ExtensionManager::callHook('simplepie_before_init', $feed, $this);
-				$mtime = $feed->init();
+				Minz_ExtensionManager::callHook('simplepie_before_init', $simplePie, $this);
+				$mtime = $simplePie->init();
 
-				if ((!$mtime) || $feed->error()) {
-					$errorMessage = $feed->error();
+				if ((!$mtime) || $simplePie->error()) {
+					$errorMessage = $simplePie->error();
 					throw new FreshRSS_Feed_Exception(
 						($errorMessage == '' ? 'Unknown error for feed' : $errorMessage) . ' [' . $url . ']'
 					);
 				}
 
-				$links = $feed->get_links('self');
+				$links = $simplePie->get_links('self');
 				$this->selfUrl = isset($links[0]) ? $links[0] : null;
-				$links = $feed->get_links('hub');
+				$links = $simplePie->get_links('hub');
 				$this->hubUrl = isset($links[0]) ? $links[0] : null;
 
 				if ($loadDetails) {
 					// si on a utilisé l'auto-discover, notre url va avoir changé
-					$subscribe_url = $feed->subscribe_url(false);
+					$subscribe_url = $simplePie->subscribe_url(false);
 
-					$title = strtr(html_only_entity_decode($feed->get_title()), array('<' => '&lt;', '>' => '&gt;', '"' => '&quot;'));	//HTML to HTML-PRE	//ENT_COMPAT except &
+					//HTML to HTML-PRE	//ENT_COMPAT except '&'
+					$title = strtr(html_only_entity_decode($simplePie->get_title()), array('<' => '&lt;', '>' => '&gt;', '"' => '&quot;'));
 					$this->_name($title == '' ? $url : $title);
 
-					$this->_website(html_only_entity_decode($feed->get_link()));
-					$this->_description(html_only_entity_decode($feed->get_description()));
+					$this->_website(html_only_entity_decode($simplePie->get_link()));
+					$this->_description(html_only_entity_decode($simplePie->get_description()));
 				} else {
 					//The case of HTTP 301 Moved Permanently
-					$subscribe_url = $feed->subscribe_url(true);
+					$subscribe_url = $simplePie->subscribe_url(true);
 				}
 
 				$clean_url = SimplePie_Misc::url_remove_credentials($subscribe_url);
@@ -316,26 +318,51 @@ class FreshRSS_Feed extends Minz_Model {
 
 				if (($mtime === true) || ($mtime > $this->lastUpdate) || $noCache) {
 					//Minz_Log::debug('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url);
-					$this->loadEntries($feed);	// et on charge les articles du flux
+					return $simplePie;
 				} else {
 					//Minz_Log::debug('FreshRSS use cache for ' . $clean_url);
-					$this->entries = array();
+					return null;
 				}
-
-				$feed->__destruct();	//http://simplepie.org/wiki/faq/i_m_getting_memory_leaks
-				unset($feed);
 			}
 		}
 	}
 
-	public function loadEntries($feed) {
-		$entries = array();
-		$guids = array();
+	public function loadGuids($simplePie) {
 		$hasUniqueGuids = true;
+		$testGuids = [];
+		$guids = [];
+		$hasBadGuids = $this->attributes('hasBadGuids');
+
+		for ($i = $simplePie->get_item_quantity() - 1; $i >= 0; $i--) {
+			$item = $simplePie->get_item($i);
+			if ($item == null) {
+				continue;
+			}
+			$guid = safe_ascii($item->get_id(false, false));
+			$hasUniqueGuids &= empty($testGuids['_' . $guid]);
+			$testGuids['_' . $guid] = true;
+			$guids[] = $guid;
+		}
+
+		if ($hasBadGuids != !$hasUniqueGuids) {
+			$hasBadGuids = !$hasUniqueGuids;
+			if ($hasBadGuids) {
+				Minz_Log::warning('Feed has invalid GUIDs: ' . $this->url);
+			} else {
+				Minz_Log::warning('Feed has valid GUIDs again: ' . $this->url);
+			}
+			$feedDAO = FreshRSS_Factory::createFeedDao();
+			$feedDAO->updateFeedAttribute($this, 'hasBadGuids', $hasBadGuids);
+		}
+		return $guids;
+	}
+
+	public function loadEntries($simplePie) {
+		$hasBadGuids = $this->attributes('hasBadGuids');
 
 		// We want chronological order and SimplePie uses reverse order.
-		for ($i = $feed->get_item_quantity() - 1; $i >= 0; $i--) {
-			$item = $feed->get_item($i);
+		for ($i = $simplePie->get_item_quantity() - 1; $i >= 0; $i--) {
+			$item = $simplePie->get_item($i);
 			if ($item == null) {
 				continue;
 			}
@@ -418,10 +445,9 @@ class FreshRSS_Feed extends Minz_Model {
 				}
 			}
 
-			$guid = $item->get_id(false, false);
+			$guid = safe_ascii($item->get_id(false, false));
 			unset($item);
-			$hasUniqueGuids &= empty($guids['_' . $guid]);
-			$guids['_' . $guid] = true;
+
 			$author_names = '';
 			if (is_array($authors)) {
 				foreach ($authors as $author) {
@@ -432,7 +458,7 @@ class FreshRSS_Feed extends Minz_Model {
 
 			$entry = new FreshRSS_Entry(
 				$this->id(),
-				$guid,
+				$hasBadGuids ? '' : $guid,
 				$title === null ? '' : $title,
 				$author_names,
 				$content === null ? '' : $content,
@@ -446,27 +472,8 @@ class FreshRSS_Feed extends Minz_Model {
 				$entry->loadCompleteContent();
 			}
 
-			$entries[] = $entry;
-		}
-
-		$hasBadGuids = $this->attributes('hasBadGuids');
-		if ($hasBadGuids != !$hasUniqueGuids) {
-			$hasBadGuids = !$hasUniqueGuids;
-			if ($hasBadGuids) {
-				Minz_Log::warning('Feed has invalid GUIDs: ' . $this->url);
-			} else {
-				Minz_Log::warning('Feed has valid GUIDs again: ' . $this->url);
-			}
-			$feedDAO = FreshRSS_Factory::createFeedDao();
-			$feedDAO->updateFeedAttribute($this, 'hasBadGuids', $hasBadGuids);
-		}
-		if (!$hasUniqueGuids) {
-			foreach ($entries as $entry) {
-				$entry->_guid('');
-			}
+			yield $entry;
 		}
-
-		$this->entries = $entries;
 	}
 
 	public function cleanOldEntries() {	//Remember to call updateCachedValue($id_feed) or updateCachedValues() just after