Better performance with yield (#2588)

* Better performance with yield

Largely decrease the time to first byte, and reduced memory consumtion.
Before, we used to make several copies in memory of the whole list of
articles before sending them to the client. Now streamed as they are
processed.

* Travis

9 files changed, 105 insertions, 108 deletions

diff --git a/app/Controllers/indexController.php b/app/Controllers/indexController.php
index 276d56acd..ddd51455d 100755
--- a/app/Controllers/indexController.php
+++ b/app/Controllers/indexController.php
@@ -32,6 +32,13 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			Minz_Error::error(404);
 		}
 
+		$this->_csp([
+			'default-src' => "'self'",
+			'frame-src' => '*',
+			'img-src' => '* data:',
+			'media-src' => '*',
+		]);
+
 		$this->view->categories = FreshRSS_Context::$categories;
 
 		$this->view->rss_title = FreshRSS_Context::$name . ' | ' . Minz_View::title();
@@ -54,42 +61,35 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			}
 		};
 
-		$this->view->callbackBeforePagination = function ($view) {
+		$this->view->callbackBeforeEntries = function ($view) {
 			try {
 				FreshRSS_Context::$number++;	//+1 for pagination
-				$entries = FreshRSS_index_Controller::listEntriesByContext();
+				$view->entries = FreshRSS_index_Controller::listEntriesByContext();
 				FreshRSS_Context::$number--;
-
-				$nb_entries = count($entries);
-				if ($nb_entries > FreshRSS_Context::$number) {
-					// We have more elements for pagination
-					$last_entry = array_pop($entries);
-					FreshRSS_Context::$next_id = $last_entry->id();
-				}
-
-				$first_entry = $nb_entries > 0 ? $entries[0] : null;
-				FreshRSS_Context::$id_max = $first_entry === null ? (time() - 1) . '000000' : $first_entry->id();
-				if (FreshRSS_Context::$order === 'ASC') {
-					// In this case we do not know but we guess id_max
-					$id_max = (time() - 1) . '000000';
-					if (strcmp($id_max, FreshRSS_Context::$id_max) > 0) {
-						FreshRSS_Context::$id_max = $id_max;
-					}
-				}
-
-				$view->entries = $entries;
+				ob_start();
 			} catch (FreshRSS_EntriesGetter_Exception $e) {
 				Minz_Log::notice($e->getMessage());
 				Minz_Error::error(404);
 			}
 		};
 
-		$this->_csp([
-			'default-src' => "'self'",
-			'frame-src' => '*',
-			'img-src' => '* data:',
-			'media-src' => '*',
-		]);
+		$this->view->callbackBeforePagination = function ($view, $nbEntries, $firstEntry, $lastEntry) {
+			if ($nbEntries >= FreshRSS_Context::$number) {
+				//We have enough entries: we discard the last one to use it for the next pagination
+				ob_clean();
+				FreshRSS_Context::$next_id = $lastEntry->id();
+			}
+			ob_end_flush();
+
+			FreshRSS_Context::$id_max = $firstEntry === null ? (time() - 1) . '000000' : $firstEntry->id();
+			if (FreshRSS_Context::$order === 'ASC') {
+				// In this case we do not know but we guess id_max
+				$id_max = (time() - 1) . '000000';
+				if (strcmp($id_max, FreshRSS_Context::$id_max) > 0) {
+					FreshRSS_Context::$id_max = $id_max;
+				}
+			}
+		};
 	}
 
 	/**
@@ -247,23 +247,13 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			$limit = FreshRSS_Context::$user_conf->max_posts_per_rss;
 		}
 
-		$entries = $entryDAO->listWhere(
-			$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
-			$limit, FreshRSS_Context::$first_id,
-			FreshRSS_Context::$search, $date_min
-		);
-
-		if (FreshRSS_Context::$sinceHours && (count($entries) < FreshRSS_Context::$user_conf->min_posts_per_rss)) {
-			$date_min = 0;
-			$limit = FreshRSS_Context::$user_conf->min_posts_per_rss;
-			$entries = $entryDAO->listWhere(
-				$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
-				$limit, FreshRSS_Context::$first_id,
-				FreshRSS_Context::$search, $date_min
-			);
+		foreach ($entryDAO->listWhere(
+					$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
+					$limit, FreshRSS_Context::$first_id,
+					FreshRSS_Context::$search, $date_min)
+				as $entry) {
+			yield $entry;
 		}
-
-		return $entries;
 	}
 
 	/**
diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php
index d2e7664fc..ebe530ec1 100644
--- a/app/Models/EntryDAO.php
+++ b/app/Models/EntryDAO.php
@@ -634,8 +634,7 @@ SQL;
 		$stm->bindParam(':guid', $guid);
 		$stm->execute();
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntries($res);
-		return isset($entries[0]) ? $entries[0] : null;
+		return isset($res[0]) ? self::daoToEntry($res[0]) : null;
 	}
 
 	public function searchById($id) {
@@ -647,8 +646,7 @@ SQL;
 		$stm->bindParam(':id', $id, PDO::PARAM_INT);
 		$stm->execute();
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntries($res);
-		return isset($entries[0]) ? $entries[0] : null;
+		return isset($res[0]) ? self::daoToEntry($res[0]) : null;
 	}
 
 	public function searchIdByGuid($id_feed, $guid) {
@@ -885,15 +883,17 @@ SQL;
 	public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filters = null, $date_min = 0) {
 		$stm = $this->listWhereRaw($type, $id, $state, $order, $limit, $firstId, $filters, $date_min);
 		if ($stm) {
-			return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC));
+			while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
+				yield self::daoToEntry($row);
+			}
 		} else {
-			return false;
+			yield false;
 		}
 	}
 
 	public function listByIds($ids, $order = 'DESC') {
 		if (count($ids) < 1) {
-			return array();
+			yield false;
 		}
 
 		$sql = 'SELECT id, guid, title, author, '
@@ -905,7 +905,9 @@ SQL;
 
 		$stm = $this->pdo->prepare($sql);
 		$stm->execute($ids);
-		return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC));
+		while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
+			yield self::daoToEntry($row);
+		}
 	}
 
 	public function listIdsWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filters = null) {	//For API
@@ -1058,20 +1060,4 @@ SQL;
 		}
 		return $entry;
 	}
-
-	private static function daoToEntries($listDAO) {
-		$list = array();
-
-		if (!is_array($listDAO)) {
-			$listDAO = array($listDAO);
-		}
-
-		foreach ($listDAO as $key => $dao) {
-			$list[] = self::daoToEntry($dao);
-		}
-
-		unset($listDAO);
-
-		return $list;
-	}
 }
diff --git a/app/layout/nav_menu.phtml b/app/layout/nav_menu.phtml
index da33d3e20..aae02c820 100644
--- a/app/layout/nav_menu.phtml
+++ b/app/layout/nav_menu.phtml
@@ -1,10 +1,6 @@
 <?php
 	$actual_view = Minz_Request::actionName();
-
 	flush();
-	if (isset($this->callbackBeforePagination)) {
-		call_user_func($this->callbackBeforePagination, $this);
-	}
 ?>
 
 <div class="nav_menu">
diff --git a/app/views/index/normal.phtml b/app/views/index/normal.phtml
index 23105df5c..50574ee4d 100644
--- a/app/views/index/normal.phtml
+++ b/app/views/index/normal.phtml
@@ -3,22 +3,32 @@
 $this->partial('aside_feed');
 $this->partial('nav_menu');
 
-if (!empty($this->entries)) {
-	$display_today = true;
-	$display_yesterday = true;
-	$display_others = true;
-	$hidePosts = !FreshRSS_Context::$user_conf->display_posts;
-	$lazyload = FreshRSS_Context::$user_conf->lazyload;
-	$content_width = FreshRSS_Context::$user_conf->content_width;
+call_user_func($this->callbackBeforeEntries, $this);
 
-	$today = @strtotime('today');
+$display_today = true;
+$display_yesterday = true;
+$display_others = true;
+$hidePosts = !FreshRSS_Context::$user_conf->display_posts;
+$lazyload = FreshRSS_Context::$user_conf->lazyload;
+$content_width = FreshRSS_Context::$user_conf->content_width;
+
+$today = @strtotime('today');
 ?>
 
-<div id="stream" class="normal<?= $hidePosts ? ' hide_posts' : '' ?>"><?php
-	?><div id="new-article">
+<div id="stream" class="normal<?= $hidePosts ? ' hide_posts' : '' ?>">
+	<div id="new-article">
 		<a href="<?= Minz_Url::display(Minz_Request::currentRequest()) ?>"><?= _t('gen.js.new_article'); /* TODO: move string in JS*/ ?></a>
 	</div><?php
-	foreach ($this->entries as $item) {
+	$firstEntry = null;
+	$lastEntry = null;
+	$nbEntries = 0;
+	foreach ($this->entries as $item):
+		if ($nbEntries === 0) {
+			$firstEntry = $item;
+		}
+		$lastEntry = $item;
+		$nbEntries++;
+		ob_flush();
 		$this->entry = Minz_ExtensionManager::callHook('entry_before_display', $item);
 		if ($this->entry == null) {
 			continue;
@@ -87,16 +97,19 @@ if (!empty($this->entries)) {
 
 		?></div>
 	</div><?php
-	}
-
-	$this->renderHelper('pagination');
-?></div>
+	endforeach;
 
-<?php if (FreshRSS_Context::$user_conf->show_nav_buttons) $this->partial('nav_entries'); ?>
-
-<?php } else { ?>
+	if ($nbEntries > 0):
+		call_user_func($this->callbackBeforePagination, $this, $nbEntries, $firstEntry, $lastEntry);
+		$this->renderHelper('pagination');
+?></div><?php
+	else:
+		ob_end_clean();	//Discard the articles headers, as we have no articles
+?>
 <div id="stream" class="prompt alert alert-warn normal">
 	<h2><?= _t('index.feed.empty') ?></h2>
 	<a href="<?= _url('subscription', 'index') ?>"><?= _t('index.feed.add') ?></a><br /><br />
 </div>
-<?php } ?>
+<?php endif; ?>
+
+<?php if ($nbEntries > 0 && FreshRSS_Context::$user_conf->show_nav_buttons) $this->partial('nav_entries'); ?>
diff --git a/app/views/index/reader.phtml b/app/views/index/reader.phtml
index e935db4a8..ee9e19327 100644
--- a/app/views/index/reader.phtml
+++ b/app/views/index/reader.phtml
@@ -2,18 +2,28 @@
 $this->partial('aside_feed');
 $this->partial('nav_menu');
 
-if (!empty($this->entries)) {
-	$lazyload = FreshRSS_Context::$user_conf->lazyload;
-	$content_width = FreshRSS_Context::$user_conf->content_width;
+call_user_func($this->callbackBeforeEntries, $this);
+
+$lazyload = FreshRSS_Context::$user_conf->lazyload;
+$content_width = FreshRSS_Context::$user_conf->content_width;
 ?>
 
 <div id="stream" class="reader">
 	<div id="new-article">
 		<a href="<?= Minz_Url::display(Minz_Request::currentRequest()) ?>"><?= _t('gen.js.new_article'); /* TODO: move string in JS*/ ?></a>
-	</div>
-	<?php foreach ($this->entries as $item) {
+	</div><?php
+	$firstEntry = null;
+	$lastEntry = null;
+	$nbEntries = 0;
+	foreach ($this->entries as $item):
+		if ($nbEntries === 0) {
+			$firstEntry = $item;
+		}
+		$lastEntry = $item;
+		$nbEntries++;
+		ob_flush();
 		$item = Minz_ExtensionManager::callHook('entry_before_display', $item);
-		if (is_null($item)) {
+		if ($item == null) {
 			continue;
 		}
 	?><div class="flux<?= !$item->isRead() ? ' not_read' : '' ?><?= $item->isFavorite() ? ' favorite' : '' ?>" id="flux_<?= $item->id() ?>">
@@ -61,15 +71,18 @@ if (!empty($this->entries)) {
 				<?= $item->content() ?>
 			</div>
 		</div>
-	</div>
-	<?php } ?>
+	</div><?php
+	endforeach;
 
-	<?php $this->renderHelper('pagination'); ?>
-</div>
-
-<?php } else { ?>
+	if ($nbEntries > 0):
+		call_user_func($this->callbackBeforePagination, $this, $nbEntries, $firstEntry, $lastEntry);
+		$this->renderHelper('pagination');
+?></div><?php
+	else:
+		ob_end_clean();	//Discard the articles headers, as we have no articles
+?>
 <div id="stream" class="prompt alert alert-warn reader">
 	<h2><?= _t('index.feed.empty') ?></h2>
 	<a href="<?= _url('subscription', 'index') ?>"><?= _t('index.feed.add') ?></a><br /><br />
 </div>
-<?php } ?>
+<?php endif; ?>
diff --git a/cli/README.md b/cli/README.md
index 4ef665e69..b4112bf63 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -47,7 +47,7 @@ cd /usr/share/FreshRSS
 ./cli/reconfigure.php
 # Same parameters as for do-install.php. Used to update an existing installation.
 
-./cli/create-user.php --user username [ --password 'password' --api_password 'api_password' --language en --email user@example.net --token 'longRandomString' --no_default_feeds --purge_after_months 3 --feed_min_articles_default 50 --feed_ttl_default 3600 --since_hours_posts_per_rss 168 --min_posts_per_rss 2 --max_posts_per_rss 400 ]
+./cli/create-user.php --user username [ --password 'password' --api_password 'api_password' --language en --email user@example.net --token 'longRandomString' --no_default_feeds --purge_after_months 3 --feed_min_articles_default 50 --feed_ttl_default 3600 --since_hours_posts_per_rss 168 --max_posts_per_rss 400 ]
 # --language can be: 'en' (default), 'fr', or one of the [supported languages](../app/i18n/)
 
 ./cli/update-user.php --user username [ ... ]
diff --git a/cli/_update-or-create-user.php b/cli/_update-or-create-user.php
index 3217a07ea..0da9963f0 100644
--- a/cli/_update-or-create-user.php
+++ b/cli/_update-or-create-user.php
@@ -11,7 +11,6 @@ $params = array(
 		'feed_min_articles_default:',
 		'feed_ttl_default:',
 		'since_hours_posts_per_rss:',
-		'min_posts_per_rss:',
 		'max_posts_per_rss:',
 	);
 
@@ -27,7 +26,7 @@ if (!validateOptions($argv, $params) || empty($options['user'])) {
 		" --language en --email user@example.net --token 'longRandomString'" .
 		($isUpdate ? '' : '--no_default_feeds') .
 		" --purge_after_months 3 --feed_min_articles_default 50 --feed_ttl_default 3600" .
-		" --since_hours_posts_per_rss 168 --min_posts_per_rss 2 --max_posts_per_rss 400 )");
+		" --since_hours_posts_per_rss 168 --max_posts_per_rss 400 )");
 }
 
 function strParam($name) {
@@ -48,7 +47,6 @@ $values = array(
 		'keep_history_default' => intParam('feed_min_articles_default'),	//TODO: Update with new mechanism
 		'ttl_default' => intParam('feed_ttl_default'),
 		'since_hours_posts_per_rss' => intParam('since_hours_posts_per_rss'),
-		'min_posts_per_rss' => intParam('min_posts_per_rss'),
 		'max_posts_per_rss' => intParam('max_posts_per_rss'),
 	);
 
diff --git a/config-user.default.php b/config-user.default.php
index 3028b709a..a814fe7c7 100644
--- a/config-user.default.php
+++ b/config-user.default.php
@@ -24,7 +24,6 @@ return array (
 	'feverKey' => '',
 	'posts_per_page' => 20,
 	'since_hours_posts_per_rss' => 168,
-	'min_posts_per_rss' => 2,
 	'max_posts_per_rss' => 400,
 	'view_mode' => 'normal',
 	'default_view' => 'adaptive',
diff --git a/p/api/greader.php b/p/api/greader.php
index c63ed12fa..6da11ec5f 100644
--- a/p/api/greader.php
+++ b/p/api/greader.php
@@ -638,6 +638,7 @@ function streamContents($path, $include_target, $start_time, $stop_time, $count,
 
 	$entryDAO = FreshRSS_Factory::createEntryDao();
 	$entries = $entryDAO->listWhere($type, $include_target, $state, $order === 'o' ? 'ASC' : 'DESC', $count, $continuation, $searches);
+	$entries = iterator_to_array($entries);	//TODO: Improve
 
 	$items = entriesToArray($entries);
 
@@ -730,6 +731,7 @@ function streamContentsItems($e_ids, $order) {
 
 	$entryDAO = FreshRSS_Factory::createEntryDao();
 	$entries = $entryDAO->listByIds($e_ids, $order === 'o' ? 'ASC' : 'DESC');
+	$entries = iterator_to_array($entries);	//TODO: Improve
 
 	$items = entriesToArray($entries);