Better performance with yield (#2588)

* Better performance with yield

Largely decrease the time to first byte, and reduced memory consumtion.
Before, we used to make several copies in memory of the whole list of
articles before sending them to the client. Now streamed as they are
processed.

* Travis

5 files changed, 101 insertions, 103 deletions

diff --git a/app/Controllers/indexController.php b/app/Controllers/indexController.php
index 276d56acd..ddd51455d 100755
--- a/app/Controllers/indexController.php
+++ b/app/Controllers/indexController.php
@@ -32,6 +32,13 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			Minz_Error::error(404);
 		}
 
+		$this->_csp([
+			'default-src' => "'self'",
+			'frame-src' => '*',
+			'img-src' => '* data:',
+			'media-src' => '*',
+		]);
+
 		$this->view->categories = FreshRSS_Context::$categories;
 
 		$this->view->rss_title = FreshRSS_Context::$name . ' | ' . Minz_View::title();
@@ -54,42 +61,35 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			}
 		};
 
-		$this->view->callbackBeforePagination = function ($view) {
+		$this->view->callbackBeforeEntries = function ($view) {
 			try {
 				FreshRSS_Context::$number++;	//+1 for pagination
-				$entries = FreshRSS_index_Controller::listEntriesByContext();
+				$view->entries = FreshRSS_index_Controller::listEntriesByContext();
 				FreshRSS_Context::$number--;
-
-				$nb_entries = count($entries);
-				if ($nb_entries > FreshRSS_Context::$number) {
-					// We have more elements for pagination
-					$last_entry = array_pop($entries);
-					FreshRSS_Context::$next_id = $last_entry->id();
-				}
-
-				$first_entry = $nb_entries > 0 ? $entries[0] : null;
-				FreshRSS_Context::$id_max = $first_entry === null ? (time() - 1) . '000000' : $first_entry->id();
-				if (FreshRSS_Context::$order === 'ASC') {
-					// In this case we do not know but we guess id_max
-					$id_max = (time() - 1) . '000000';
-					if (strcmp($id_max, FreshRSS_Context::$id_max) > 0) {
-						FreshRSS_Context::$id_max = $id_max;
-					}
-				}
-
-				$view->entries = $entries;
+				ob_start();
 			} catch (FreshRSS_EntriesGetter_Exception $e) {
 				Minz_Log::notice($e->getMessage());
 				Minz_Error::error(404);
 			}
 		};
 
-		$this->_csp([
-			'default-src' => "'self'",
-			'frame-src' => '*',
-			'img-src' => '* data:',
-			'media-src' => '*',
-		]);
+		$this->view->callbackBeforePagination = function ($view, $nbEntries, $firstEntry, $lastEntry) {
+			if ($nbEntries >= FreshRSS_Context::$number) {
+				//We have enough entries: we discard the last one to use it for the next pagination
+				ob_clean();
+				FreshRSS_Context::$next_id = $lastEntry->id();
+			}
+			ob_end_flush();
+
+			FreshRSS_Context::$id_max = $firstEntry === null ? (time() - 1) . '000000' : $firstEntry->id();
+			if (FreshRSS_Context::$order === 'ASC') {
+				// In this case we do not know but we guess id_max
+				$id_max = (time() - 1) . '000000';
+				if (strcmp($id_max, FreshRSS_Context::$id_max) > 0) {
+					FreshRSS_Context::$id_max = $id_max;
+				}
+			}
+		};
 	}
 
 	/**
@@ -247,23 +247,13 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			$limit = FreshRSS_Context::$user_conf->max_posts_per_rss;
 		}
 
-		$entries = $entryDAO->listWhere(
-			$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
-			$limit, FreshRSS_Context::$first_id,
-			FreshRSS_Context::$search, $date_min
-		);
-
-		if (FreshRSS_Context::$sinceHours && (count($entries) < FreshRSS_Context::$user_conf->min_posts_per_rss)) {
-			$date_min = 0;
-			$limit = FreshRSS_Context::$user_conf->min_posts_per_rss;
-			$entries = $entryDAO->listWhere(
-				$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
-				$limit, FreshRSS_Context::$first_id,
-				FreshRSS_Context::$search, $date_min
-			);
+		foreach ($entryDAO->listWhere(
+					$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
+					$limit, FreshRSS_Context::$first_id,
+					FreshRSS_Context::$search, $date_min)
+				as $entry) {
+			yield $entry;
 		}
-
-		return $entries;
 	}
 
 	/**
diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php
index d2e7664fc..ebe530ec1 100644
--- a/app/Models/EntryDAO.php
+++ b/app/Models/EntryDAO.php
@@ -634,8 +634,7 @@ SQL;
 		$stm->bindParam(':guid', $guid);
 		$stm->execute();
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntries($res);
-		return isset($entries[0]) ? $entries[0] : null;
+		return isset($res[0]) ? self::daoToEntry($res[0]) : null;
 	}
 
 	public function searchById($id) {
@@ -647,8 +646,7 @@ SQL;
 		$stm->bindParam(':id', $id, PDO::PARAM_INT);
 		$stm->execute();
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntries($res);
-		return isset($entries[0]) ? $entries[0] : null;
+		return isset($res[0]) ? self::daoToEntry($res[0]) : null;
 	}
 
 	public function searchIdByGuid($id_feed, $guid) {
@@ -885,15 +883,17 @@ SQL;
 	public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filters = null, $date_min = 0) {
 		$stm = $this->listWhereRaw($type, $id, $state, $order, $limit, $firstId, $filters, $date_min);
 		if ($stm) {
-			return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC));
+			while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
+				yield self::daoToEntry($row);
+			}
 		} else {
-			return false;
+			yield false;
 		}
 	}
 
 	public function listByIds($ids, $order = 'DESC') {
 		if (count($ids) < 1) {
-			return array();
+			yield false;
 		}
 
 		$sql = 'SELECT id, guid, title, author, '
@@ -905,7 +905,9 @@ SQL;
 
 		$stm = $this->pdo->prepare($sql);
 		$stm->execute($ids);
-		return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC));
+		while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
+			yield self::daoToEntry($row);
+		}
 	}
 
 	public function listIdsWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filters = null) {	//For API
@@ -1058,20 +1060,4 @@ SQL;
 		}
 		return $entry;
 	}
-
-	private static function daoToEntries($listDAO) {
-		$list = array();
-
-		if (!is_array($listDAO)) {
-			$listDAO = array($listDAO);
-		}
-
-		foreach ($listDAO as $key => $dao) {
-			$list[] = self::daoToEntry($dao);
-		}
-
-		unset($listDAO);
-
-		return $list;
-	}
 }
diff --git a/app/layout/nav_menu.phtml b/app/layout/nav_menu.phtml
index da33d3e20..aae02c820 100644
--- a/app/layout/nav_menu.phtml
+++ b/app/layout/nav_menu.phtml
@@ -1,10 +1,6 @@
 <?php
 	$actual_view = Minz_Request::actionName();
-
 	flush();
-	if (isset($this->callbackBeforePagination)) {
-		call_user_func($this->callbackBeforePagination, $this);
-	}
 ?>
 
 <div class="nav_menu">
diff --git a/app/views/index/normal.phtml b/app/views/index/normal.phtml
index 23105df5c..50574ee4d 100644
--- a/app/views/index/normal.phtml
+++ b/app/views/index/normal.phtml
@@ -3,22 +3,32 @@
 $this->partial('aside_feed');
 $this->partial('nav_menu');
 
-if (!empty($this->entries)) {
-	$display_today = true;
-	$display_yesterday = true;
-	$display_others = true;
-	$hidePosts = !FreshRSS_Context::$user_conf->display_posts;
-	$lazyload = FreshRSS_Context::$user_conf->lazyload;
-	$content_width = FreshRSS_Context::$user_conf->content_width;
+call_user_func($this->callbackBeforeEntries, $this);
 
-	$today = @strtotime('today');
+$display_today = true;
+$display_yesterday = true;
+$display_others = true;
+$hidePosts = !FreshRSS_Context::$user_conf->display_posts;
+$lazyload = FreshRSS_Context::$user_conf->lazyload;
+$content_width = FreshRSS_Context::$user_conf->content_width;
+
+$today = @strtotime('today');
 ?>
 
-<div id="stream" class="normal<?= $hidePosts ? ' hide_posts' : '' ?>"><?php
-	?><div id="new-article">
+<div id="stream" class="normal<?= $hidePosts ? ' hide_posts' : '' ?>">
+	<div id="new-article">
 		<a href="<?= Minz_Url::display(Minz_Request::currentRequest()) ?>"><?= _t('gen.js.new_article'); /* TODO: move string in JS*/ ?></a>
 	</div><?php
-	foreach ($this->entries as $item) {
+	$firstEntry = null;
+	$lastEntry = null;
+	$nbEntries = 0;
+	foreach ($this->entries as $item):
+		if ($nbEntries === 0) {
+			$firstEntry = $item;
+		}
+		$lastEntry = $item;
+		$nbEntries++;
+		ob_flush();
 		$this->entry = Minz_ExtensionManager::callHook('entry_before_display', $item);
 		if ($this->entry == null) {
 			continue;
@@ -87,16 +97,19 @@ if (!empty($this->entries)) {
 
 		?></div>
 	</div><?php
-	}
-
-	$this->renderHelper('pagination');
-?></div>
+	endforeach;
 
-<?php if (FreshRSS_Context::$user_conf->show_nav_buttons) $this->partial('nav_entries'); ?>
-
-<?php } else { ?>
+	if ($nbEntries > 0):
+		call_user_func($this->callbackBeforePagination, $this, $nbEntries, $firstEntry, $lastEntry);
+		$this->renderHelper('pagination');
+?></div><?php
+	else:
+		ob_end_clean();	//Discard the articles headers, as we have no articles
+?>
 <div id="stream" class="prompt alert alert-warn normal">
 	<h2><?= _t('index.feed.empty') ?></h2>
 	<a href="<?= _url('subscription', 'index') ?>"><?= _t('index.feed.add') ?></a><br /><br />
 </div>
-<?php } ?>
+<?php endif; ?>
+
+<?php if ($nbEntries > 0 && FreshRSS_Context::$user_conf->show_nav_buttons) $this->partial('nav_entries'); ?>
diff --git a/app/views/index/reader.phtml b/app/views/index/reader.phtml
index e935db4a8..ee9e19327 100644
--- a/app/views/index/reader.phtml
+++ b/app/views/index/reader.phtml
@@ -2,18 +2,28 @@
 $this->partial('aside_feed');
 $this->partial('nav_menu');
 
-if (!empty($this->entries)) {
-	$lazyload = FreshRSS_Context::$user_conf->lazyload;
-	$content_width = FreshRSS_Context::$user_conf->content_width;
+call_user_func($this->callbackBeforeEntries, $this);
+
+$lazyload = FreshRSS_Context::$user_conf->lazyload;
+$content_width = FreshRSS_Context::$user_conf->content_width;
 ?>
 
 <div id="stream" class="reader">
 	<div id="new-article">
 		<a href="<?= Minz_Url::display(Minz_Request::currentRequest()) ?>"><?= _t('gen.js.new_article'); /* TODO: move string in JS*/ ?></a>
-	</div>
-	<?php foreach ($this->entries as $item) {
+	</div><?php
+	$firstEntry = null;
+	$lastEntry = null;
+	$nbEntries = 0;
+	foreach ($this->entries as $item):
+		if ($nbEntries === 0) {
+			$firstEntry = $item;
+		}
+		$lastEntry = $item;
+		$nbEntries++;
+		ob_flush();
 		$item = Minz_ExtensionManager::callHook('entry_before_display', $item);
-		if (is_null($item)) {
+		if ($item == null) {
 			continue;
 		}
 	?><div class="flux<?= !$item->isRead() ? ' not_read' : '' ?><?= $item->isFavorite() ? ' favorite' : '' ?>" id="flux_<?= $item->id() ?>">
@@ -61,15 +71,18 @@ if (!empty($this->entries)) {
 				<?= $item->content() ?>
 			</div>
 		</div>
-	</div>
-	<?php } ?>
+	</div><?php
+	endforeach;
 
-	<?php $this->renderHelper('pagination'); ?>
-</div>
-
-<?php } else { ?>
+	if ($nbEntries > 0):
+		call_user_func($this->callbackBeforePagination, $this, $nbEntries, $firstEntry, $lastEntry);
+		$this->renderHelper('pagination');
+?></div><?php
+	else:
+		ob_end_clean();	//Discard the articles headers, as we have no articles
+?>
 <div id="stream" class="prompt alert alert-warn reader">
 	<h2><?= _t('index.feed.empty') ?></h2>
 	<a href="<?= _url('subscription', 'index') ?>"><?= _t('index.feed.add') ?></a><br /><br />
 </div>
-<?php } ?>
+<?php endif; ?>