Stream JSON export

Avoid large in-memory copies
https://github.com/FreshRSS/FreshRSS/issues/1372

3 files changed, 78 insertions, 60 deletions

diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php
index 3ba91a243..d86587ea9 100644
--- a/app/Controllers/importExportController.php
+++ b/app/Controllers/importExportController.php
@@ -641,13 +641,13 @@ class FreshRSS_importExport_Controller extends Minz_ActionController {
 			$this->view->list_title = _t('sub.import_export.starred_list');
 			$this->view->type = 'starred';
 			$unread_fav = $this->entryDAO->countUnreadReadFavorites();
-			$this->view->entries = $this->entryDAO->listWhere(
+			$this->view->entriesRaw = $this->entryDAO->listWhereRaw(
 				's', '', FreshRSS_Entry::STATE_ALL, 'ASC', $unread_fav['all']
 			);
 		} elseif ($type === 'feed' && $feed != null) {
 			$this->view->list_title = _t('sub.import_export.feed_list', $feed->name());
 			$this->view->type = 'feed/' . $feed->id();
-			$this->view->entries = $this->entryDAO->listWhere(
+			$this->view->entriesRaw = $this->entryDAO->listWhereRaw(
 				'f', $feed->id(), FreshRSS_Entry::STATE_ALL, 'ASC',
 				$maxFeedEntries
 			);
diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php
index d8a4a486d..397471baa 100644
--- a/app/Models/EntryDAO.php
+++ b/app/Models/EntryDAO.php
@@ -518,7 +518,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 
 		$stm->execute($values);
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntry($res);
+		$entries = self::daoToEntries($res);
 		return isset($entries[0]) ? $entries[0] : null;
 	}
 
@@ -533,7 +533,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 
 		$stm->execute($values);
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntry($res);
+		$entries = self::daoToEntries($res);
 		return isset($entries[0]) ? $entries[0] : null;
 	}
 
@@ -666,7 +666,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 			. ($limit > 0 ? ' LIMIT ' . $limit : ''));	//TODO: See http://explainextended.com/2009/10/23/mysql-order-by-limit-performance-late-row-lookups/
 	}
 
-	public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) {
+	public function listWhereRaw($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) {
 		list($values, $sql) = $this->sqlListWhere($type, $id, $state, $order, $limit, $firstId, $filter, $date_min);
 
 		$sql = 'SELECT e0.id, e0.guid, e0.title, e0.author, '
@@ -680,8 +680,12 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 
 		$stm = $this->bd->prepare($sql);
 		$stm->execute($values);
+		return $stm;
+	}
 
-		return self::daoToEntry($stm->fetchAll(PDO::FETCH_ASSOC));
+	public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) {
+		$stm = $this->listWhereRaw($type, $id, $state, $order, $limit, $firstId, $filter, $date_min);
+		return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC));
 	}
 
 	public function listIdsWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) {	//For API
@@ -810,15 +814,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return $res[0];
 	}
 
-	public static function daoToEntry($listDAO) {
-		$list = array();
-
-		if (!is_array($listDAO)) {
-			$listDAO = array($listDAO);
-		}
-
-		foreach ($listDAO as $key => $dao) {
-			$entry = new FreshRSS_Entry(
+	public static function daoToEntry($dao) {
+		$entry = new FreshRSS_Entry(
 				$dao['id_feed'],
 				$dao['guid'],
 				$dao['title'],
@@ -830,10 +827,21 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 				$dao['is_favorite'],
 				$dao['tags']
 			);
-			if (isset($dao['id'])) {
-				$entry->_id($dao['id']);
-			}
-			$list[] = $entry;
+		if (isset($dao['id'])) {
+			$entry->_id($dao['id']);
+		}
+		return $entry;
+	}
+
+	private static function daoToEntries($listDAO) {
+		$list = array();
+
+		if (!is_array($listDAO)) {
+			$listDAO = array($listDAO);
+		}
+
+		foreach ($listDAO as $key => $dao) {
+			$list[] = self::daoToEntry($dao);
 		}
 
 		unset($listDAO);
diff --git a/app/views/helpers/export/articles.phtml b/app/views/helpers/export/articles.phtml
index ffdca1daa..2ad4a3abf 100644
--- a/app/views/helpers/export/articles.phtml
+++ b/app/views/helpers/export/articles.phtml
@@ -1,47 +1,57 @@
 <?php
-    $username = Minz_Session::param('currentUser', '_');
+	$username = Minz_Session::param('currentUser', '_');
 
-    $articles = array(
-        'id' => 'user/' . str_replace('/', '', $username) . '/state/org.freshrss/' . $this->type,
-        'title' => $this->list_title,
-        'author' => $username,
-        'items' => array()
-    );
+	$options = 0;
+	if (version_compare(PHP_VERSION, '5.4.0') >= 0) {
+		$options = JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
+	}
 
-    foreach ($this->entries as $entry) {
-        if (!isset($this->feed)) {
-            $feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed ());
-        } else {
-            $feed = $this->feed;
-        }
+	$articles = array(
+		'id' => 'user/' . str_replace('/', '', $username) . '/state/org.freshrss/' . $this->type,
+		'title' => $this->list_title,
+		'author' => $username,
+		'items' => array(),
+	);
 
-        $articles['items'][] = array(
-            'id' => $entry->guid(),
-            'categories' => array_values($entry->tags()),
-            'title' => $entry->title(),
-            'author' => $entry->author(),
-            'published' => $entry->date(true),
-            'updated' => $entry->date(true),
-            'alternate' => array(array(
-                'href' => $entry->link(),
-                'type' => 'text/html'
-            )),
-            'content' => array(
-                'content' => $entry->content()
-            ),
-            'origin' => array(
-                'streamId' => $feed->id(),
-                'title' => $feed->name(),
-                'htmlUrl' => $feed->website(),
-                'feedUrl' => $feed->url()
-            )
-        );
-    }
+	echo rtrim(json_encode($articles, $options), " ]}\n\r\t");
+	$first = true;
 
-    $options = 0;
-    if (version_compare(PHP_VERSION, '5.4.0') >= 0) {
-        $options = JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
-    }
+	foreach ($this->entriesRaw as $entryRaw) {
+		$entry = FreshRSS_EntryDAO::daoToEntry($entryRaw);
+		if (!isset($this->feed)) {
+			$feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed ());
+		} else {
+			$feed = $this->feed;
+		}
 
-    echo json_encode($articles, $options);
-?>
+		$article = array(
+			'id' => $entry->guid(),
+			'categories' => array_values($entry->tags()),
+			'title' => $entry->title(),
+			'author' => $entry->author(),
+			'published' => $entry->date(true),
+			'updated' => $entry->date(true),
+			'alternate' => array(array(
+				'href' => $entry->link(),
+				'type' => 'text/html',
+			)),
+			'content' => array(
+				'content' => $entry->content(),
+			),
+			'origin' => array(
+				'streamId' => $feed->id(),
+				'title' => $feed->name(),
+				'htmlUrl' => $feed->website(),
+				'feedUrl' => $feed->url(),
+			)
+		);
+
+		if ($first) {
+			$first = false;
+		} else {
+			echo ",\n";
+		}
+		echo json_encode($article, $options);
+	}
+
+	echo "\n]}\n";