From f59de4e2b679efb1ba4bbfd576a5ea07422cc169 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Tue, 15 Nov 2016 20:43:06 +0100 Subject: Stream JSON export Avoid large in-memory copies https://github.com/FreshRSS/FreshRSS/issues/1372 --- app/Controllers/importExportController.php | 4 +- app/Models/EntryDAO.php | 42 ++++++++------ app/views/helpers/export/articles.phtml | 92 +++++++++++++++++------------- 3 files changed, 78 insertions(+), 60 deletions(-) diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index 3ba91a243..d86587ea9 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -641,13 +641,13 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { $this->view->list_title = _t('sub.import_export.starred_list'); $this->view->type = 'starred'; $unread_fav = $this->entryDAO->countUnreadReadFavorites(); - $this->view->entries = $this->entryDAO->listWhere( + $this->view->entriesRaw = $this->entryDAO->listWhereRaw( 's', '', FreshRSS_Entry::STATE_ALL, 'ASC', $unread_fav['all'] ); } elseif ($type === 'feed' && $feed != null) { $this->view->list_title = _t('sub.import_export.feed_list', $feed->name()); $this->view->type = 'feed/' . $feed->id(); - $this->view->entries = $this->entryDAO->listWhere( + $this->view->entriesRaw = $this->entryDAO->listWhereRaw( 'f', $feed->id(), FreshRSS_Entry::STATE_ALL, 'ASC', $maxFeedEntries ); diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index d8a4a486d..397471baa 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -518,7 +518,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $stm->execute($values); $res = $stm->fetchAll(PDO::FETCH_ASSOC); - $entries = self::daoToEntry($res); + $entries = self::daoToEntries($res); return isset($entries[0]) ? $entries[0] : null; } @@ -533,7 +533,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $stm->execute($values); $res = $stm->fetchAll(PDO::FETCH_ASSOC); - $entries = self::daoToEntry($res); + $entries = self::daoToEntries($res); return isset($entries[0]) ? $entries[0] : null; } @@ -666,7 +666,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { . ($limit > 0 ? ' LIMIT ' . $limit : '')); //TODO: See http://explainextended.com/2009/10/23/mysql-order-by-limit-performance-late-row-lookups/ } - public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) { + public function listWhereRaw($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) { list($values, $sql) = $this->sqlListWhere($type, $id, $state, $order, $limit, $firstId, $filter, $date_min); $sql = 'SELECT e0.id, e0.guid, e0.title, e0.author, ' @@ -680,8 +680,12 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $stm = $this->bd->prepare($sql); $stm->execute($values); + return $stm; + } - return self::daoToEntry($stm->fetchAll(PDO::FETCH_ASSOC)); + public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) { + $stm = $this->listWhereRaw($type, $id, $state, $order, $limit, $firstId, $filter, $date_min); + return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC)); } public function listIdsWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) { //For API @@ -810,15 +814,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $res[0]; } - public static function daoToEntry($listDAO) { - $list = array(); - - if (!is_array($listDAO)) { - $listDAO = array($listDAO); - } - - foreach ($listDAO as $key => $dao) { - $entry = new FreshRSS_Entry( + public static function daoToEntry($dao) { + $entry = new FreshRSS_Entry( $dao['id_feed'], $dao['guid'], $dao['title'], @@ -830,10 +827,21 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $dao['is_favorite'], $dao['tags'] ); - if (isset($dao['id'])) { - $entry->_id($dao['id']); - } - $list[] = $entry; + if (isset($dao['id'])) { + $entry->_id($dao['id']); + } + return $entry; + } + + private static function daoToEntries($listDAO) { + $list = array(); + + if (!is_array($listDAO)) { + $listDAO = array($listDAO); + } + + foreach ($listDAO as $key => $dao) { + $list[] = self::daoToEntry($dao); } unset($listDAO); diff --git a/app/views/helpers/export/articles.phtml b/app/views/helpers/export/articles.phtml index ffdca1daa..2ad4a3abf 100644 --- a/app/views/helpers/export/articles.phtml +++ b/app/views/helpers/export/articles.phtml @@ -1,47 +1,57 @@ 'user/' . str_replace('/', '', $username) . '/state/org.freshrss/' . $this->type, - 'title' => $this->list_title, - 'author' => $username, - 'items' => array() - ); + $options = 0; + if (version_compare(PHP_VERSION, '5.4.0') >= 0) { + $options = JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; + } - foreach ($this->entries as $entry) { - if (!isset($this->feed)) { - $feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed ()); - } else { - $feed = $this->feed; - } + $articles = array( + 'id' => 'user/' . str_replace('/', '', $username) . '/state/org.freshrss/' . $this->type, + 'title' => $this->list_title, + 'author' => $username, + 'items' => array(), + ); - $articles['items'][] = array( - 'id' => $entry->guid(), - 'categories' => array_values($entry->tags()), - 'title' => $entry->title(), - 'author' => $entry->author(), - 'published' => $entry->date(true), - 'updated' => $entry->date(true), - 'alternate' => array(array( - 'href' => $entry->link(), - 'type' => 'text/html' - )), - 'content' => array( - 'content' => $entry->content() - ), - 'origin' => array( - 'streamId' => $feed->id(), - 'title' => $feed->name(), - 'htmlUrl' => $feed->website(), - 'feedUrl' => $feed->url() - ) - ); - } + echo rtrim(json_encode($articles, $options), " ]}\n\r\t"); + $first = true; - $options = 0; - if (version_compare(PHP_VERSION, '5.4.0') >= 0) { - $options = JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; - } + foreach ($this->entriesRaw as $entryRaw) { + $entry = FreshRSS_EntryDAO::daoToEntry($entryRaw); + if (!isset($this->feed)) { + $feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed ()); + } else { + $feed = $this->feed; + } - echo json_encode($articles, $options); -?> + $article = array( + 'id' => $entry->guid(), + 'categories' => array_values($entry->tags()), + 'title' => $entry->title(), + 'author' => $entry->author(), + 'published' => $entry->date(true), + 'updated' => $entry->date(true), + 'alternate' => array(array( + 'href' => $entry->link(), + 'type' => 'text/html', + )), + 'content' => array( + 'content' => $entry->content(), + ), + 'origin' => array( + 'streamId' => $feed->id(), + 'title' => $feed->name(), + 'htmlUrl' => $feed->website(), + 'feedUrl' => $feed->url(), + ) + ); + + if ($first) { + $first = false; + } else { + echo ",\n"; + } + echo json_encode($article, $options); + } + + echo "\n]}\n"; -- cgit v1.2.3 From 8d3d2a7d7fc7d623e998b6dd97b7f4419cf0d60c Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Tue, 15 Nov 2016 20:54:56 +0100 Subject: Minor code formatting of export helper --- app/views/helpers/export/articles.phtml | 96 ++++++++++++++++----------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/app/views/helpers/export/articles.phtml b/app/views/helpers/export/articles.phtml index 2ad4a3abf..9acf3cc83 100644 --- a/app/views/helpers/export/articles.phtml +++ b/app/views/helpers/export/articles.phtml @@ -1,57 +1,57 @@ = 0) { - $options = JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; - } +$options = 0; +if (version_compare(PHP_VERSION, '5.4.0') >= 0) { + $options = JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; +} - $articles = array( - 'id' => 'user/' . str_replace('/', '', $username) . '/state/org.freshrss/' . $this->type, - 'title' => $this->list_title, - 'author' => $username, - 'items' => array(), - ); +$articles = array( + 'id' => 'user/' . str_replace('/', '', $username) . '/state/org.freshrss/' . $this->type, + 'title' => $this->list_title, + 'author' => $username, + 'items' => array(), +); - echo rtrim(json_encode($articles, $options), " ]}\n\r\t"); - $first = true; +echo rtrim(json_encode($articles, $options), " ]}\n\r\t"), "\n"; +$first = true; - foreach ($this->entriesRaw as $entryRaw) { - $entry = FreshRSS_EntryDAO::daoToEntry($entryRaw); - if (!isset($this->feed)) { - $feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed ()); - } else { - $feed = $this->feed; - } +foreach ($this->entriesRaw as $entryRaw) { + $entry = FreshRSS_EntryDAO::daoToEntry($entryRaw); + if (!isset($this->feed)) { + $feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed()); + } else { + $feed = $this->feed; + } - $article = array( - 'id' => $entry->guid(), - 'categories' => array_values($entry->tags()), - 'title' => $entry->title(), - 'author' => $entry->author(), - 'published' => $entry->date(true), - 'updated' => $entry->date(true), - 'alternate' => array(array( - 'href' => $entry->link(), - 'type' => 'text/html', - )), - 'content' => array( - 'content' => $entry->content(), - ), - 'origin' => array( - 'streamId' => $feed->id(), - 'title' => $feed->name(), - 'htmlUrl' => $feed->website(), - 'feedUrl' => $feed->url(), - ) - ); + $article = array( + 'id' => $entry->guid(), + 'categories' => array_values($entry->tags()), + 'title' => $entry->title(), + 'author' => $entry->author(), + 'published' => $entry->date(true), + 'updated' => $entry->date(true), + 'alternate' => array(array( + 'href' => $entry->link(), + 'type' => 'text/html', + )), + 'content' => array( + 'content' => $entry->content(), + ), + 'origin' => array( + 'streamId' => $feed->id(), + 'title' => $feed->name(), + 'htmlUrl' => $feed->website(), + 'feedUrl' => $feed->url(), + ) + ); - if ($first) { - $first = false; - } else { - echo ",\n"; - } - echo json_encode($article, $options); + if ($first) { + $first = false; + } else { + echo ",\n"; } + echo json_encode($article, $options); +} - echo "\n]}\n"; +echo "\n]}\n"; -- cgit v1.2.3 From 568b737b6c62ace5aaa6b3f6c968e2595ea70f55 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Tue, 15 Nov 2016 21:38:18 +0100 Subject: Function to disable MySQL buffering for large exports --- app/Controllers/importExportController.php | 2 ++ lib/Minz/ModelPdo.php | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index d86587ea9..6ae89defb 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -531,6 +531,8 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { $this->entryDAO = FreshRSS_Factory::createEntryDao($username); $this->feedDAO = FreshRSS_Factory::createFeedDao($username); + $this->entryDAO->disableBuffering(); + if ($export_feeds === true) { //All feeds $export_feeds = $this->feedDAO->listFeedsIds(); diff --git a/lib/Minz/ModelPdo.php b/lib/Minz/ModelPdo.php index 6e8d60bc9..caab1d114 100644 --- a/lib/Minz/ModelPdo.php +++ b/lib/Minz/ModelPdo.php @@ -116,6 +116,12 @@ class Minz_ModelPdo { self::$sharedBd = null; self::$sharedPrefix = ''; } + + public function disableBuffering() { + if ((self::$sharedDbType === 'mysql') && defined('PDO::MYSQL_ATTR_USE_BUFFERED_QUERY')) { + $this->bd->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false); + } + } } class MinzPDO extends PDO { -- cgit v1.2.3 From 1d5006d83fc84a0c653d8e2db42a4cfbdebdc5bb Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 16 Nov 2016 17:49:20 +0100 Subject: Error edge cases for JSON export --- app/views/helpers/export/articles.phtml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/app/views/helpers/export/articles.phtml b/app/views/helpers/export/articles.phtml index 9acf3cc83..49c370023 100644 --- a/app/views/helpers/export/articles.phtml +++ b/app/views/helpers/export/articles.phtml @@ -17,9 +17,15 @@ echo rtrim(json_encode($articles, $options), " ]}\n\r\t"), "\n"; $first = true; foreach ($this->entriesRaw as $entryRaw) { + if (empty($entryRaw)) { + continue; + } $entry = FreshRSS_EntryDAO::daoToEntry($entryRaw); if (!isset($this->feed)) { $feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed()); + if ($feed == null) { + $feed = $entry->feed(true); + } } else { $feed = $this->feed; } @@ -39,19 +45,22 @@ foreach ($this->entriesRaw as $entryRaw) { 'content' => $entry->content(), ), 'origin' => array( - 'streamId' => $feed->id(), - 'title' => $feed->name(), - 'htmlUrl' => $feed->website(), - 'feedUrl' => $feed->url(), + 'streamId' => $feed == null ? '' : $feed->id(), + 'title' => $feed == null ? '' : $feed->name(), + 'htmlUrl' => $feed == null ? '' : $feed->website(), + 'feedUrl' => $feed == null ? '' : $feed->url(), ) ); - if ($first) { - $first = false; - } else { - echo ",\n"; + $line = json_encode($article, $options); + if ($line != '') { + if ($first) { + $first = false; + } else { + echo ",\n"; + } + echo $line; } - echo json_encode($article, $options); } echo "\n]}\n"; -- cgit v1.2.3 From 801e23044c223d56d63a62a0b351b11f6d71796c Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Thu, 17 Nov 2016 21:34:31 +0100 Subject: Changelog More robust export function in the case of large datasets --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dcb1bfec..d24434169 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ * Fix bug in estimating last user activity [#1358](https://github.com/FreshRSS/FreshRSS/issues/1358) * PostgreSQL: fix bug when updating cached values [#1360](https://github.com/FreshRSS/FreshRSS/issues/1360) * Fix small bugs in installer [#1363](https://github.com/FreshRSS/FreshRSS/pull/1363) +* Misc. + * More robust export function in the case of large datasets [#1372](https://github.com/FreshRSS/FreshRSS/issues/1372) ## 2016-11-02 FreshRSS 1.6.1 @@ -66,7 +68,7 @@ * Download icon 💾 for podcasts [#1236](https://github.com/FreshRSS/FreshRSS/issues/1236) * SimplePie * Fix auto-discovery of RSS feeds in Web pages served as `text/xml` [#1264](https://github.com/FreshRSS/FreshRSS/issues/1264) -* Mics. +* Misc. * Removed *resource-priorities* attributes (`defer`, `lazyload`), deprecated by W3C [#1222](https://github.com/FreshRSS/FreshRSS/pull/1222) -- cgit v1.2.3