From d6f414108667f32fe2b480adeb7ec9c218db2f4a Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Thu, 3 Jul 2014 21:26:30 +0200 Subject: Preparation for SQLite https://github.com/marienfressinaud/FreshRSS/issues/100 --- app/Controllers/importExportController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/Controllers/importExportController.php') diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index 3cd791781..12154d9b6 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -12,7 +12,7 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { require_once(LIB_PATH . '/lib_opml.php'); $this->catDAO = new FreshRSS_CategoryDAO(); - $this->entryDAO = new FreshRSS_EntryDAO(); + $this->entryDAO = FreshRSS_Factory::createEntryDao(); $this->feedDAO = new FreshRSS_FeedDAO(); } -- cgit v1.2.3 From b34f59e85aa851b82210fdc50074918034f960db Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Thu, 3 Jul 2014 22:48:29 +0200 Subject: Preparation #3 for SQLite https://github.com/marienfressinaud/FreshRSS/issues/100 --- app/Controllers/configureController.php | 6 +++--- app/Controllers/entryController.php | 4 ++-- app/Controllers/feedController.php | 8 ++++---- app/Controllers/importExportController.php | 2 +- app/Controllers/indexController.php | 4 ++-- app/Controllers/javascriptController.php | 2 +- app/Models/Category.php | 2 +- app/Models/Entry.php | 2 +- app/Models/EntryDAOSQLite.php | 10 +++++----- app/Models/Factory.php | 9 +++++++++ app/Models/Feed.php | 4 ++-- app/Models/FeedDAO.php | 16 ++++++++-------- app/Models/FeedDAOSQLite.php | 5 +++++ p/api/greader.php | 2 +- 14 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 app/Models/FeedDAOSQLite.php (limited to 'app/Controllers/importExportController.php') diff --git a/app/Controllers/configureController.php b/app/Controllers/configureController.php index cb8e6528f..a608df162 100755 --- a/app/Controllers/configureController.php +++ b/app/Controllers/configureController.php @@ -14,7 +14,7 @@ class FreshRSS_configure_Controller extends Minz_ActionController { } public function categorizeAction () { - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $catDAO = new FreshRSS_CategoryDAO (); $defaultCategory = $catDAO->getDefault (); $defaultId = $defaultCategory->id (); @@ -70,7 +70,7 @@ class FreshRSS_configure_Controller extends Minz_ActionController { $catDAO = new FreshRSS_CategoryDAO (); $this->view->categories = $catDAO->listCategories (false); - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $this->view->feeds = $feedDAO->listFeeds (); $id = Minz_Request::param ('id'); @@ -336,7 +336,7 @@ class FreshRSS_configure_Controller extends Minz_ActionController { ); break; case 'f': - $dao = new FreshRSS_FeedDAO(); + $dao = FreshRSS_Factory::createFeedDao(); $feed = $dao->searchById(substr($query['get'], 2)); $this->view->query_get[$key] = array( 'type' => 'feed', diff --git a/app/Controllers/entryController.php b/app/Controllers/entryController.php index c2d897cf3..2d7fa718a 100755 --- a/app/Controllers/entryController.php +++ b/app/Controllers/entryController.php @@ -100,7 +100,7 @@ class FreshRSS_entry_Controller extends Minz_ActionController { $entryDAO = FreshRSS_Factory::createEntryDao(); $entryDAO->optimizeTable(); - $feedDAO = new FreshRSS_FeedDAO(); + $feedDAO = FreshRSS_Factory::createFeedDao(); $feedDAO->updateCachedValues(); invalidateHttpCache(); @@ -124,7 +124,7 @@ class FreshRSS_entry_Controller extends Minz_ActionController { $nb_month_old = max($this->view->conf->old_entries, 1); $date_min = time() - (3600 * 24 * 30 * $nb_month_old); - $feedDAO = new FreshRSS_FeedDAO(); + $feedDAO = FreshRSS_Factory::createFeedDao(); $feeds = $feedDAO->listFeedsOrderUpdate(); $nbTotal = 0; diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 149557a48..d30b60877 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -31,7 +31,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { ), true); } - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $this->catDAO = new FreshRSS_CategoryDAO (); $this->catDAO->checkDefault (); @@ -201,7 +201,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { public function truncateAction () { if (Minz_Request::isPost ()) { $id = Minz_Request::param ('id'); - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $n = $feedDAO->truncate($id); $notif = array( 'type' => $n === false ? 'bad' : 'good', @@ -216,7 +216,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { public function actualizeAction () { @set_time_limit(300); - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $entryDAO = FreshRSS_Factory::createEntryDao(); Minz_Session::_param('actualize_feeds', false); @@ -375,7 +375,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $type = Minz_Request::param ('type', 'feed'); $id = Minz_Request::param ('id'); - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); if ($type == 'category') { if ($feedDAO->deleteFeedByCategory ($id)) { $notif = array ( diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index 12154d9b6..6b4c3e81a 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -13,7 +13,7 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { $this->catDAO = new FreshRSS_CategoryDAO(); $this->entryDAO = FreshRSS_Factory::createEntryDao(); - $this->feedDAO = new FreshRSS_FeedDAO(); + $this->feedDAO = FreshRSS_Factory::createFeedDao(); } public function indexAction() { diff --git a/app/Controllers/indexController.php b/app/Controllers/indexController.php index 46c9518c9..4340865ec 100755 --- a/app/Controllers/indexController.php +++ b/app/Controllers/indexController.php @@ -126,7 +126,7 @@ class FreshRSS_index_Controller extends Minz_ActionController { // on essaye de récupérer tous les articles if ($state === FreshRSS_Entry::STATE_NOT_READ && empty($entries) && ($state_param === null)) { Minz_Log::record ('Conflicting information about nbNotRead!', Minz_Log::DEBUG); - $feedDAO = new FreshRSS_FeedDAO(); + $feedDAO = FreshRSS_Factory::createFeedDao(); try { $feedDAO->updateCachedValues(); } catch (Exception $ex) { @@ -187,7 +187,7 @@ class FreshRSS_index_Controller extends Minz_ActionController { case 'f': $feed = FreshRSS_CategoryDAO::findFeed($this->view->cat_aside, $getId); if (empty($feed)) { - $feedDAO = new FreshRSS_FeedDAO(); + $feedDAO = FreshRSS_Factory::createFeedDao(); $feed = $feedDAO->searchById($getId); } if ($feed) { diff --git a/app/Controllers/javascriptController.php b/app/Controllers/javascriptController.php index 3d741e298..737908c91 100755 --- a/app/Controllers/javascriptController.php +++ b/app/Controllers/javascriptController.php @@ -7,7 +7,7 @@ class FreshRSS_javascript_Controller extends Minz_ActionController { public function actualizeAction () { header('Content-Type: text/javascript; charset=UTF-8'); - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $this->view->feeds = $feedDAO->listFeedsOrderUpdate(); } diff --git a/app/Models/Category.php b/app/Models/Category.php index 328bae799..0a0dbd3ca 100644 --- a/app/Models/Category.php +++ b/app/Models/Category.php @@ -44,7 +44,7 @@ class FreshRSS_Category extends Minz_Model { } public function feeds () { if ($this->feeds === null) { - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $this->feeds = $feedDAO->listByCategory ($this->id ()); $this->nbFeed = 0; $this->nbNotRead = 0; diff --git a/app/Models/Entry.php b/app/Models/Entry.php index a24a902d5..0bf1f2616 100644 --- a/app/Models/Entry.php +++ b/app/Models/Entry.php @@ -74,7 +74,7 @@ class FreshRSS_Entry extends Minz_Model { } public function feed ($object = false) { if ($object) { - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); return $feedDAO->searchById ($this->feed); } else { return $this->feed; diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php index 45d3a3ea9..0a837b3aa 100644 --- a/app/Models/EntryDAOSQLite.php +++ b/app/Models/EntryDAOSQLite.php @@ -13,24 +13,24 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO { } } else { $this->bd->beginTransaction(); - $sql = 'UPDATE `' . $this->prefix . 'entry` e SET e.is_read = ? WHERE e.id=? AND e.is_read<>?'; + $sql = 'UPDATE `' . $this->prefix . 'entry` SET is_read=? WHERE id=? AND is_read<>?'; $values = array($is_read ? 1 : 0, $ids, $is_read ? 1 : 0); $stm = $this->bd->prepare($sql); if (!($stm && $stm->execute ($values))) { $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); - Minz_Log::record('SQL error markRead: ' . $info[2], Minz_Log::ERROR); + Minz_Log::record('SQL error markRead 1: ' . $info[2], Minz_Log::ERROR); $this->bd->rollBack (); return false; } $affected = $stm->rowCount(); if ($affected > 0) { - $sql = 'UPDATE `' . $this->prefix . 'feed` f SET f.cache_nbUnreads=f.cache_nbUnreads' . ($is_read ? '-' : '+') . '1 ' - . 'WHERE f.id=(SELECT e.id_feed FROM `' . $this->prefix . 'entry` e WHERE e.id=?)'; + $sql = 'UPDATE `' . $this->prefix . 'feed` SET cache_nbUnreads=cache_nbUnreads' . ($is_read ? '-' : '+') . '1 ' + . 'WHERE id=(SELECT e.id_feed FROM `' . $this->prefix . 'entry` e WHERE e.id=?)'; $values = array($ids); $stm = $this->bd->prepare($sql); if (!($stm && $stm->execute ($values))) { $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); - Minz_Log::record('SQL error markRead: ' . $info[2], Minz_Log::ERROR); + Minz_Log::record('SQL error markRead 2: ' . $info[2], Minz_Log::ERROR); $this->bd->rollBack (); return false; } diff --git a/app/Models/Factory.php b/app/Models/Factory.php index 3ef68c41d..95d21a277 100644 --- a/app/Models/Factory.php +++ b/app/Models/Factory.php @@ -2,6 +2,15 @@ class FreshRSS_Factory { + public static function createFeedDao() { + $db = Minz_Configuration::dataBase(); + if ($db['type'] === 'sqlite') { + return new FreshRSS_FeedDAOSQLite(); + } else { + return new FreshRSS_FeedDAO(); + } + } + public static function createEntryDao() { $db = Minz_Configuration::dataBase(); if ($db['type'] === 'sqlite') { diff --git a/app/Models/Feed.php b/app/Models/Feed.php index ba142c8c8..8093b2bfd 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -87,7 +87,7 @@ class FreshRSS_Feed extends Minz_Model { } public function nbEntries () { if ($this->nbEntries < 0) { - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $this->nbEntries = $feedDAO->countEntries ($this->id ()); } @@ -95,7 +95,7 @@ class FreshRSS_Feed extends Minz_Model { } public function nbNotRead () { if ($this->nbNotRead < 0) { - $feedDAO = new FreshRSS_FeedDAO (); + $feedDAO = FreshRSS_Factory::createFeedDao(); $this->nbNotRead = $feedDAO->countNotRead ($this->id ()); } diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 3f3847aa4..9534d61bd 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -84,15 +84,15 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo { public function updateLastUpdate ($id, $inError = 0, $updateCache = true) { if ($updateCache) { - $sql = 'UPDATE `' . $this->prefix . 'feed` f ' //2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE - . 'SET f.cache_nbEntries=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=f.id),' - . 'f.cache_nbUnreads=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=f.id AND e2.is_read=0),' + $sql = 'UPDATE `' . $this->prefix . 'feed` ' //2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE + . 'SET cache_nbEntries=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=feed.id),' + . 'cache_nbUnreads=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=feed.id AND e2.is_read=0),' . 'lastUpdate=?, error=? ' - . 'WHERE f.id=?'; + . 'WHERE id=?'; } else { - $sql = 'UPDATE `' . $this->prefix . 'feed` f ' + $sql = 'UPDATE `' . $this->prefix . 'feed` ' . 'SET lastUpdate=?, error=? ' - . 'WHERE f.id=?'; + . 'WHERE id=?'; } $values = array ( @@ -320,8 +320,8 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo { } $affected = $stm->rowCount(); - $sql = 'UPDATE `' . $this->prefix . 'feed` f ' - . 'SET f.cache_nbEntries=0, f.cache_nbUnreads=0 WHERE f.id=?'; + $sql = 'UPDATE `' . $this->prefix . 'feed` ' + . 'SET cache_nbEntries=0, cache_nbUnreads=0 WHERE id=?'; $values = array ($id); $stm = $this->bd->prepare ($sql); if (!($stm && $stm->execute ($values))) { diff --git a/app/Models/FeedDAOSQLite.php b/app/Models/FeedDAOSQLite.php new file mode 100644 index 000000000..f3c92149e --- /dev/null +++ b/app/Models/FeedDAOSQLite.php @@ -0,0 +1,5 @@ +arrayFeedCategoryNames(); switch ($path) { -- cgit v1.2.3 From 439a0e2991231db51232646736a4bf78cfb2bffd Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 7 Jul 2014 18:25:48 +0200 Subject: SQL: improved performance for adding new articles --- app/Controllers/feedController.php | 28 +++++++++++++++++----------- app/Controllers/importExportController.php | 1 + app/Models/EntryDAO.php | 8 ++++++-- 3 files changed, 24 insertions(+), 13 deletions(-) (limited to 'app/Controllers/importExportController.php') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 36425ca9b..3326b2059 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -109,18 +109,23 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $nb_month_old = $this->view->conf->old_entries; $date_min = time () - (3600 * 24 * 30 * $nb_month_old); + //MySQL: http://docs.oracle.com/cd/E17952_01/refman-5.5-en/optimizing-innodb-transaction-management.html + //SQLite: http://stackoverflow.com/questions/1711631/how-do-i-improve-the-performance-of-sqlite + $preparedStatement = $entryDAO->addEntryPrepare(); $transactionStarted = true; - $feedDAO->beginTransaction (); + $feedDAO->beginTransaction(); // on ajoute les articles en masse sans vérification foreach ($entries as $entry) { - $values = $entry->toArray (); - $values['id_feed'] = $feed->id (); - $values['id'] = min(time(), $entry->date (true)) . uSecString(); + $values = $entry->toArray(); + $values['id_feed'] = $feed->id(); + $values['id'] = min(time(), $entry->date(true)) . uSecString(); $values['is_read'] = $is_read; - $entryDAO->addEntry ($values); + $entryDAO->addEntry($values, $preparedStatement); + } + $feedDAO->updateLastUpdate($feed->id()); + if ($transactionStarted) { + $feedDAO->commit(); } - $feedDAO->updateLastUpdate ($feed->id ()); - $feedDAO->commit (); $transactionStarted = false; // ok, ajout terminé @@ -265,22 +270,23 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $feedHistory = $this->view->conf->keep_history_default; } + $preparedStatement = $entryDAO->addEntryPrepare(); $hasTransaction = true; $feedDAO->beginTransaction(); // On ne vérifie pas strictement que l'article n'est pas déjà en BDD // La BDD refusera l'ajout car (id_feed, guid) doit être unique foreach ($entries as $entry) { - $eDate = $entry->date (true); - if ((!isset ($existingGuids[$entry->guid ()])) && + $eDate = $entry->date(true); + if ((!isset($existingGuids[$entry->guid()])) && (($feedHistory != 0) || ($eDate >= $date_min))) { - $values = $entry->toArray (); + $values = $entry->toArray(); //Use declared date at first import, otherwise use discovery date $values['id'] = ($useDeclaredDate || $eDate < $date_min) ? min(time(), $eDate) . uSecString() : uTimeString(); $values['is_read'] = $is_read; - $entryDAO->addEntry ($values); + $entryDAO->addEntry($values, $preparedStatement); } } } diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index 6b4c3e81a..ba172cc6d 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -266,6 +266,7 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { ); $entry->_tags($tags); + //FIME: Use entryDAO->addEntryPrepare(). Do not call entryDAO->listLastGuidsByFeed() for each entry. Consider using a transaction. $id = $this->entryDAO->addEntryObject( $entry, $this->view->conf, $feed->keepHistory() ); diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 1775af63c..6f3f472f6 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -6,14 +6,18 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo { return parent::$sharedDbType !== 'sqlite'; } - public function addEntry($valuesTmp) { + public function addEntryPrepare() { $sql = 'INSERT INTO `' . $this->prefix . 'entry`(id, guid, title, author, ' . ($this->isCompressed() ? 'content_bin' : 'content') . ', link, date, is_read, is_favorite, id_feed, tags) ' . 'VALUES(?, ?, ?, ?, ' . ($this->isCompressed() ? 'COMPRESS(?)' : '?') . ', ?, ?, ?, ?, ?, ?)'; - $stm = $this->bd->prepare($sql); + return $this->bd->prepare($sql); + } + + public function addEntry($valuesTmp, $preparedStatement = null) { + $stm = $preparedStatement === null ? addEntryPrepare() : $preparedStatement; $values = array( $valuesTmp['id'], -- cgit v1.2.3