From 5b90e1f4a0057aa78fd7d8d4d748b01676ec9073 Mon Sep 17 00:00:00 2001 From: Alexis Degrugillier Date: Sun, 1 Mar 2015 09:18:06 -0500 Subject: Introduce user queries objects There is now an object to manipulate user queries. It allows to move logic to handle those from the view and the controller in the model. Thus making the view and the controller easier to read. I introduced a new interface to start using dependency injection. There is still some rough edges but we are moving in the right direction. The new object is fully tested but it still need some improvements, for instance, it is still tied to the search object. There might be a better way to do that. --- app/Models/FeedDAO.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/Models/FeedDAO.php') diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 74597c730..e8b07acbd 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -1,6 +1,6 @@ prefix . 'feed` (url, category, name, website, description, lastUpdate, priority, httpAuth, error, keep_history, ttl) VALUES(?, ?, ?, ?, ?, ?, 10, ?, 0, -2, -2)'; $stm = $this->bd->prepare($sql); -- cgit v1.2.3 From 24f6c1eabb4cea941e40307c2f732c0ca384ffd2 Mon Sep 17 00:00:00 2001 From: Alexis Degrugillier Date: Thu, 5 Mar 2015 06:47:13 -0500 Subject: Fix spacing --- app/Models/CategoryDAO.php | 2 +- app/Models/EntryDAO.php | 2 +- app/Models/FeedDAO.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/Models/FeedDAO.php') diff --git a/app/Models/CategoryDAO.php b/app/Models/CategoryDAO.php index 4eee226ba..189a5f0e4 100644 --- a/app/Models/CategoryDAO.php +++ b/app/Models/CategoryDAO.php @@ -1,6 +1,6 @@ prefix . 'category`(name) VALUES(?)'; $stm = $this->bd->prepare($sql); diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index b8a1a43b0..9736d5cd3 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -1,6 +1,6 @@ prefix . 'feed` (url, category, name, website, description, lastUpdate, priority, httpAuth, error, keep_history, ttl) VALUES(?, ?, ?, ?, ?, ?, 10, ?, 0, -2, -2)'; $stm = $this->bd->prepare($sql); -- cgit v1.2.3 From 993466844405bd9854d890d6d5ebf763ed8b78cb Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 9 May 2015 23:37:56 +0200 Subject: SQL: more robust purge https://github.com/FreshRSS/FreshRSS/issues/798 https://github.com/FreshRSS/FreshRSS/issues/493 --- app/Controllers/feedController.php | 13 +++++-------- app/Models/Feed.php | 3 +-- app/Models/FeedDAO.php | 8 +++++--- 3 files changed, 11 insertions(+), 13 deletions(-) (limited to 'app/Models/FeedDAO.php') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 59c9174fb..5657d4a88 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -329,7 +329,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // For this feed, check existing GUIDs already in database. $existingHashForGuids = $entryDAO->listHashForFeedGuids($feed->id(), $newGuids); unset($newGuids); - $use_declared_date = empty($existingHashForGuids); $oldGuids = array(); // Add entries in database if possible. @@ -353,14 +352,14 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // This entry should not be added considering configuration and date. $oldGuids[] = $entry->guid(); } else { - $id = uTimeString(); - if ($use_declared_date || $entry_date < $date_min) { - // Use declared date at first import. + if ($entry_date < $date_min) { $id = min(time(), $entry_date) . uSecString(); + $entry->_isRead(true); //Old article that was not in database. Probably an error, so mark as read + } else { + $id = uTimeString(); + $entry->_isRead($is_read); } - $entry->_id($id); - $entry->_isRead($is_read); $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); if ($entry === null) { @@ -376,7 +375,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } $entryDAO->updateLastSeen($feed->id(), $oldGuids); } - //TODO: updateLastSeen old GUIDS once in a while, in the case of caching (i.e. the whole feed content has not changed) if ($feed_history >= 0 && rand(0, 30) === 1) { // TODO: move this function in web cron when available (see entry::purge) @@ -384,7 +382,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController { if (!$entryDAO->hasTransaction()) { $entryDAO->beginTransaction(); } - //TODO: more robust system based on entry.lastSeen to avoid cleaning entries that are still published in the RSS feed. $nb = $feedDAO->cleanOldEntries($feed->id(), $date_min, diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 27c83ffd5..5d377de9a 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -245,7 +245,7 @@ class FreshRSS_Feed extends Minz_Model { $this->_url($clean_url); } - if (($mtime === true) ||($mtime > $this->lastUpdate)) { + if (($mtime === true) || ($mtime > $this->lastUpdate)) { Minz_Log::notice('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url); $this->loadEntries($feed); // et on charge les articles du flux } else { @@ -255,7 +255,6 @@ class FreshRSS_Feed extends Minz_Model { $feed->__destruct(); //http://simplepie.org/wiki/faq/i_m_getting_memory_leaks unset($feed); - //TODO: Return a different information in case of cache/no-cache, and give access to the GUIDs in case of cache } } } diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index f48beee6e..c13e2b008 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -322,10 +322,12 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $affected; } - public function cleanOldEntries($id, $date_min, $keep = 15) { //Remember to call updateLastUpdate($id) just after + public function cleanOldEntries($id, $date_min, $keep = 15) { //Remember to call updateLastUpdate($id) or updateCachedValues() just after $sql = 'DELETE FROM `' . $this->prefix . 'entry` ' - . 'WHERE id_feed = :id_feed AND id <= :id_max AND is_favorite=0 AND id NOT IN ' - . '(SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed = :id_feed ORDER BY id DESC LIMIT :keep) keep)'; //Double select MySQL doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery' + . 'WHERE id_feed = :id_feed AND id <= :id_max ' + . 'AND is_favorite=0 ' //Do not remove favourites + . 'AND lastSeen < (SELECT maxLastSeen FROM (SELECT (MAX(e3.lastSeen) - 99) AS maxLastSeen FROM `' . $this->prefix . 'entry` e3 WHERE e3.id_feed = :id_feed) recent) ' //Do not remove the most newly seen articles, plus a few seconds of tolerance + . 'AND id NOT IN (SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed = :id_feed ORDER BY id DESC LIMIT :keep) keep)'; //Double select: MySQL doesn't support 'LIMIT & IN/ALL/ANY/SOME subquery' $stm = $this->bd->prepare($sql); $id_max = intval($date_min) . '000000'; -- cgit v1.2.3 From a7bc54bb996a87a66127101be548d181dc8dd935 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 9 May 2015 23:45:52 +0200 Subject: Minor spaces --- app/Models/FeedDAO.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'app/Models/FeedDAO.php') diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index c13e2b008..76025ff53 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -324,10 +324,10 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { public function cleanOldEntries($id, $date_min, $keep = 15) { //Remember to call updateLastUpdate($id) or updateCachedValues() just after $sql = 'DELETE FROM `' . $this->prefix . 'entry` ' - . 'WHERE id_feed = :id_feed AND id <= :id_max ' + . 'WHERE id_feed=:id_feed AND id<=:id_max ' . 'AND is_favorite=0 ' //Do not remove favourites - . 'AND lastSeen < (SELECT maxLastSeen FROM (SELECT (MAX(e3.lastSeen) - 99) AS maxLastSeen FROM `' . $this->prefix . 'entry` e3 WHERE e3.id_feed = :id_feed) recent) ' //Do not remove the most newly seen articles, plus a few seconds of tolerance - . 'AND id NOT IN (SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed = :id_feed ORDER BY id DESC LIMIT :keep) keep)'; //Double select: MySQL doesn't support 'LIMIT & IN/ALL/ANY/SOME subquery' + . 'AND lastSeen < (SELECT maxLastSeen FROM (SELECT (MAX(e3.lastSeen)-99) AS maxLastSeen FROM `' . $this->prefix . 'entry` e3 WHERE e3.id_feed=:id_feed) recent) ' //Do not remove the most newly seen articles, plus a few seconds of tolerance + . 'AND id NOT IN (SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=:id_feed ORDER BY id DESC LIMIT :keep) keep)'; //Double select: MySQL doesn't support 'LIMIT & IN/ALL/ANY/SOME subquery' $stm = $this->bd->prepare($sql); $id_max = intval($date_min) . '000000'; -- cgit v1.2.3 From 217c191a1ba3ac03b847d261a32e19975380fcad Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 11 May 2015 22:42:41 +0200 Subject: More SQLite compatibility Additional changes to add compatibility with SQLite for the new hash/lastSeen mode of updating articles. --- app/Models/EntryDAO.php | 71 ++++++++++++++++++++++++------------------ app/Models/EntryDAOSQLite.php | 15 +++++++++ app/Models/FeedDAO.php | 11 ++++--- app/SQL/install.sql.mysql.php | 2 +- app/SQL/install.sql.sqlite.php | 2 +- app/install.php | 2 ++ lib/Minz/ModelPdo.php | 5 +++ 7 files changed, 70 insertions(+), 38 deletions(-) (limited to 'app/Models/FeedDAO.php') diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 172eac897..eae9683ad 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -6,38 +6,48 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return parent::$sharedDbType !== 'sqlite'; } - protected function autoAddColumn($errorInfo) { - if (isset($errorInfo[0])) { - if ($errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR - $hasTransaction = false; - try { - $stm = null; - if (stripos($errorInfo[2], 'lastSeen') !== false) { //v1.2 - if (!$this->bd->inTransaction()) { - $this->bd->beginTransaction(); - $hasTransaction = true; - } - $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN lastSeen INT(11) NOT NULL'); - if ($stm && $stm->execute()) { - $stm = $this->bd->prepare('CREATE INDEX entry_lastSeen_index ON `' . $this->prefix . 'entry`(`lastSeen`);'); //"IF NOT EXISTS" does not exist in MySQL 5.7 - if ($stm && $stm->execute()) { - if ($hasTransaction) { - $this->bd->commit(); - } - return true; - } - } + protected function addColumn($name) { + Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn: ' . $name); + $hasTransaction = false; + try { + $stm = null; + if ($name === 'lastSeen') { //v1.2 + if (!$this->bd->inTransaction()) { + $this->bd->beginTransaction(); + $hasTransaction = true; + } + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN lastSeen INT(11) DEFAULT 0'); + if ($stm && $stm->execute()) { + $stm = $this->bd->prepare('CREATE INDEX entry_lastSeen_index ON `' . $this->prefix . 'entry`(`lastSeen`);'); //"IF NOT EXISTS" does not exist in MySQL 5.7 + if ($stm && $stm->execute()) { if ($hasTransaction) { - $this->bd->rollBack(); + $this->bd->commit(); } - } elseif (stripos($errorInfo[2], 'hash') !== false) { //v1.2 - $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN hash BINARY(16) NOT NULL'); - return $stm && $stm->execute(); + return true; } - } catch (Exception $e) { - Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn error: ' . $e->getMessage()); - if ($hasTransaction) { - $this->bd->rollBack(); + } + if ($hasTransaction) { + $this->bd->rollBack(); + } + } elseif ($name === 'hash') { //v1.2 + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN hash BINARY(16)'); + return $stm && $stm->execute(); + } + } catch (Exception $e) { + Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn error: ' . $e->getMessage()); + if ($hasTransaction) { + $this->bd->rollBack(); + } + } + return false; + } + + protected function autoAddColumn($errorInfo) { + if (isset($errorInfo[0])) { + if ($errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR + foreach (array('lastSeen', 'hash') as $column) { + if (stripos($errorInfo[2], $column) !== false) { + return $this->addColumn($column); } } } @@ -82,7 +92,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $this->addEntry($valuesTmp); } elseif ((int)($info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] - . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']); + . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']. ' ' . $this->addEntryPrepared); } return false; } @@ -597,7 +607,6 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } return $result; } else { - $info = $stm == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $stm->errorInfo(); if ($this->autoAddColumn($info)) { return $this->listHashForFeedGuids($id_feed, $guids); diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php index ffe0f037c..ff049d813 100644 --- a/app/Models/EntryDAOSQLite.php +++ b/app/Models/EntryDAOSQLite.php @@ -2,6 +2,21 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO { + protected function autoAddColumn($errorInfo) { + if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR + if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) { + $showCreate = $tableInfo->fetchColumn(); + Minz_Log::debug('FreshRSS_EntryDAOSQLite::autoAddColumn: ' . $showCreate); + foreach (array('lastSeen', 'hash') as $column) { + if (stripos($showCreate, $column) === false) { + return $this->addColumn($column); + } + } + } + } + return false; + } + protected function sqlConcat($s1, $s2) { return $s1 . '||' . $s2; } diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 76025ff53..475d39286 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -330,11 +330,12 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { . 'AND id NOT IN (SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=:id_feed ORDER BY id DESC LIMIT :keep) keep)'; //Double select: MySQL doesn't support 'LIMIT & IN/ALL/ANY/SOME subquery' $stm = $this->bd->prepare($sql); - $id_max = intval($date_min) . '000000'; - - $stm->bindParam(':id_feed', $id, PDO::PARAM_INT); - $stm->bindParam(':id_max', $id_max, PDO::PARAM_STR); - $stm->bindParam(':keep', $keep, PDO::PARAM_INT); + if ($stm) { + $id_max = intval($date_min) . '000000'; + $stm->bindParam(':id_feed', $id, PDO::PARAM_INT); + $stm->bindParam(':id_max', $id_max, PDO::PARAM_STR); + $stm->bindParam(':keep', $keep, PDO::PARAM_INT); + } if ($stm && $stm->execute()) { return $stm->rowCount(); diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index afdd821b2..9c6af405d 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( `content_bin` blob, -- v0.7 `link` varchar(1023) CHARACTER SET latin1 NOT NULL, `date` int(11), -- Until year 2038 - `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `lastSeen` INT(11) DEFAULT 0, -- v1.2, Until year 2038 `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 7517ead45..77e8e094c 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -39,7 +39,7 @@ $SQL_CREATE_TABLES = array( `content` text, `link` varchar(1023) NOT NULL, `date` int(11), -- Until year 2038 - `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `lastSeen` INT(11) DEFAULT 0, -- v1.2, Until year 2038 `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, diff --git a/app/install.php b/app/install.php index 177173fdb..86afb9318 100644 --- a/app/install.php +++ b/app/install.php @@ -168,8 +168,10 @@ function saveStep3() { $_SESSION['bd_prefix_user'] = $_SESSION['bd_prefix'] .(empty($_SESSION['default_user']) ? '' :($_SESSION['default_user'] . '_')); } + //TODO: load `config.default.php` as default $config_array = array( 'environment' => 'production', + 'simplepie_syslog_enabled' => true, 'salt' => $_SESSION['salt'], 'title' => $_SESSION['title'], 'default_user' => $_SESSION['default_user'], diff --git a/lib/Minz/ModelPdo.php b/lib/Minz/ModelPdo.php index ac7a1bed7..3e8ec1f43 100644 --- a/lib/Minz/ModelPdo.php +++ b/lib/Minz/ModelPdo.php @@ -134,4 +134,9 @@ class MinzPDO extends PDO { MinzPDO::check($statement); return parent::exec($statement); } + + public function query($statement) { + MinzPDO::check($statement); + return parent::query($statement); + } } -- cgit v1.2.3