diff options
| author | 2017-04-01 11:32:24 +0200 | |
|---|---|---|
| committer | 2017-04-01 11:32:24 +0200 | |
| commit | f98cd52a02eb1e2d17b46ef9fddf327b0ebd55e2 (patch) | |
| tree | 85b8e7a60922781ea898c57f649e97b3640f3d31 /app/Models | |
| parent | f2b4bfc67da9eba8ca45a02860e6f6a58a13d927 (diff) | |
| parent | 8acdf273b2f832ed07bbc5f630c068c9fe73cbe0 (diff) | |
Merge pull request #1470 from Alkarex/defered-insertion
Implementation of defered insertion
Diffstat (limited to 'app/Models')
| -rw-r--r-- | app/Models/EntryDAO.php | 114 | ||||
| -rw-r--r-- | app/Models/EntryDAOPGSQL.php | 26 | ||||
| -rw-r--r-- | app/Models/EntryDAOSQLite.php | 39 | ||||
| -rw-r--r-- | app/Models/Factory.php | 9 | ||||
| -rw-r--r-- | app/Models/FeedDAO.php | 56 | ||||
| -rw-r--r-- | app/Models/FeedDAOSQLite.php | 19 | ||||
| -rw-r--r-- | app/Models/UserDAO.php | 10 |
7 files changed, 172 insertions, 101 deletions
diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 96790c69c..61ec48d08 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -88,6 +88,38 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return false; } + protected function createEntryTempTable() { + $ok = false; + $hadTransaction = $this->bd->inTransaction(); + if ($hadTransaction) { + $this->bd->commit(); + } + try { + $db = FreshRSS_Context::$system_conf->db; + require_once(APP_PATH . '/SQL/install.sql.' . $db['type'] . '.php'); + Minz_Log::warning('SQL CREATE TABLE entrytmp...'); + if (defined('SQL_CREATE_TABLE_ENTRYTMP')) { + $sql = sprintf(SQL_CREATE_TABLE_ENTRYTMP, $this->prefix); + $stm = $this->bd->prepare($sql); + $ok = $stm && $stm->execute(); + } else { + global $SQL_CREATE_TABLE_ENTRYTMP; + $ok = !empty($SQL_CREATE_TABLE_ENTRYTMP); + foreach ($SQL_CREATE_TABLE_ENTRYTMP as $instruction) { + $sql = sprintf($instruction, $this->prefix); + $stm = $this->bd->prepare($sql); + $ok &= $stm && $stm->execute(); + } + } + } catch (Exception $e) { + Minz_Log::error('FreshRSS_EntryDAO::createEntryTempTable error: ' . $e->getMessage()); + } + if ($hadTransaction) { + $this->bd->beginTransaction(); + } + return $ok; + } + protected function autoUpdateDb($errorInfo) { if (isset($errorInfo[0])) { if ($errorInfo[0] === '42S22') { //ER_BAD_FIELD_ERROR @@ -97,6 +129,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $this->addColumn($column); } } + } elseif ($errorInfo[0] === '42S02' && stripos($errorInfo[2], 'entrytmp') !== false) { //ER_BAD_TABLE_ERROR + return $this->createEntryTempTable(); //v1.7 } } if (isset($errorInfo[1])) { @@ -110,8 +144,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { private $addEntryPrepared = null; public function addEntry($valuesTmp) { - if ($this->addEntryPrepared === null) { - $sql = 'INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, ' + if ($this->addEntryPrepared == null) { + $sql = 'INSERT INTO `' . $this->prefix . 'entrytmp` (id, guid, title, author, ' . ($this->isCompressed() ? 'content_bin' : 'content') . ', link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' . 'VALUES(:id, :guid, :title, :author, ' @@ -121,41 +155,43 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { . ', :is_read, :is_favorite, :id_feed, :tags)'; $this->addEntryPrepared = $this->bd->prepare($sql); } - $this->addEntryPrepared->bindParam(':id', $valuesTmp['id']); - $valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760); - $valuesTmp['guid'] = safe_ascii($valuesTmp['guid']); - $this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']); - $valuesTmp['title'] = substr($valuesTmp['title'], 0, 255); - $this->addEntryPrepared->bindParam(':title', $valuesTmp['title']); - $valuesTmp['author'] = substr($valuesTmp['author'], 0, 255); - $this->addEntryPrepared->bindParam(':author', $valuesTmp['author']); - $this->addEntryPrepared->bindParam(':content', $valuesTmp['content']); - $valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023); - $valuesTmp['link'] = safe_ascii($valuesTmp['link']); - $this->addEntryPrepared->bindParam(':link', $valuesTmp['link']); - $this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT); - $valuesTmp['lastSeen'] = time(); - $this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT); - $valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0; - $this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT); - $valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0; - $this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT); - $this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT); - $valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023); - $this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']); - - if ($this->hasNativeHex()) { - $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']); - } else { - $valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']); //hex2bin() is PHP5.4+ - $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']); + if ($this->addEntryPrepared) { + $this->addEntryPrepared->bindParam(':id', $valuesTmp['id']); + $valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760); + $valuesTmp['guid'] = safe_ascii($valuesTmp['guid']); + $this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']); + $valuesTmp['title'] = substr($valuesTmp['title'], 0, 255); + $this->addEntryPrepared->bindParam(':title', $valuesTmp['title']); + $valuesTmp['author'] = substr($valuesTmp['author'], 0, 255); + $this->addEntryPrepared->bindParam(':author', $valuesTmp['author']); + $this->addEntryPrepared->bindParam(':content', $valuesTmp['content']); + $valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023); + $valuesTmp['link'] = safe_ascii($valuesTmp['link']); + $this->addEntryPrepared->bindParam(':link', $valuesTmp['link']); + $this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT); + $valuesTmp['lastSeen'] = time(); + $this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT); + $valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0; + $this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT); + $valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0; + $this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT); + $this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT); + $valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023); + $this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']); + + if ($this->hasNativeHex()) { + $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']); + } else { + $valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']); //hex2bin() is PHP5.4+ + $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']); + } } - if ($this->addEntryPrepared && $this->addEntryPrepared->execute()) { return true; } else { $info = $this->addEntryPrepared == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $this->addEntryPrepared->errorInfo(); if ($this->autoUpdateDb($info)) { + $this->addEntryPrepared = null; return $this->addEntry($valuesTmp); } elseif ((int)((int)$info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] @@ -165,6 +201,22 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } + public function commitNewEntries() { + $sql = 'SET @rank=(SELECT MAX(id) - COUNT(*) FROM `' . $this->prefix . 'entrytmp`); ' . //MySQL-specific + 'INSERT IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' . + 'SELECT @rank:=@rank+1 AS id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date; ' . + 'DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= @rank;'; + $hadTransaction = $this->bd->inTransaction(); + if (!$hadTransaction) { + $this->bd->beginTransaction(); + } + $result = $this->bd->exec($sql) !== false; + if (!$hadTransaction) { + $this->bd->commit(); + } + return $result; + } + private $updateEntryPrepared = null; public function updateEntry($valuesTmp) { diff --git a/app/Models/EntryDAOPGSQL.php b/app/Models/EntryDAOPGSQL.php index b96a62ebc..b25993c47 100644 --- a/app/Models/EntryDAOPGSQL.php +++ b/app/Models/EntryDAOPGSQL.php @@ -11,6 +11,11 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite { } protected function autoUpdateDb($errorInfo) { + if (isset($errorInfo[0])) { + if ($errorInfo[0] === '42P01' && stripos($errorInfo[2], 'entrytmp') !== false) { //undefined_table + return $this->createEntryTempTable(); + } + } return false; } @@ -18,6 +23,27 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite { return false; } + public function commitNewEntries() { + $sql = 'DO $$ +DECLARE +maxrank bigint := (SELECT MAX(id) FROM `' . $this->prefix . 'entrytmp`); +rank bigint := (SELECT maxrank - COUNT(*) FROM `' . $this->prefix . 'entrytmp`); +BEGIN + INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) + (SELECT rank + row_number() OVER(ORDER BY date) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date); + DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= maxrank; +END $$;'; + $hadTransaction = $this->bd->inTransaction(); + if (!$hadTransaction) { + $this->bd->beginTransaction(); + } + $result = $this->bd->exec($sql) !== false; + if (!$hadTransaction) { + $this->bd->commit(); + } + return $result; + } + public function size($all = true) { $db = FreshRSS_Context::$system_conf->db; $sql = 'SELECT pg_size_pretty(pg_database_size(?))'; diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php index 34e854608..ad7bcd865 100644 --- a/app/Models/EntryDAOSQLite.php +++ b/app/Models/EntryDAOSQLite.php @@ -7,21 +7,42 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO { } protected function autoUpdateDb($errorInfo) { - if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR - //autoAddColumn - if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) { - $showCreate = $tableInfo->fetchColumn(); - Minz_Log::debug('FreshRSS_EntryDAOSQLite::autoUpdateDb: ' . $showCreate); - foreach (array('lastSeen', 'hash') as $column) { - if (stripos($showCreate, $column) === false) { - return $this->addColumn($column); - } + Minz_Log::error('FreshRSS_EntryDAO::autoUpdateDb error: ' . print_r($errorInfo, true)); + if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entrytmp'")) { + $showCreate = $tableInfo->fetchColumn(); + if (stripos($showCreate, 'entrytmp') === false) { + return $this->createEntryTempTable(); + } + } + if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) { + $showCreate = $tableInfo->fetchColumn(); + foreach (array('lastSeen', 'hash') as $column) { + if (stripos($showCreate, $column) === false) { + return $this->addColumn($column); } } } return false; } + public function commitNewEntries() { + $sql = ' +CREATE TEMP TABLE `tmp` AS SELECT id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date; +INSERT OR IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) + SELECT rowid + (SELECT MAX(id) - COUNT(*) FROM `tmp`) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `tmp` ORDER BY date; +DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= (SELECT MAX(id) FROM `tmp`); +DROP TABLE `tmp`;'; + $hadTransaction = $this->bd->inTransaction(); + if (!$hadTransaction) { + $this->bd->beginTransaction(); + } + $result = $this->bd->exec($sql) !== false; + if (!$hadTransaction) { + $this->bd->commit(); + } + return $result; + } + protected function sqlConcat($s1, $s2) { return $s1 . '||' . $s2; } diff --git a/app/Models/Factory.php b/app/Models/Factory.php index 6502c38b7..dfccc883e 100644 --- a/app/Models/Factory.php +++ b/app/Models/Factory.php @@ -3,14 +3,7 @@ class FreshRSS_Factory { public static function createFeedDao($username = null) { - $conf = Minz_Configuration::get('system'); - switch ($conf->db['type']) { - case 'sqlite': - case 'pgsql': - return new FreshRSS_FeedDAOSQLite($username); - default: - return new FreshRSS_FeedDAO($username); - } + return new FreshRSS_FeedDAO($username); } public static function createEntryDao($username = null) { diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 0168aebd9..d278122e3 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -92,29 +92,15 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } - public function updateLastUpdate($id, $inError = false, $updateCache = true, $mtime = 0) { - if ($updateCache) { - $sql = 'UPDATE `' . $this->prefix . 'feed` ' //2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE - . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' - . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0),' - . '`lastUpdate`=?, error=? ' - . 'WHERE id=?'; - } else { - $sql = 'UPDATE `' . $this->prefix . 'feed` ' - . 'SET `lastUpdate`=?, error=? ' - . 'WHERE id=?'; - } - - if ($mtime <= 0) { - $mtime = time(); - } - + public function updateLastUpdate($id, $inError = false, $mtime = 0) { //See also updateCachedValue() + $sql = 'UPDATE `' . $this->prefix . 'feed` ' + . 'SET `lastUpdate`=?, error=? ' + . 'WHERE id=?'; $values = array( - $mtime, + $mtime <= 0 ? time() : $mtime, $inError ? 1 : 0, $id, ); - $stm = $this->bd->prepare($sql); if ($stm && $stm->execute($values)) { @@ -294,18 +280,28 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $res[0]['count']; } - public function updateCachedValues() { //For one single feed, call updateLastUpdate($id) - $sql = 'UPDATE `' . $this->prefix . 'feed` f ' - . 'INNER JOIN (' - . 'SELECT e.id_feed, ' - . 'COUNT(CASE WHEN e.is_read = 0 THEN 1 END) AS nbUnreads, ' - . 'COUNT(e.id) AS nbEntries ' - . 'FROM `' . $this->prefix . 'entry` e ' - . 'GROUP BY e.id_feed' - . ') x ON x.id_feed=f.id ' - . 'SET f.`cache_nbEntries`=x.nbEntries, f.`cache_nbUnreads`=x.nbUnreads'; + public function updateCachedValue($id) { //For multiple feeds, call updateCachedValues() + $sql = 'UPDATE `' . $this->prefix . 'feed` ' //2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE + . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' + . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0) ' + . 'WHERE id=?'; + $values = array($id); $stm = $this->bd->prepare($sql); + if ($stm && $stm->execute($values)) { + return $stm->rowCount(); + } else { + $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); + Minz_Log::error('SQL error updateCachedValue: ' . $info[2]); + return false; + } + } + + public function updateCachedValues() { //For one single feed, call updateCachedValue($id) + $sql = 'UPDATE `' . $this->prefix . 'feed` ' + . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' + . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0)'; + $stm = $this->bd->prepare($sql); if ($stm && $stm->execute()) { return $stm->rowCount(); } else { @@ -343,7 +339,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $affected; } - public function cleanOldEntries($id, $date_min, $keep = 15) { //Remember to call updateLastUpdate($id) or updateCachedValues() just after + public function cleanOldEntries($id, $date_min, $keep = 15) { //Remember to call updateCachedValue($id) or updateCachedValues() just after $sql = 'DELETE FROM `' . $this->prefix . 'entry` ' . 'WHERE id_feed=:id_feed AND id<=:id_max ' . 'AND is_favorite=0 ' //Do not remove favourites diff --git a/app/Models/FeedDAOSQLite.php b/app/Models/FeedDAOSQLite.php deleted file mode 100644 index 440ae74da..000000000 --- a/app/Models/FeedDAOSQLite.php +++ /dev/null @@ -1,19 +0,0 @@ -<?php - -class FreshRSS_FeedDAOSQLite extends FreshRSS_FeedDAO { - - public function updateCachedValues() { //For one single feed, call updateLastUpdate($id) - $sql = 'UPDATE `' . $this->prefix . 'feed` ' - . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' - . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0)'; - $stm = $this->bd->prepare($sql); - if ($stm && $stm->execute()) { - return $stm->rowCount(); - } else { - $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); - Minz_Log::error('SQL error updateCachedValues: ' . $info[2]); - return false; - } - } - -} diff --git a/app/Models/UserDAO.php b/app/Models/UserDAO.php index a60caf395..310c7c096 100644 --- a/app/Models/UserDAO.php +++ b/app/Models/UserDAO.php @@ -14,21 +14,23 @@ class FreshRSS_UserDAO extends Minz_ModelPdo { $ok = false; $bd_prefix_user = $db['prefix'] . $username . '_'; if (defined('SQL_CREATE_TABLES')) { //E.g. MySQL - $sql = sprintf(SQL_CREATE_TABLES, $bd_prefix_user, _t('gen.short.default_category')); + $sql = sprintf(SQL_CREATE_TABLES . SQL_CREATE_TABLE_ENTRYTMP, $bd_prefix_user, _t('gen.short.default_category')); $stm = $userPDO->bd->prepare($sql); $ok = $stm && $stm->execute(); } else { //E.g. SQLite global $SQL_CREATE_TABLES; + global $SQL_CREATE_TABLE_ENTRYTMP; if (is_array($SQL_CREATE_TABLES)) { - $ok = true; - foreach ($SQL_CREATE_TABLES as $instruction) { + $instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP); + $ok = !empty($instructions); + foreach ($instructions as $instruction) { $sql = sprintf($instruction, $bd_prefix_user, _t('gen.short.default_category')); $stm = $userPDO->bd->prepare($sql); $ok &= ($stm && $stm->execute()); } } } - if ($insertDefaultFeeds) { + if ($ok && $insertDefaultFeeds) { if (defined('SQL_INSERT_FEEDS')) { //E.g. MySQL $sql = sprintf(SQL_INSERT_FEEDS, $bd_prefix_user); $stm = $userPDO->bd->prepare($sql); |
