Candidate implementation of defered insertion

https://github.com/FreshRSS/FreshRSS/issues/530

4 files changed, 146 insertions, 44 deletions

diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php
index afcde3d7f..decae9307 100644
--- a/app/Models/EntryDAO.php
+++ b/app/Models/EntryDAO.php
@@ -88,6 +88,38 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return false;
 	}
 
+	protected function createEntryTempTable() {
+		$ok = false;
+		$hadTransaction = $this->bd->inTransaction();
+		if ($hadTransaction) {
+			$this->bd->commit();
+		}
+		try {
+			$db = FreshRSS_Context::$system_conf->db;
+			require_once(APP_PATH . '/SQL/install.sql.' . $db['type'] . '.php');
+			Minz_Log::warning('SQL CREATE TABLE entrytmp...');
+			if (defined('SQL_CREATE_TABLE_ENTRYTMP')) {
+				$sql = sprintf(SQL_CREATE_TABLE_ENTRYTMP, $this->prefix);
+				$stm = $this->bd->prepare($sql);
+				$ok = $stm && $stm->execute();
+			} else {
+				global $SQL_CREATE_TABLE_ENTRYTMP;
+				$ok = !empty($SQL_CREATE_TABLE_ENTRYTMP);
+				foreach ($SQL_CREATE_TABLE_ENTRYTMP as $instruction) {
+					$sql = sprintf($instruction, $this->prefix);
+					$stm = $this->bd->prepare($sql);
+					$ok &= $stm && $stm->execute();
+				}
+			}
+		} catch (Exception $e) {
+			Minz_Log::error('FreshRSS_EntryDAO::createEntryTempTable error: ' . $e->getMessage());
+		}
+		if ($hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		return $ok;
+	}
+
 	protected function autoUpdateDb($errorInfo) {
 		if (isset($errorInfo[0])) {
 			if ($errorInfo[0] === '42S22') {	//ER_BAD_FIELD_ERROR
@@ -97,6 +129,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 						return $this->addColumn($column);
 					}
 				}
+			} elseif ($errorInfo[0] === '42S02' && stripos($errorInfo[2], 'entrytmp') !== false) {	//ER_BAD_TABLE_ERROR
+				return $this->createEntryTempTable();	//v1.7
 			}
 		}
 		if (isset($errorInfo[1])) {
@@ -110,8 +144,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 	private $addEntryPrepared = null;
 
 	public function addEntry($valuesTmp) {
-		if ($this->addEntryPrepared === null) {
-			$sql = 'INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, '
+		if ($this->addEntryPrepared == null) {
+			$sql = 'INSERT INTO `' . $this->prefix . 'entrytmp` (id, guid, title, author, '
 			     . ($this->isCompressed() ? 'content_bin' : 'content')
 			     . ', link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) '
 			     . 'VALUES(:id, :guid, :title, :author, '
@@ -121,41 +155,43 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 			     . ', :is_read, :is_favorite, :id_feed, :tags)';
 			$this->addEntryPrepared = $this->bd->prepare($sql);
 		}
-		$this->addEntryPrepared->bindParam(':id', $valuesTmp['id']);
-		$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760);
-		$valuesTmp['guid'] = safe_ascii($valuesTmp['guid']);
-		$this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']);
-		$valuesTmp['title'] = substr($valuesTmp['title'], 0, 255);
-		$this->addEntryPrepared->bindParam(':title', $valuesTmp['title']);
-		$valuesTmp['author'] = substr($valuesTmp['author'], 0, 255);
-		$this->addEntryPrepared->bindParam(':author', $valuesTmp['author']);
-		$this->addEntryPrepared->bindParam(':content', $valuesTmp['content']);
-		$valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023);
-		$valuesTmp['link'] = safe_ascii($valuesTmp['link']);
-		$this->addEntryPrepared->bindParam(':link', $valuesTmp['link']);
-		$this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT);
-		$valuesTmp['lastSeen'] = time();
-		$this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT);
-		$valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0;
-		$this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT);
-		$valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0;
-		$this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT);
-		$this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT);
-		$valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023);
-		$this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']);
-
-		if ($this->hasNativeHex()) {
-			$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']);
-		} else {
-			$valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']);	//hex2bin() is PHP5.4+
-			$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']);
+		if ($this->addEntryPrepared) {
+			$this->addEntryPrepared->bindParam(':id', $valuesTmp['id']);
+			$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760);
+			$valuesTmp['guid'] = safe_ascii($valuesTmp['guid']);
+			$this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']);
+			$valuesTmp['title'] = substr($valuesTmp['title'], 0, 255);
+			$this->addEntryPrepared->bindParam(':title', $valuesTmp['title']);
+			$valuesTmp['author'] = substr($valuesTmp['author'], 0, 255);
+			$this->addEntryPrepared->bindParam(':author', $valuesTmp['author']);
+			$this->addEntryPrepared->bindParam(':content', $valuesTmp['content']);
+			$valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023);
+			$valuesTmp['link'] = safe_ascii($valuesTmp['link']);
+			$this->addEntryPrepared->bindParam(':link', $valuesTmp['link']);
+			$this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT);
+			$valuesTmp['lastSeen'] = time();
+			$this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT);
+			$valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0;
+			$this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT);
+			$valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0;
+			$this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT);
+			$this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT);
+			$valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023);
+			$this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']);
+
+			if ($this->hasNativeHex()) {
+				$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']);
+			} else {
+				$valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']);	//hex2bin() is PHP5.4+
+				$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']);
+			}
 		}
-
 		if ($this->addEntryPrepared && $this->addEntryPrepared->execute()) {
 			return $this->bd->lastInsertId();
 		} else {
 			$info = $this->addEntryPrepared == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $this->addEntryPrepared->errorInfo();
 			if ($this->autoUpdateDb($info)) {
+				$this->addEntryPrepared = null;
 				return $this->addEntry($valuesTmp);
 			} elseif ((int)($info[0] / 1000) !== 23) {	//Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries
 				Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2]
@@ -165,6 +201,23 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		}
 	}
 
+	public function commitNewEntries() {
+		$sql = 'SET @rank=SELECT MAX(id) - COUNT(*) FROM `' . $this->prefix . 'entrytmp`; ' .	//MySQL-specific
+			'INSERT IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' .
+				'SELECT @rank:=@rank+1 AS id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date; ' .
+			'DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= @rank;';
+		$stm = $this->bd->prepare($sql);
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $stm ? $stm->execute() : false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	private $updateEntryPrepared = null;
 
 	public function updateEntry($valuesTmp) {
diff --git a/app/Models/EntryDAOPGSQL.php b/app/Models/EntryDAOPGSQL.php
index b96a62ebc..b25993c47 100644
--- a/app/Models/EntryDAOPGSQL.php
+++ b/app/Models/EntryDAOPGSQL.php
@@ -11,6 +11,11 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite {
 	}
 
 	protected function autoUpdateDb($errorInfo) {
+		if (isset($errorInfo[0])) { 
+			if ($errorInfo[0] === '42P01' && stripos($errorInfo[2], 'entrytmp') !== false) {	//undefined_table
+				return $this->createEntryTempTable();
+			}
+		}
 		return false;
 	}
 
@@ -18,6 +23,27 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite {
 		return false;
 	}
 
+	public function commitNewEntries() {
+		$sql = 'DO $$
+DECLARE
+maxrank bigint := (SELECT MAX(id) FROM `' . $this->prefix . 'entrytmp`);
+rank bigint := (SELECT maxrank - COUNT(*) FROM `' . $this->prefix . 'entrytmp`);
+BEGIN
+	INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags)
+		(SELECT rank + row_number() OVER(ORDER BY date) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date);
+	DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= maxrank;
+END $$;';
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $this->bd->exec($sql) !== false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	public function size($all = true) {
 		$db = FreshRSS_Context::$system_conf->db;
 		$sql = 'SELECT pg_size_pretty(pg_database_size(?))';
diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php
index 34e854608..ad7bcd865 100644
--- a/app/Models/EntryDAOSQLite.php
+++ b/app/Models/EntryDAOSQLite.php
@@ -7,21 +7,42 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO {
 	}
 
 	protected function autoUpdateDb($errorInfo) {
-		if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') {	//ER_BAD_FIELD_ERROR
-			//autoAddColumn
-			if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) {
-				$showCreate = $tableInfo->fetchColumn();
-				Minz_Log::debug('FreshRSS_EntryDAOSQLite::autoUpdateDb: ' . $showCreate);
-				foreach (array('lastSeen', 'hash') as $column) {
-					if (stripos($showCreate, $column) === false) {
-						return $this->addColumn($column);
-					}
+		Minz_Log::error('FreshRSS_EntryDAO::autoUpdateDb error: ' . print_r($errorInfo, true));
+		if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entrytmp'")) {
+			$showCreate = $tableInfo->fetchColumn();
+			if (stripos($showCreate, 'entrytmp') === false) {
+				return $this->createEntryTempTable();
+			}
+		}
+		if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) {
+			$showCreate = $tableInfo->fetchColumn();
+			foreach (array('lastSeen', 'hash') as $column) {
+				if (stripos($showCreate, $column) === false) {
+					return $this->addColumn($column);
 				}
 			}
 		}
 		return false;
 	}
 
+	public function commitNewEntries() {
+		$sql = '
+CREATE TEMP TABLE `tmp` AS SELECT id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date;
+INSERT OR IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags)
+	SELECT rowid + (SELECT MAX(id) - COUNT(*) FROM `tmp`) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `tmp` ORDER BY date;
+DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= (SELECT MAX(id) FROM `tmp`);
+DROP TABLE `tmp`;';
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $this->bd->exec($sql) !== false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	protected function sqlConcat($s1, $s2) {
 		return $s1 . '||' . $s2;
 	}
diff --git a/app/Models/UserDAO.php b/app/Models/UserDAO.php
index a60caf395..310c7c096 100644
--- a/app/Models/UserDAO.php
+++ b/app/Models/UserDAO.php
@@ -14,21 +14,23 @@ class FreshRSS_UserDAO extends Minz_ModelPdo {
 			$ok = false;
 			$bd_prefix_user = $db['prefix'] . $username . '_';
 			if (defined('SQL_CREATE_TABLES')) {	//E.g. MySQL
-				$sql = sprintf(SQL_CREATE_TABLES, $bd_prefix_user, _t('gen.short.default_category'));
+				$sql = sprintf(SQL_CREATE_TABLES . SQL_CREATE_TABLE_ENTRYTMP, $bd_prefix_user, _t('gen.short.default_category'));
 				$stm = $userPDO->bd->prepare($sql);
 				$ok = $stm && $stm->execute();
 			} else {	//E.g. SQLite
 				global $SQL_CREATE_TABLES;
+				global $SQL_CREATE_TABLE_ENTRYTMP;
 				if (is_array($SQL_CREATE_TABLES)) {
-					$ok = true;
-					foreach ($SQL_CREATE_TABLES as $instruction) {
+					$instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP);
+					$ok = !empty($instructions);
+					foreach ($instructions as $instruction) {
 						$sql = sprintf($instruction, $bd_prefix_user, _t('gen.short.default_category'));
 						$stm = $userPDO->bd->prepare($sql);
 						$ok &= ($stm && $stm->execute());
 					}
 				}
 			}
-			if ($insertDefaultFeeds) {
+			if ($ok && $insertDefaultFeeds) {
 				if (defined('SQL_INSERT_FEEDS')) {	//E.g. MySQL
 					$sql = sprintf(SQL_INSERT_FEEDS, $bd_prefix_user);
 					$stm = $userPDO->bd->prepare($sql);