From 22b41f3bfcbd5a54d59789c2cebfda6dc23b7dde Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Sun, 26 Mar 2017 00:01:11 +0100
Subject: Candidate implementation of defered insertion

https://github.com/FreshRSS/FreshRSS/issues/530
---
 app/Controllers/feedController.php         |   8 +-
 app/Controllers/importExportController.php |   1 +
 app/Models/EntryDAO.php                    | 115 +++++++++++++++++++++--------
 app/Models/EntryDAOPGSQL.php               |  26 +++++++
 app/Models/EntryDAOSQLite.php              |  39 +++++++---
 app/Models/UserDAO.php                     |  10 ++-
 app/SQL/install.sql.mysql.php              |  23 ++++++
 app/SQL/install.sql.pgsql.php              |  24 +++++-
 app/SQL/install.sql.sqlite.php             |  24 ++++++
 app/install.php                            |  36 +++------
 10 files changed, 233 insertions(+), 73 deletions(-)

(limited to 'app')

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index f71f26a4e..a2d9d5c35 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -226,7 +226,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 		}
 	}
 
-	public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush = null, $isNewFeed = false) {
+	public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush = null, $isNewFeed = false, $noCommit = false) {
 		@set_time_limit(300);
 
 		$feedDAO = FreshRSS_Factory::createFeedDao();
@@ -434,6 +434,9 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				break;
 			}
 		}
+		if (!$noCommit) {
+			$entryDAO->commitNewEntries();
+		}
 		return array($updated_feeds, reset($feeds));
 	}
 
@@ -452,8 +455,9 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 		$id = Minz_Request::param('id');
 		$url = Minz_Request::param('url');
 		$force = Minz_Request::param('force');
+		$noCommit = Minz_Session::param('isLastFeed', 1) != 1;
 
-		list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force);
+		list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force, null, false, $noCommit);
 
 		if (Minz_Request::param('ajax')) {
 			// Most of the time, ajax request is for only one feed. But since
diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php
index 6ae89defb..ededfe506 100644
--- a/app/Controllers/importExportController.php
+++ b/app/Controllers/importExportController.php
@@ -475,6 +475,7 @@ class FreshRSS_importExport_Controller extends Minz_ActionController {
 			}
 		}
 		$this->entryDAO->commit();
+		$entryDAO->commitNewEntries();
 
 		return !$error;
 	}
diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php
index afcde3d7f..decae9307 100644
--- a/app/Models/EntryDAO.php
+++ b/app/Models/EntryDAO.php
@@ -88,6 +88,38 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return false;
 	}
 
+	protected function createEntryTempTable() {
+		$ok = false;
+		$hadTransaction = $this->bd->inTransaction();
+		if ($hadTransaction) {
+			$this->bd->commit();
+		}
+		try {
+			$db = FreshRSS_Context::$system_conf->db;
+			require_once(APP_PATH . '/SQL/install.sql.' . $db['type'] . '.php');
+			Minz_Log::warning('SQL CREATE TABLE entrytmp...');
+			if (defined('SQL_CREATE_TABLE_ENTRYTMP')) {
+				$sql = sprintf(SQL_CREATE_TABLE_ENTRYTMP, $this->prefix);
+				$stm = $this->bd->prepare($sql);
+				$ok = $stm && $stm->execute();
+			} else {
+				global $SQL_CREATE_TABLE_ENTRYTMP;
+				$ok = !empty($SQL_CREATE_TABLE_ENTRYTMP);
+				foreach ($SQL_CREATE_TABLE_ENTRYTMP as $instruction) {
+					$sql = sprintf($instruction, $this->prefix);
+					$stm = $this->bd->prepare($sql);
+					$ok &= $stm && $stm->execute();
+				}
+			}
+		} catch (Exception $e) {
+			Minz_Log::error('FreshRSS_EntryDAO::createEntryTempTable error: ' . $e->getMessage());
+		}
+		if ($hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		return $ok;
+	}
+
 	protected function autoUpdateDb($errorInfo) {
 		if (isset($errorInfo[0])) {
 			if ($errorInfo[0] === '42S22') {	//ER_BAD_FIELD_ERROR
@@ -97,6 +129,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 						return $this->addColumn($column);
 					}
 				}
+			} elseif ($errorInfo[0] === '42S02' && stripos($errorInfo[2], 'entrytmp') !== false) {	//ER_BAD_TABLE_ERROR
+				return $this->createEntryTempTable();	//v1.7
 			}
 		}
 		if (isset($errorInfo[1])) {
@@ -110,8 +144,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 	private $addEntryPrepared = null;
 
 	public function addEntry($valuesTmp) {
-		if ($this->addEntryPrepared === null) {
-			$sql = 'INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, '
+		if ($this->addEntryPrepared == null) {
+			$sql = 'INSERT INTO `' . $this->prefix . 'entrytmp` (id, guid, title, author, '
 			     . ($this->isCompressed() ? 'content_bin' : 'content')
 			     . ', link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) '
 			     . 'VALUES(:id, :guid, :title, :author, '
@@ -121,41 +155,43 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 			     . ', :is_read, :is_favorite, :id_feed, :tags)';
 			$this->addEntryPrepared = $this->bd->prepare($sql);
 		}
-		$this->addEntryPrepared->bindParam(':id', $valuesTmp['id']);
-		$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760);
-		$valuesTmp['guid'] = safe_ascii($valuesTmp['guid']);
-		$this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']);
-		$valuesTmp['title'] = substr($valuesTmp['title'], 0, 255);
-		$this->addEntryPrepared->bindParam(':title', $valuesTmp['title']);
-		$valuesTmp['author'] = substr($valuesTmp['author'], 0, 255);
-		$this->addEntryPrepared->bindParam(':author', $valuesTmp['author']);
-		$this->addEntryPrepared->bindParam(':content', $valuesTmp['content']);
-		$valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023);
-		$valuesTmp['link'] = safe_ascii($valuesTmp['link']);
-		$this->addEntryPrepared->bindParam(':link', $valuesTmp['link']);
-		$this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT);
-		$valuesTmp['lastSeen'] = time();
-		$this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT);
-		$valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0;
-		$this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT);
-		$valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0;
-		$this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT);
-		$this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT);
-		$valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023);
-		$this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']);
-
-		if ($this->hasNativeHex()) {
-			$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']);
-		} else {
-			$valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']);	//hex2bin() is PHP5.4+
-			$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']);
+		if ($this->addEntryPrepared) {
+			$this->addEntryPrepared->bindParam(':id', $valuesTmp['id']);
+			$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760);
+			$valuesTmp['guid'] = safe_ascii($valuesTmp['guid']);
+			$this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']);
+			$valuesTmp['title'] = substr($valuesTmp['title'], 0, 255);
+			$this->addEntryPrepared->bindParam(':title', $valuesTmp['title']);
+			$valuesTmp['author'] = substr($valuesTmp['author'], 0, 255);
+			$this->addEntryPrepared->bindParam(':author', $valuesTmp['author']);
+			$this->addEntryPrepared->bindParam(':content', $valuesTmp['content']);
+			$valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023);
+			$valuesTmp['link'] = safe_ascii($valuesTmp['link']);
+			$this->addEntryPrepared->bindParam(':link', $valuesTmp['link']);
+			$this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT);
+			$valuesTmp['lastSeen'] = time();
+			$this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT);
+			$valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0;
+			$this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT);
+			$valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0;
+			$this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT);
+			$this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT);
+			$valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023);
+			$this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']);
+
+			if ($this->hasNativeHex()) {
+				$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']);
+			} else {
+				$valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']);	//hex2bin() is PHP5.4+
+				$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']);
+			}
 		}
-
 		if ($this->addEntryPrepared && $this->addEntryPrepared->execute()) {
 			return $this->bd->lastInsertId();
 		} else {
 			$info = $this->addEntryPrepared == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $this->addEntryPrepared->errorInfo();
 			if ($this->autoUpdateDb($info)) {
+				$this->addEntryPrepared = null;
 				return $this->addEntry($valuesTmp);
 			} elseif ((int)($info[0] / 1000) !== 23) {	//Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries
 				Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2]
@@ -165,6 +201,23 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		}
 	}
 
+	public function commitNewEntries() {
+		$sql = 'SET @rank=SELECT MAX(id) - COUNT(*) FROM `' . $this->prefix . 'entrytmp`; ' .	//MySQL-specific
+			'INSERT IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' .
+				'SELECT @rank:=@rank+1 AS id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date; ' .
+			'DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= @rank;';
+		$stm = $this->bd->prepare($sql);
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $stm ? $stm->execute() : false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	private $updateEntryPrepared = null;
 
 	public function updateEntry($valuesTmp) {
diff --git a/app/Models/EntryDAOPGSQL.php b/app/Models/EntryDAOPGSQL.php
index b96a62ebc..b25993c47 100644
--- a/app/Models/EntryDAOPGSQL.php
+++ b/app/Models/EntryDAOPGSQL.php
@@ -11,6 +11,11 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite {
 	}
 
 	protected function autoUpdateDb($errorInfo) {
+		if (isset($errorInfo[0])) { 
+			if ($errorInfo[0] === '42P01' && stripos($errorInfo[2], 'entrytmp') !== false) {	//undefined_table
+				return $this->createEntryTempTable();
+			}
+		}
 		return false;
 	}
 
@@ -18,6 +23,27 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite {
 		return false;
 	}
 
+	public function commitNewEntries() {
+		$sql = 'DO $$
+DECLARE
+maxrank bigint := (SELECT MAX(id) FROM `' . $this->prefix . 'entrytmp`);
+rank bigint := (SELECT maxrank - COUNT(*) FROM `' . $this->prefix . 'entrytmp`);
+BEGIN
+	INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags)
+		(SELECT rank + row_number() OVER(ORDER BY date) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date);
+	DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= maxrank;
+END $$;';
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $this->bd->exec($sql) !== false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	public function size($all = true) {
 		$db = FreshRSS_Context::$system_conf->db;
 		$sql = 'SELECT pg_size_pretty(pg_database_size(?))';
diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php
index 34e854608..ad7bcd865 100644
--- a/app/Models/EntryDAOSQLite.php
+++ b/app/Models/EntryDAOSQLite.php
@@ -7,21 +7,42 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO {
 	}
 
 	protected function autoUpdateDb($errorInfo) {
-		if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') {	//ER_BAD_FIELD_ERROR
-			//autoAddColumn
-			if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) {
-				$showCreate = $tableInfo->fetchColumn();
-				Minz_Log::debug('FreshRSS_EntryDAOSQLite::autoUpdateDb: ' . $showCreate);
-				foreach (array('lastSeen', 'hash') as $column) {
-					if (stripos($showCreate, $column) === false) {
-						return $this->addColumn($column);
-					}
+		Minz_Log::error('FreshRSS_EntryDAO::autoUpdateDb error: ' . print_r($errorInfo, true));
+		if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entrytmp'")) {
+			$showCreate = $tableInfo->fetchColumn();
+			if (stripos($showCreate, 'entrytmp') === false) {
+				return $this->createEntryTempTable();
+			}
+		}
+		if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) {
+			$showCreate = $tableInfo->fetchColumn();
+			foreach (array('lastSeen', 'hash') as $column) {
+				if (stripos($showCreate, $column) === false) {
+					return $this->addColumn($column);
 				}
 			}
 		}
 		return false;
 	}
 
+	public function commitNewEntries() {
+		$sql = '
+CREATE TEMP TABLE `tmp` AS SELECT id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date;
+INSERT OR IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags)
+	SELECT rowid + (SELECT MAX(id) - COUNT(*) FROM `tmp`) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `tmp` ORDER BY date;
+DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= (SELECT MAX(id) FROM `tmp`);
+DROP TABLE `tmp`;';
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $this->bd->exec($sql) !== false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	protected function sqlConcat($s1, $s2) {
 		return $s1 . '||' . $s2;
 	}
diff --git a/app/Models/UserDAO.php b/app/Models/UserDAO.php
index a60caf395..310c7c096 100644
--- a/app/Models/UserDAO.php
+++ b/app/Models/UserDAO.php
@@ -14,21 +14,23 @@ class FreshRSS_UserDAO extends Minz_ModelPdo {
 			$ok = false;
 			$bd_prefix_user = $db['prefix'] . $username . '_';
 			if (defined('SQL_CREATE_TABLES')) {	//E.g. MySQL
-				$sql = sprintf(SQL_CREATE_TABLES, $bd_prefix_user, _t('gen.short.default_category'));
+				$sql = sprintf(SQL_CREATE_TABLES . SQL_CREATE_TABLE_ENTRYTMP, $bd_prefix_user, _t('gen.short.default_category'));
 				$stm = $userPDO->bd->prepare($sql);
 				$ok = $stm && $stm->execute();
 			} else {	//E.g. SQLite
 				global $SQL_CREATE_TABLES;
+				global $SQL_CREATE_TABLE_ENTRYTMP;
 				if (is_array($SQL_CREATE_TABLES)) {
-					$ok = true;
-					foreach ($SQL_CREATE_TABLES as $instruction) {
+					$instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP);
+					$ok = !empty($instructions);
+					foreach ($instructions as $instruction) {
 						$sql = sprintf($instruction, $bd_prefix_user, _t('gen.short.default_category'));
 						$stm = $userPDO->bd->prepare($sql);
 						$ok &= ($stm && $stm->execute());
 					}
 				}
 			}
-			if ($insertDefaultFeeds) {
+			if ($ok && $insertDefaultFeeds) {
 				if (defined('SQL_INSERT_FEEDS')) {	//E.g. MySQL
 					$sql = sprintf(SQL_INSERT_FEEDS, $bd_prefix_user);
 					$stm = $userPDO->bd->prepare($sql);
diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php
index a454829d5..ceca07f93 100644
--- a/app/SQL/install.sql.mysql.php
+++ b/app/SQL/install.sql.mysql.php
@@ -61,6 +61,29 @@ ENGINE = INNODB;
 INSERT IGNORE INTO `%1$scategory` (id, name) VALUES(1, "%2$s");
 ');
 
+define('SQL_CREATE_TABLE_ENTRYTMP', '
+CREATE TABLE IF NOT EXISTS `%1$sentrytmp` (	-- v1.7
+	`id` bigint NOT NULL,
+	`guid` varchar(760) CHARACTER SET latin1 NOT NULL,
+	`title` varchar(255) NOT NULL,
+	`author` varchar(255),
+	`content_bin` blob,
+	`link` varchar(1023) CHARACTER SET latin1 NOT NULL,
+	`date` int(11),
+	`lastSeen` INT(11) DEFAULT 0,
+	`hash` BINARY(16),
+	`is_read` boolean NOT NULL DEFAULT 0,
+	`is_favorite` boolean NOT NULL DEFAULT 0,
+	`id_feed` SMALLINT,
+	`tags` varchar(1023),
+	PRIMARY KEY (`id`),
+	FOREIGN KEY (`id_feed`) REFERENCES `%1$sfeed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+	UNIQUE KEY (`id_feed`,`guid`),
+	INDEX (`date`),
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
+ENGINE = INNODB;
+');
+
 define('SQL_INSERT_FEEDS', '
 INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("http://freshrss.org/feeds/all.atom.xml", 1, "FreshRSS.org", "http://freshrss.org/", "FreshRSS, a free, self-hostable aggregator…", 86400);
 INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS @ GitHub", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400);
diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php
index 9f4240b98..1a666cb8a 100644
--- a/app/SQL/install.sql.pgsql.php
+++ b/app/SQL/install.sql.pgsql.php
@@ -32,7 +32,7 @@ $SQL_CREATE_TABLES = array(
 
 'CREATE TABLE IF NOT EXISTS "%1$sentry" (
 	"id" BIGINT NOT NULL PRIMARY KEY,
-	"guid" VARCHAR(760) UNIQUE NOT NULL,
+	"guid" VARCHAR(760) NOT NULL,
 	"title" VARCHAR(255) NOT NULL,
 	"author" VARCHAR(255),
 	"content" TEXT,
@@ -54,6 +54,28 @@ $SQL_CREATE_TABLES = array(
 'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);',
 );
 
+global $SQL_CREATE_TABLE_ENTRYTMP;
+$SQL_CREATE_TABLE_ENTRYTMP = array(
+'CREATE TABLE IF NOT EXISTS "%1$sentrytmp" (	-- v1.7
+	"id" BIGINT NOT NULL PRIMARY KEY,
+	"guid" VARCHAR(760) NOT NULL,
+	"title" VARCHAR(255) NOT NULL,
+	"author" VARCHAR(255),
+	"content" TEXT,
+	"link" VARCHAR(1023) NOT NULL,
+	"date" INT,
+	"lastSeen" INT DEFAULT 0,
+	"hash" BYTEA,
+	"is_read" SMALLINT NOT NULL DEFAULT 0,
+	"is_favorite" SMALLINT NOT NULL DEFAULT 0,
+	"id_feed" SMALLINT,
+	"tags" VARCHAR(1023),
+	FOREIGN KEY ("id_feed") REFERENCES "%1$sfeed" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+	UNIQUE ("id_feed","guid")
+);',
+'CREATE INDEX %1$sentrytmp_date_index ON "%1$sentrytmp" ("date");',
+);
+
 global $SQL_INSERT_FEEDS;
 $SQL_INSERT_FEEDS = array(
 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'http://freshrss.org/feeds/all.atom.xml\');',
diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php
index 68d93ba92..ad7d525fd 100644
--- a/app/SQL/install.sql.sqlite.php
+++ b/app/SQL/install.sql.sqlite.php
@@ -57,6 +57,30 @@ $SQL_CREATE_TABLES = array(
 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");',
 );
 
+global $SQL_CREATE_TABLE_ENTRYTMP;
+$SQL_CREATE_TABLE_ENTRYTMP = array(
+'CREATE TABLE IF NOT EXISTS `entrytmp` (	-- v1.7
+	`id` bigint NOT NULL,
+	`guid` varchar(760) NOT NULL,
+	`title` varchar(255) NOT NULL,
+	`author` varchar(255),
+	`content` text,
+	`link` varchar(1023) NOT NULL,
+	`date` int(11),
+	`lastSeen` INT(11) DEFAULT 0,
+	`hash` BINARY(16),
+	`is_read` boolean NOT NULL DEFAULT 0,
+	`is_favorite` boolean NOT NULL DEFAULT 0,
+	`id_feed` SMALLINT,
+	`tags` varchar(1023),
+	PRIMARY KEY (`id`),
+	FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+	UNIQUE (`id_feed`,`guid`)
+);',
+
+'CREATE INDEX IF NOT EXISTS entrytmp_date_index ON `entrytmp`(`date`);',
+);
+
 global $SQL_INSERT_FEEDS;
 $SQL_INSERT_FEEDS = array(
 'INSERT OR IGNORE INTO `feed` (url, category, name, website, description, ttl) VALUES("http://freshrss.org/feeds/all.atom.xml", 1, "FreshRSS.org", "http://freshrss.org/", "FreshRSS, a free, self-hostable aggregator…", 86400);',
diff --git a/app/install.php b/app/install.php
index 9a88e0f37..17037c384 100644
--- a/app/install.php
+++ b/app/install.php
@@ -342,35 +342,19 @@ function checkDbUser(&$dbOptions) {
 	$driver_options = $dbOptions['options'];
 	try {
 		$c = new PDO($str, $dbOptions['user'], $dbOptions['password'], $driver_options);
-
 		if (defined('SQL_CREATE_TABLES')) {
-			$sql = sprintf(SQL_CREATE_TABLES, $dbOptions['prefix_user'], _t('gen.short.default_category'));
-			$stm = $c->prepare($sql);
-			$ok = $stm->execute();
-		} else {
-			global $SQL_CREATE_TABLES;
-			if (is_array($SQL_CREATE_TABLES)) {
-				$ok = true;
-				foreach ($SQL_CREATE_TABLES as $instruction) {
-					$sql = sprintf($instruction, $dbOptions['prefix_user'], _t('gen.short.default_category'));
-					$stm = $c->prepare($sql);
-					$ok &= $stm->execute();
-				}
-			}
-		}
-
-		if (defined('SQL_INSERT_FEEDS')) {
-			$sql = sprintf(SQL_INSERT_FEEDS, $dbOptions['prefix_user']);
+			$sql = sprintf(SQL_CREATE_TABLES . SQL_CREATE_TABLE_ENTRYTMP . SQL_INSERT_FEEDS,
+				$dbOptions['prefix_user'], _t('gen.short.default_category'));
 			$stm = $c->prepare($sql);
-			$ok &= $stm->execute();
+			$ok = $stm && $stm->execute();
 		} else {
-			global $SQL_INSERT_FEEDS;
-			if (is_array($SQL_INSERT_FEEDS)) {
-				foreach ($SQL_INSERT_FEEDS as $instruction) {
-					$sql = sprintf($instruction, $dbOptions['prefix_user']);
-					$stm = $c->prepare($sql);
-					$ok &= $stm->execute();
-				}
+			global $SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_INSERT_FEEDS;
+			$instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_INSERT_FEEDS);
+			$ok = !empty($instructions);
+			foreach ($instructions as $instruction) {
+				$sql = sprintf($instruction, $dbOptions['prefix_user'], _t('gen.short.default_category'));
+				$stm = $c->prepare($sql);
+				$ok &= $stm && $stm->execute();
 			}
 		}
 	} catch (PDOException $e) {
-- 
cgit v1.2.3


From a20fd9db9f0ed0e27c65671bb10402ced10587b1 Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Sun, 26 Mar 2017 01:41:08 +0100
Subject: Defered insertion MySQL bug

The update of cached values remains to be optimized
---
 app/Controllers/feedController.php | 1 +
 app/Models/EntryDAO.php            | 6 +++---
 app/SQL/install.sql.mysql.php      | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

(limited to 'app')

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index a2d9d5c35..b565d0439 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -436,6 +436,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 		}
 		if (!$noCommit) {
 			$entryDAO->commitNewEntries();
+			$feedDAO->updateCachedValues();	//TODO: Optimize
 		}
 		return array($updated_feeds, reset($feeds));
 	}
diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php
index decae9307..9d11cec6a 100644
--- a/app/Models/EntryDAO.php
+++ b/app/Models/EntryDAO.php
@@ -121,6 +121,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 	}
 
 	protected function autoUpdateDb($errorInfo) {
+		Minz_Log::warning('FreshRSS_EntryDAO::autoUpdateDb: ' . print_r($errorInfo, true));
 		if (isset($errorInfo[0])) {
 			if ($errorInfo[0] === '42S22') {	//ER_BAD_FIELD_ERROR
 				//autoAddColumn
@@ -202,16 +203,15 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 	}
 
 	public function commitNewEntries() {
-		$sql = 'SET @rank=SELECT MAX(id) - COUNT(*) FROM `' . $this->prefix . 'entrytmp`; ' .	//MySQL-specific
+		$sql = 'SET @rank=(SELECT MAX(id) - COUNT(*) FROM `' . $this->prefix . 'entrytmp`); ' .	//MySQL-specific
 			'INSERT IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' .
 				'SELECT @rank:=@rank+1 AS id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date; ' .
 			'DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= @rank;';
-		$stm = $this->bd->prepare($sql);
 		$hadTransaction = $this->bd->inTransaction();
 		if (!$hadTransaction) {
 			$this->bd->beginTransaction();
 		}
-		$result = $stm ? $stm->execute() : false;
+		$result = $this->bd->exec($sql) !== false;
 		if (!$hadTransaction) {
 			$this->bd->commit();
 		}
diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php
index ceca07f93..f42e08ad3 100644
--- a/app/SQL/install.sql.mysql.php
+++ b/app/SQL/install.sql.mysql.php
@@ -79,7 +79,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentrytmp` (	-- v1.7
 	PRIMARY KEY (`id`),
 	FOREIGN KEY (`id_feed`) REFERENCES `%1$sfeed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
 	UNIQUE KEY (`id_feed`,`guid`),
-	INDEX (`date`),
+	INDEX (`date`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
 ENGINE = INNODB;
 ');
-- 
cgit v1.2.3


From e956aee53d561fbdc11a78a50ad7cc041108e5b5 Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Sun, 26 Mar 2017 14:07:06 +0200
Subject: More defered insertion. New SQL index

New index entry_feed_read_index
TODO: Auto add this index to existing version
---
 app/Controllers/feedController.php |  8 +++++-
 app/Models/EntryDAO.php            |  1 -
 app/Models/Factory.php             |  9 +-----
 app/Models/FeedDAO.php             | 56 ++++++++++++++++++--------------------
 app/Models/FeedDAOSQLite.php       | 19 -------------
 app/SQL/install.sql.mysql.php      |  1 +
 app/SQL/install.sql.pgsql.php      |  1 +
 app/SQL/install.sql.sqlite.php     |  1 +
 8 files changed, 37 insertions(+), 59 deletions(-)
 delete mode 100644 app/Models/FeedDAOSQLite.php

(limited to 'app')

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index b565d0439..bfc8b2045 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -393,7 +393,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				}
 			}
 
-			$feedDAO->updateLastUpdate($feed->id(), false, $entryDAO->inTransaction(), $mtime);
+			$feedDAO->updateLastUpdate($feed->id(), false, $mtime);
 			if ($entryDAO->inTransaction()) {
 				$entryDAO->commit();
 			}
@@ -435,8 +435,14 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 			}
 		}
 		if (!$noCommit) {
+			if (!$entryDAO->inTransaction()) {
+				$entryDAO->beginTransaction();
+			}
 			$entryDAO->commitNewEntries();
 			$feedDAO->updateCachedValues();	//TODO: Optimize
+			if ($entryDAO->inTransaction()) {
+				$entryDAO->commit();
+			}
 		}
 		return array($updated_feeds, reset($feeds));
 	}
diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php
index 9d11cec6a..39c00f01c 100644
--- a/app/Models/EntryDAO.php
+++ b/app/Models/EntryDAO.php
@@ -121,7 +121,6 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 	}
 
 	protected function autoUpdateDb($errorInfo) {
-		Minz_Log::warning('FreshRSS_EntryDAO::autoUpdateDb: ' . print_r($errorInfo, true));
 		if (isset($errorInfo[0])) {
 			if ($errorInfo[0] === '42S22') {	//ER_BAD_FIELD_ERROR
 				//autoAddColumn
diff --git a/app/Models/Factory.php b/app/Models/Factory.php
index 6502c38b7..dfccc883e 100644
--- a/app/Models/Factory.php
+++ b/app/Models/Factory.php
@@ -3,14 +3,7 @@
 class FreshRSS_Factory {
 
 	public static function createFeedDao($username = null) {
-		$conf = Minz_Configuration::get('system');
-		switch ($conf->db['type']) {
-			case 'sqlite':
-			case 'pgsql':
-				return new FreshRSS_FeedDAOSQLite($username);
-			default:
-				return new FreshRSS_FeedDAO($username);
-		}
+		return new FreshRSS_FeedDAO($username);
 	}
 
 	public static function createEntryDao($username = null) {
diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php
index 0168aebd9..d278122e3 100644
--- a/app/Models/FeedDAO.php
+++ b/app/Models/FeedDAO.php
@@ -92,29 +92,15 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		}
 	}
 
-	public function updateLastUpdate($id, $inError = false, $updateCache = true, $mtime = 0) {
-		if ($updateCache) {
-			$sql = 'UPDATE `' . $this->prefix . 'feed` '	//2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE
-			     . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),'
-			     . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0),'
-			     . '`lastUpdate`=?, error=? '
-			     . 'WHERE id=?';
-		} else {
-			$sql = 'UPDATE `' . $this->prefix . 'feed` '
-			     . 'SET `lastUpdate`=?, error=? '
-			     . 'WHERE id=?';
-		}
-
-		if ($mtime <= 0) {
-			$mtime = time();
-		}
-
+	public function updateLastUpdate($id, $inError = false, $mtime = 0) {	//See also updateCachedValue()
+		$sql = 'UPDATE `' . $this->prefix . 'feed` '
+		     . 'SET `lastUpdate`=?, error=? '
+		     . 'WHERE id=?';
 		$values = array(
-			$mtime,
+			$mtime <= 0 ? time() : $mtime,
 			$inError ? 1 : 0,
 			$id,
 		);
-
 		$stm = $this->bd->prepare($sql);
 
 		if ($stm && $stm->execute($values)) {
@@ -294,18 +280,28 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return $res[0]['count'];
 	}
 
-	public function updateCachedValues() {	//For one single feed, call updateLastUpdate($id)
-		$sql = 'UPDATE `' . $this->prefix . 'feed` f '
-		     . 'INNER JOIN ('
-		     .	'SELECT e.id_feed, '
-		     .	'COUNT(CASE WHEN e.is_read = 0 THEN 1 END) AS nbUnreads, '
-		     .	'COUNT(e.id) AS nbEntries '
-		     .	'FROM `' . $this->prefix . 'entry` e '
-		     .	'GROUP BY e.id_feed'
-		     . ') x ON x.id_feed=f.id '
-		     . 'SET f.`cache_nbEntries`=x.nbEntries, f.`cache_nbUnreads`=x.nbUnreads';
+	public function updateCachedValue($id) {	//For multiple feeds, call updateCachedValues()
+		$sql = 'UPDATE `' . $this->prefix . 'feed` '	//2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE
+		     . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),'
+		     . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0) '
+		     . 'WHERE id=?';
+		$values = array($id);
 		$stm = $this->bd->prepare($sql);
 
+		if ($stm && $stm->execute($values)) {
+			return $stm->rowCount();
+		} else {
+			$info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo();
+			Minz_Log::error('SQL error updateCachedValue: ' . $info[2]);
+			return false;
+		}
+	}
+
+	public function updateCachedValues() {	//For one single feed, call updateCachedValue($id)
+		$sql = 'UPDATE `' . $this->prefix . 'feed` '
+		     . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),'
+		     . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0)';
+		$stm = $this->bd->prepare($sql);
 		if ($stm && $stm->execute()) {
 			return $stm->rowCount();
 		} else {
@@ -343,7 +339,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return $affected;
 	}
 
-	public function cleanOldEntries($id, $date_min, $keep = 15) {	//Remember to call updateLastUpdate($id) or updateCachedValues() just after
+	public function cleanOldEntries($id, $date_min, $keep = 15) {	//Remember to call updateCachedValue($id) or updateCachedValues() just after
 		$sql = 'DELETE FROM `' . $this->prefix . 'entry` '
 		     . 'WHERE id_feed=:id_feed AND id<=:id_max '
 		     . 'AND is_favorite=0 '	//Do not remove favourites
diff --git a/app/Models/FeedDAOSQLite.php b/app/Models/FeedDAOSQLite.php
deleted file mode 100644
index 440ae74da..000000000
--- a/app/Models/FeedDAOSQLite.php
+++ /dev/null
@@ -1,19 +0,0 @@
-<?php
-
-class FreshRSS_FeedDAOSQLite extends FreshRSS_FeedDAO {
-
-	public function updateCachedValues() {	//For one single feed, call updateLastUpdate($id)
-		$sql = 'UPDATE `' . $this->prefix . 'feed` '
-		     . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),'
-		     . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0)';
-		$stm = $this->bd->prepare($sql);
-		if ($stm && $stm->execute()) {
-			return $stm->rowCount();
-		} else {
-			$info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo();
-			Minz_Log::error('SQL error updateCachedValues: ' . $info[2]);
-			return false;
-		}
-	}
-
-}
diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php
index f42e08ad3..3ad68173f 100644
--- a/app/SQL/install.sql.mysql.php
+++ b/app/SQL/install.sql.mysql.php
@@ -55,6 +55,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` (
 	INDEX (`is_favorite`),	-- v0.7
 	INDEX (`is_read`),	-- v0.7
 	INDEX `entry_lastSeen_index` (`lastSeen`)	-- v1.1.1
+	INDEX `entry_feed_read_index` (`id_feed`,`is_read`)	-- v1.7	//TODO: Auto add this index to existing version
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
 ENGINE = INNODB;
 
diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php
index 1a666cb8a..ca4dfa925 100644
--- a/app/SQL/install.sql.pgsql.php
+++ b/app/SQL/install.sql.pgsql.php
@@ -50,6 +50,7 @@ $SQL_CREATE_TABLES = array(
 'CREATE INDEX %1$sis_favorite_index ON "%1$sentry" ("is_favorite");',
 'CREATE INDEX %1$sis_read_index ON "%1$sentry" ("is_read");',
 'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");',
+'CREATE INDEX %1$sentry_feed_read_index ON "%1$sentry" ("id_feed","is_read");',	//v1.7	//TODO: Auto add this index to existing version
 
 'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);',
 );
diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php
index ad7d525fd..dcb7a351a 100644
--- a/app/SQL/install.sql.sqlite.php
+++ b/app/SQL/install.sql.sqlite.php
@@ -53,6 +53,7 @@ $SQL_CREATE_TABLES = array(
 'CREATE INDEX IF NOT EXISTS entry_is_favorite_index ON `entry`(`is_favorite`);',
 'CREATE INDEX IF NOT EXISTS entry_is_read_index ON `entry`(`is_read`);',
 'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `entry`(`lastSeen`);',	//v1.1.1
+'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);',	//v1.7	//TODO: Auto add this index to existing version
 
 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");',
 );
-- 
cgit v1.2.3


From a7064dc700f1a41129511866cb2fb36c522c0d6c Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Sun, 26 Mar 2017 18:29:10 +0200
Subject: SQLite create index if not exists

---
 app/SQL/install.sql.sqlite.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php
index dcb7a351a..1fac5fcf7 100644
--- a/app/SQL/install.sql.sqlite.php
+++ b/app/SQL/install.sql.sqlite.php
@@ -53,7 +53,7 @@ $SQL_CREATE_TABLES = array(
 'CREATE INDEX IF NOT EXISTS entry_is_favorite_index ON `entry`(`is_favorite`);',
 'CREATE INDEX IF NOT EXISTS entry_is_read_index ON `entry`(`is_read`);',
 'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `entry`(`lastSeen`);',	//v1.1.1
-'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);',	//v1.7	//TODO: Auto add this index to existing version
+'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);',	//v1.7
 
 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");',
 );
@@ -80,6 +80,7 @@ $SQL_CREATE_TABLE_ENTRYTMP = array(
 );',
 
 'CREATE INDEX IF NOT EXISTS entrytmp_date_index ON `entrytmp`(`date`);',
+'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);',	//v1.7	//This line is used for auto-update
 );
 
 global $SQL_INSERT_FEEDS;
-- 
cgit v1.2.3


From bc580c5905522143cb3d73a37641f7bf21225545 Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Sun, 26 Mar 2017 22:08:17 +0200
Subject: SQL index auto added

At the same time as the new temp table
---
 app/SQL/install.sql.mysql.php  | 4 +++-
 app/SQL/install.sql.pgsql.php  | 3 ++-
 app/SQL/install.sql.sqlite.php | 7 ++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'app')

diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php
index 3ad68173f..6b07f5965 100644
--- a/app/SQL/install.sql.mysql.php
+++ b/app/SQL/install.sql.mysql.php
@@ -55,7 +55,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` (
 	INDEX (`is_favorite`),	-- v0.7
 	INDEX (`is_read`),	-- v0.7
 	INDEX `entry_lastSeen_index` (`lastSeen`)	-- v1.1.1
-	INDEX `entry_feed_read_index` (`id_feed`,`is_read`)	-- v1.7	//TODO: Auto add this index to existing version
+	-- INDEX `entry_feed_read_index` (`id_feed`,`is_read`)	-- v1.7 Located futher down
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
 ENGINE = INNODB;
 
@@ -83,6 +83,8 @@ CREATE TABLE IF NOT EXISTS `%1$sentrytmp` (	-- v1.7
 	INDEX (`date`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
 ENGINE = INNODB;
+
+CREATE INDEX `entry_feed_read_index` ON `%1$sentry`(`id_feed`,`is_read`);	-- v1.7 Located here to be auto-added
 ');
 
 define('SQL_INSERT_FEEDS', '
diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php
index ca4dfa925..74c6d1b79 100644
--- a/app/SQL/install.sql.pgsql.php
+++ b/app/SQL/install.sql.pgsql.php
@@ -50,7 +50,6 @@ $SQL_CREATE_TABLES = array(
 'CREATE INDEX %1$sis_favorite_index ON "%1$sentry" ("is_favorite");',
 'CREATE INDEX %1$sis_read_index ON "%1$sentry" ("is_read");',
 'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");',
-'CREATE INDEX %1$sentry_feed_read_index ON "%1$sentry" ("id_feed","is_read");',	//v1.7	//TODO: Auto add this index to existing version
 
 'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);',
 );
@@ -75,6 +74,8 @@ $SQL_CREATE_TABLE_ENTRYTMP = array(
 	UNIQUE ("id_feed","guid")
 );',
 'CREATE INDEX %1$sentrytmp_date_index ON "%1$sentrytmp" ("date");',
+
+'CREATE INDEX %1$sentry_feed_read_index ON "%1$sentry" ("id_feed","is_read");',	//v1.7
 );
 
 global $SQL_INSERT_FEEDS;
diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php
index 1fac5fcf7..bce9f5a95 100644
--- a/app/SQL/install.sql.sqlite.php
+++ b/app/SQL/install.sql.sqlite.php
@@ -26,7 +26,6 @@ $SQL_CREATE_TABLES = array(
 	FOREIGN KEY (`category`) REFERENCES `category`(`id`) ON DELETE SET NULL ON UPDATE CASCADE,
 	UNIQUE (`url`)
 );',
-
 'CREATE INDEX IF NOT EXISTS feed_name_index ON `feed`(`name`);',
 'CREATE INDEX IF NOT EXISTS feed_priority_index ON `feed`(`priority`);',
 'CREATE INDEX IF NOT EXISTS feed_keep_history_index ON `feed`(`keep_history`);',
@@ -49,11 +48,9 @@ $SQL_CREATE_TABLES = array(
 	FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
 	UNIQUE (`id_feed`,`guid`)
 );',
-
 'CREATE INDEX IF NOT EXISTS entry_is_favorite_index ON `entry`(`is_favorite`);',
 'CREATE INDEX IF NOT EXISTS entry_is_read_index ON `entry`(`is_read`);',
 'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `entry`(`lastSeen`);',	//v1.1.1
-'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);',	//v1.7
 
 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");',
 );
@@ -78,9 +75,9 @@ $SQL_CREATE_TABLE_ENTRYTMP = array(
 	FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
 	UNIQUE (`id_feed`,`guid`)
 );',
-
 'CREATE INDEX IF NOT EXISTS entrytmp_date_index ON `entrytmp`(`date`);',
-'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);',	//v1.7	//This line is used for auto-update
+
+'CREATE INDEX IF NOT EXISTS `entry_feed_read_index` ON `entry`(`id_feed`,`is_read`);',	//v1.7
 );
 
 global $SQL_INSERT_FEEDS;
-- 
cgit v1.2.3


From 5541e3951262bf93fc0eeb4938d6b93b01bfd1bd Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Mon, 27 Mar 2017 21:26:38 +0200
Subject: More defered insertion

---
 app/Controllers/feedController.php         | 10 ++++++++--
 app/Controllers/importExportController.php |  6 +++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

(limited to 'app')

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index bfc8b2045..5359ad198 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -321,6 +321,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				unset($newGuids);
 
 				$oldGuids = array();
+				$needFeedCacheRefresh = false;
 				// Add entries in database if possible.
 				foreach ($entries as $entry) {
 					$entry_date = $entry->date(true);
@@ -333,11 +334,12 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 							//Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->id() .
 								//', old hash ' . $existingHash . ', new hash ' . $entry->hash());
 							//TODO: Make an updated/is_read policy by feed, in addition to the global one.
+							$needFeedCacheRefresh = FreshRSS_Context::$user_conf->mark_updated_article_unread;
 							$entry->_isRead(FreshRSS_Context::$user_conf->mark_updated_article_unread ? false : null);	//Change is_read according to policy.
 							if (!$entryDAO->inTransaction()) {
 								$entryDAO->beginTransaction();
 							}
-							$entryDAO->updateEntry($entry->toArray());
+							$entryDAO->updateEntry($entry->toArray());	//TODO: Need to refresh cache
 						}
 					} elseif ($feed_history == 0 && $entry_date < $date_min) {
 						// This entry should not be added considering configuration and date.
@@ -388,12 +390,16 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				                                $date_min,
 				                                max($feed_history, count($entries) + 10));
 				if ($nb > 0) {
+					$needFeedCacheRefresh = true;
 					Minz_Log::debug($nb . ' old entries cleaned in feed [' .
 					                $feed->url() . ']');
 				}
 			}
 
 			$feedDAO->updateLastUpdate($feed->id(), false, $mtime);
+			if ($needFeedCacheRefresh) {
+				$feedDAO->updateCachedValue($feed->id());
+			}
 			if ($entryDAO->inTransaction()) {
 				$entryDAO->commit();
 			}
@@ -439,7 +445,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				$entryDAO->beginTransaction();
 			}
 			$entryDAO->commitNewEntries();
-			$feedDAO->updateCachedValues();	//TODO: Optimize
+			$feedDAO->updateCachedValues();
 			if ($entryDAO->inTransaction()) {
 				$entryDAO->commit();
 			}
diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php
index 82c49cc6a..2bc68848c 100644
--- a/app/Controllers/importExportController.php
+++ b/app/Controllers/importExportController.php
@@ -474,7 +474,11 @@ class FreshRSS_importExport_Controller extends Minz_ActionController {
 
 		}
 		$this->entryDAO->commit();
-		$entryDAO->commitNewEntries();
+
+		$this->entryDAO->beginTransaction();
+		$this->entryDAO->commitNewEntries();
+		$this->feedDAO->updateCachedValues();
+		$this->entryDAO->commit();
 
 		return !$error;
 	}
-- 
cgit v1.2.3


From 282ea0cfd782a69e7a9ca774d9a9e0f4f1b5c401 Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Mon, 27 Mar 2017 22:09:18 +0200
Subject: Defered insertion: feedController bug

---
 app/Controllers/feedController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index 5359ad198..bfa0ae160 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -308,6 +308,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				// -2 means we take the default value from configuration
 				$feed_history = FreshRSS_Context::$user_conf->keep_history_default;
 			}
+			$needFeedCacheRefresh = false;
 
 			// We want chronological order and SimplePie uses reverse order.
 			$entries = array_reverse($feed->entries());
@@ -321,7 +322,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				unset($newGuids);
 
 				$oldGuids = array();
-				$needFeedCacheRefresh = false;
 				// Add entries in database if possible.
 				foreach ($entries as $entry) {
 					$entry_date = $entry->date(true);
-- 
cgit v1.2.3


From 56bbdc543d02ef1ca92fa26aa18413d27a833748 Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Mon, 27 Mar 2017 22:16:59 +0200
Subject: Removing TODO

Done
---
 app/Controllers/feedController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index bfa0ae160..f6668d228 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -339,7 +339,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 							if (!$entryDAO->inTransaction()) {
 								$entryDAO->beginTransaction();
 							}
-							$entryDAO->updateEntry($entry->toArray());	//TODO: Need to refresh cache
+							$entryDAO->updateEntry($entry->toArray());
 						}
 					} elseif ($feed_history == 0 && $entry_date < $date_min) {
 						// This entry should not be added considering configuration and date.
-- 
cgit v1.2.3


From 6c604bc1dd75cc67392ff37186855faea3c29d48 Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Mon, 27 Mar 2017 23:42:19 +0200
Subject: Minz param bug fix

---
 app/Controllers/feedController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index f6668d228..07a763613 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -468,7 +468,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 		$id = Minz_Request::param('id');
 		$url = Minz_Request::param('url');
 		$force = Minz_Request::param('force');
-		$noCommit = Minz_Session::param('isLastFeed', 1) != 1;
+		$noCommit = Minz_Request::fetchPOST('isLastFeed', 1) != 1;
 
 		list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force, null, false, $noCommit);
 
-- 
cgit v1.2.3


From 0d4c26c673d548a1367a0f96f49546ca27619d90 Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Wed, 29 Mar 2017 21:42:40 +0200
Subject: Add manual commit & refresh cache to deferred insertion

---
 app/Controllers/feedController.php | 15 +++++++++++++--
 p/scripts/main.js                  | 15 +++++++++++----
 2 files changed, 24 insertions(+), 6 deletions(-)

(limited to 'app')

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index 07a763613..d1f0d3535 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -460,6 +460,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 	 *   - id (default: false): Feed ID
 	 *   - url (default: false): Feed URL
 	 *   - force (default: false)
+	 *   - noCommit (default: 0): Set to 1 to prevent committing the new articles to the main database
 	 * If id and url are not specified, all the feeds are actualized. But if force is
 	 * false, process stops at 10 feeds to avoid time execution problem.
 	 */
@@ -468,9 +469,19 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 		$id = Minz_Request::param('id');
 		$url = Minz_Request::param('url');
 		$force = Minz_Request::param('force');
-		$noCommit = Minz_Request::fetchPOST('isLastFeed', 1) != 1;
+		$noCommit = Minz_Request::fetchPOST('noCommit', 0) == 1;
 
-		list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force, null, false, $noCommit);
+		if ($id == -1 && !$noCommit) {	//Special request only to commit & refresh DB cache
+			$updated_feeds = 0;
+			$entryDAO = FreshRSS_Factory::createEntryDao();
+			$feedDAO = FreshRSS_Factory::createFeedDao();
+			$entryDAO->beginTransaction();
+			$entryDAO->commitNewEntries();
+			$feedDAO->updateCachedValues();
+			$entryDAO->commit();
+		} else {
+			list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force, null, false, $noCommit);
+		}
 
 		if (Minz_Request::param('ajax')) {
 			// Most of the time, ajax request is for only one feed. But since
diff --git a/p/scripts/main.js b/p/scripts/main.js
index c2f60bf7f..9aca75b93 100644
--- a/p/scripts/main.js
+++ b/p/scripts/main.js
@@ -803,13 +803,12 @@ function updateFeed(feeds, feeds_count) {
 	if (!feed) {
 		return;
 	}
-
 	$.ajax({
 		type: 'POST',
 		url: feed.url,
 		data : {
 			_csrf: context.csrf,
-			isLastFeed: feeds.length <= 0 ? 1 : 0,
+			noCommit: feeds.length > 0 ? 1 : 0,
 		},
 	}).always(function (data) {
 		feed_processed++;
@@ -831,7 +830,6 @@ function init_actualize() {
 		if (ajax_loading) {
 			return false;
 		}
-
 		ajax_loading = true;
 
 		$.getJSON('./?c=javascript&a=actualize').done(function (data) {
@@ -842,7 +840,16 @@ function init_actualize() {
 			}
 			if (data.feeds.length === 0) {
 				openNotification(data.feedback_no_refresh, "good");
-				ajax_loading = false;
+				$.ajax({	//Empty request to force refresh server database cache
+					type: 'POST',
+					url: './?c=feed&a=actualize&id=-1',
+					data : {
+						_csrf: context.csrf,
+						noCommit: 0,
+					},
+				}).always(function (data) {
+					ajax_loading = false;
+				});
 				return;
 			}
 			//Progress bar
-- 
cgit v1.2.3