From 7c1b5e322cca0134f57b3a436129985ba9170b9f Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Tue, 2 Aug 2016 22:49:35 +0200 Subject: PostgreSQL draft https://github.com/FreshRSS/FreshRSS/issues/416 Based on @Damstre work https://github.com/FreshRSS/FreshRSS/pull/1071 Not tested --- app/SQL/install.sql.pgsql.php | 91 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 app/SQL/install.sql.pgsql.php (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php new file mode 100644 index 000000000..0f243bdb5 --- /dev/null +++ b/app/SQL/install.sql.pgsql.php @@ -0,0 +1,91 @@ + Date: Tue, 23 Aug 2016 00:02:54 +0200 Subject: More PostgreSQL --- app/Models/EntryDAO.php | 75 ++++++++++++++--------- app/Models/EntryDAOPGSQL.php | 79 +++--------------------- app/Models/EntryDAOSQLite.php | 4 ++ app/SQL/install.sql.pgsql.php | 136 +++++++++++++++++------------------------- app/install.php | 26 ++------ lib/Minz/ModelPdo.php | 29 +++++---- 6 files changed, 138 insertions(+), 211 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 3a1dc1ba5..8d136cd6c 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -10,6 +10,14 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return parent::$sharedDbType !== 'sqlite'; } + public function sqlHexDecode($x) { + return 'X' . $x; + } + + public function sqlHexEncode($x) { + return 'hex(' . $x . ')'; + } + protected function addColumn($name) { Minz_Log::warning('FreshRSS_EntryDAO::addColumn: ' . $name); $hasTransaction = false; @@ -106,31 +114,42 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $sql = 'INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, ' . ($this->isCompressed() ? 'content_bin' : 'content') . ', link, date, lastSeen, hash, is_read, is_favorite, id_feed, tags) ' - . 'VALUES(?, ?, ?, ?, ' - . ($this->isCompressed() ? 'COMPRESS(?)' : '?') - . ', ?, ?, ?, ' - . ($this->hasNativeHex() ? 'X?' : '?') - . ', ?, ?, ?, ?)'; + . 'VALUES(:id, :guid, :title, :author, ' + . ($this->isCompressed() ? 'COMPRESS(:content)' : ':content') + . ', :link, :date, :last_seen, ' + . $this->sqlHexDecode(':hash') + . ', :is_read, :is_favorite, :id_feed, :tags)'; $this->addEntryPrepared = $this->bd->prepare($sql); } + $this->addEntryPrepared->bindParam(':id', $valuesTmp['id']); + $valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760); + $this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']); + $valuesTmp['title'] = substr($valuesTmp['title'], 0, 255); + $this->addEntryPrepared->bindParam(':title', $valuesTmp['title']); + $valuesTmp['author'] = substr($valuesTmp['author'], 0, 255); + $this->addEntryPrepared->bindParam(':author', $valuesTmp['author']); + $this->addEntryPrepared->bindParam(':content', $valuesTmp['content']); + $valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023); + $this->addEntryPrepared->bindParam(':link', $valuesTmp['link']); + $this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT); + $valuesTmp['lastSeen'] = time(); + $this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT); + $valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0; + $this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT); + $valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0; + $this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT); + $this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT); + $valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023); + $this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']); + + if ($this->hasNativeHex()) { + $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']); + } else { + $valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']); + $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']); // X'09AF' hexadecimal literals do not work with SQLite/PDO //hex2bin() is PHP5.4+ + } - $values = array( - $valuesTmp['id'], - substr($valuesTmp['guid'], 0, 760), - substr($valuesTmp['title'], 0, 255), - substr($valuesTmp['author'], 0, 255), - $valuesTmp['content'], - substr($valuesTmp['link'], 0, 1023), - $valuesTmp['date'], - time(), - $this->hasNativeHex() ? $valuesTmp['hash'] : pack('H*', $valuesTmp['hash']), // X'09AF' hexadecimal literals do not work with SQLite/PDO //hex2bin() is PHP5.4+ - $valuesTmp['is_read'] ? 1 : 0, - $valuesTmp['is_favorite'] ? 1 : 0, - $valuesTmp['id_feed'], - substr($valuesTmp['tags'], 0, 1023), - ); - - if ($this->addEntryPrepared && $this->addEntryPrepared->execute($values)) { + if ($this->addEntryPrepared && $this->addEntryPrepared->execute()) { return $this->bd->lastInsertId(); } else { $info = $this->addEntryPrepared == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $this->addEntryPrepared->errorInfo(); @@ -156,7 +175,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { . 'SET title=?, author=?, ' . ($this->isCompressed() ? 'content_bin=COMPRESS(?)' : 'content=?') . ', link=?, date=?, lastSeen=?, hash=' - . ($this->hasNativeHex() ? 'X?' : '?') + . ($this->hasNativeHex() ? 'X?' : '?') //TODO PostgreSQL . ', ' . ($valuesTmp['is_read'] === null ? '' : 'is_read=?, ') . 'tags=? ' . 'WHERE id_feed=? AND guid=?'; @@ -430,12 +449,12 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } $this->bd->beginTransaction(); - $sql = 'UPDATE `' . $this->prefix . 'entry` e ' - . 'SET e.is_read=1 ' - . 'WHERE e.id_feed=? AND e.is_read=0 AND e.id <= ?'; + $sql = 'UPDATE `' . $this->prefix . 'entry` ' + . 'SET is_read=1 ' + . 'WHERE id_feed=? AND is_read=0 AND id <= ?'; $values = array($id_feed, $idMax); - list($searchValues, $search) = $this->sqlListEntriesWhere('e.', $filter, $state); + list($searchValues, $search) = $this->sqlListEntriesWhere('', $filter, $state); $stm = $this->bd->prepare($sql . $search); if (!($stm && $stm->execute(array_merge($values, $searchValues)))) { @@ -658,7 +677,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { if (count($guids) < 1) { return array(); } - $sql = 'SELECT guid, hex(hash) AS hexHash FROM `' . $this->prefix . 'entry` WHERE id_feed=? AND guid IN (' . str_repeat('?,', count($guids) - 1). '?)'; + $sql = 'SELECT guid, ' . $this->sqlHexEncode('hash') . ' AS hexHash FROM `' . $this->prefix . 'entry` WHERE id_feed=? AND guid IN (' . str_repeat('?,', count($guids) - 1). '?)'; $stm = $this->bd->prepare($sql); $values = array($id_feed); $values = array_merge($values, $guids); diff --git a/app/Models/EntryDAOPGSQL.php b/app/Models/EntryDAOPGSQL.php index 95c12ff5d..b96a62ebc 100644 --- a/app/Models/EntryDAOPGSQL.php +++ b/app/Models/EntryDAOPGSQL.php @@ -1,82 +1,21 @@ bd->beginTransaction(); - - $sql = 'UPDATE "' . $this->prefix . 'entry" ' - . 'SET is_read=:is_read ' - . 'WHERE id_feed=:id_feed AND NOT is_read AND id <= :idmax'; - $values = array($id_feed, $idMax); - $stm = $this->bd->prepare($sql); - $stm->bindValue(':is_read', true, PDO::PARAM_BOOL); - $stm->bindValue(':id_feed', $id_feed); - $stm->bindValue(':idmax', $idMax); - - if (!($stm && $stm->execute())) { - $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); - Minz_Log::error('SQL error markReadFeed: ' . $info[2]); - $this->bd->rollBack(); - return false; - } - $affected = $stm->rowCount(); - - $this->bd->commit(); - return $affected; + public function sqlHexEncode($x) { + return 'encode(' . $x . ", 'hex')"; } - public function listHashForFeedGuids($id_feed, $guids) { - if (count($guids) < 1) { - return array(); - } - $sql = 'SELECT guid, hash AS hexHash FROM "' . $this->prefix . 'entry" WHERE id_feed=? AND guid IN (' . str_repeat('?,', count($guids) - 1). '?)'; - $stm = $this->bd->prepare($sql); - $values = array($id_feed); - $values = array_merge($values, $guids); - if ($stm && $stm->execute($values)) { - $result = array(); - $rows = $stm->fetchAll(PDO::FETCH_ASSOC); - foreach ($rows as $row) { - $result[$row['guid']] = $row['hexHash']; - } - return $result; - } else { - $info = $stm == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $stm->errorInfo(); - if ($this->autoAddColumn($info)) { - return $this->listHashForFeedGuids($id_feed, $guids); - } - Minz_Log::error('SQL error listHashForFeedGuids: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] - . ' while querying feed ' . $id_feed); - return false; - } + protected function autoUpdateDb($errorInfo) { + return false; } - public function optimizeTable() { - return null; + protected function addColumn($name) { + return false; } public function size($all = true) { diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php index dad34a93d..80dbcca6b 100644 --- a/app/Models/EntryDAOSQLite.php +++ b/app/Models/EntryDAOSQLite.php @@ -2,6 +2,10 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO { + public function sqlHexDecode($x) { + return $x; + } + protected function autoUpdateDb($errorInfo) { if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR //autoAddColumn diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 0f243bdb5..04e35af68 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -1,91 +1,65 @@ 'SET NAMES utf8mb4', - ); - break; - case 'sqlite': - $str = 'sqlite:' . join_path(USERS_PATH, $_SESSION['default_user'], 'db.sqlite'); - $driver_options = array( - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - ); - break; - default: - return false; - } - return new PDO($str, $_SESSION['bd_user'], $_SESSION['bd_password'], $driver_options); -} - function deleteInstall() { $res = unlink(join_path(DATA_PATH, 'do-install.txt')); @@ -444,11 +427,12 @@ function checkBD() { ); try { // on ouvre une connexion juste pour créer la base si elle n'existe pas - $str = 'pgsql:host=' . $_SESSION['bd_host'] . ';'; + $str = 'pgsql:host=' . $_SESSION['bd_host'] . ';dbname=postgres'; $c = new PDO($str, $_SESSION['bd_user'], $_SESSION['bd_password'], $driver_options); $sql = sprintf(SQL_CREATE_DB, $_SESSION['bd_base']); $res = $c->query($sql); } catch (PDOException $e) { + syslog(LOG_DEBUG, 'pgsql ' . $e->getMessage()); } // on écrase la précédente connexion en sélectionnant la nouvelle BDD diff --git a/lib/Minz/ModelPdo.php b/lib/Minz/ModelPdo.php index b98a26d06..41a9f60bf 100644 --- a/lib/Minz/ModelPdo.php +++ b/lib/Minz/ModelPdo.php @@ -60,17 +60,17 @@ class Minz_ModelPdo { $string = 'mysql:host=' . $db['host'] . ';dbname=' . $db['base'] . ';charset=utf8mb4'; $driver_options[PDO::MYSQL_ATTR_INIT_COMMAND] = 'SET NAMES utf8mb4'; $this->prefix = $db['prefix'] . $currentUser . '_'; - $this->bd = new MinzPDO($string, $db['user'], $db['password'], $driver_options); + $this->bd = new MinzPDOMySql($string, $db['user'], $db['password'], $driver_options); //TODO Consider: $this->bd->exec("SET SESSION sql_mode = 'ANSI_QUOTES';"); break; case 'sqlite': $string = 'sqlite:' . join_path(DATA_PATH, 'users', $currentUser, 'db.sqlite'); $this->prefix = ''; - $this->bd = new MinzPDO($string, $db['user'], $db['password'], $driver_options); + $this->bd = new MinzPDOMSQLite($string, $db['user'], $db['password'], $driver_options); $this->bd->exec('PRAGMA foreign_keys = ON;'); break; case 'pgsql': - $string = 'pgsql:host=' . $db['host'] . ';dbname=' . $db['base'] . ';charset=utf8'; + $string = 'pgsql:host=' . $db['host'] . ';dbname=' . $db['base']; $this->prefix = $db['prefix'] . $currentUser . '_'; $this->bd = new MinzPDOPGSQL($string, $db['user'], $db['password'], $driver_options); $this->bd->exec("SET NAMES 'UTF8';"); @@ -125,33 +125,40 @@ class MinzPDO extends PDO { public function prepare($statement, $driver_options = array()) { MinzPDO::check($statement); - $statement = MinzPDO::compatibility($statement); + $statement = $this->compatibility($statement); return parent::prepare($statement, $driver_options); } public function exec($statement) { MinzPDO::check($statement); - $statement = MinzPDO::compatibility($statement); + $statement = $this->compatibility($statement); return parent::exec($statement); } public function query($statement) { MinzPDO::check($statement); - $statement = MinzPDO::compatibility($statement); + $statement = $this->compatibility($statement); return parent::query($statement); } +} +class MinzPDOMySql extends PDO { public function lastInsertId($name = null) { return parent::lastInsertId(); //We discard the name, only used by PostgreSQL } } -class MinzPDOPGSQL extends MinzPDO { - protected function compatibility($statement) { - return str_replace(array('`', " X'"), array('"', " E'\\x"), $statement); +class MinzPDOMSQLite extends PDO { + public function lastInsertId($name = null) { + return parent::lastInsertId(); //We discard the name, only used by PostgreSQL } +} - public function lastInsertId($name = null) { - return parent::lastInsertId($name); +class MinzPDOPGSQL extends MinzPDO { + protected function compatibility($statement) { + return str_replace( + array('`', 'lastUpdate', 'pathEntries', 'httpAuth', 'cache_nbEntries', 'cache_nbUnreads', 'lastSeen'), + array('"', '"lastUpdate"', '"pathEntries"', '"httpAuth"', '"cache_nbEntries"', '"cache_nbUnreads"', '"lastSeen"'), + $statement); } } -- cgit v1.2.3 From c16a973d453078a037a2a6a6e8e3d17ffd3bf2e6 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 1 Oct 2016 09:24:46 +0200 Subject: PostgreSQL schema update --- app/Models/StatsDAOPGSQL.php | 2 +- app/SQL/install.sql.pgsql.php | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/Models/StatsDAOPGSQL.php b/app/Models/StatsDAOPGSQL.php index 649dfa8b5..1effbb64b 100644 --- a/app/Models/StatsDAOPGSQL.php +++ b/app/Models/StatsDAOPGSQL.php @@ -61,7 +61,7 @@ SQL; $repartition[(int) $value['period']] = (int) $value['count']; } - return $this->convertToSerie($repartition); + return $repartition; } } diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 04e35af68..c2c32ef59 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -24,7 +24,7 @@ $SQL_CREATE_TABLES = array( "ttl" INT NOT NULL DEFAULT -2, "cache_nbEntries" int DEFAULT 0, "cache_nbUnreads" int DEFAULT 0, - FOREIGN KEY (category) REFERENCES "%1$scategory" ("id") ON DELETE SET NULL ON UPDATE CASCADE + FOREIGN KEY ("category") REFERENCES "%1$scategory" ("id") ON DELETE SET NULL ON UPDATE CASCADE );', 'CREATE INDEX name_index ON "%1$sfeed" ("name");', 'CREATE INDEX priority_index ON "%1$sfeed" ("priority");', @@ -44,19 +44,13 @@ $SQL_CREATE_TABLES = array( "is_favorite" smallint NOT NULL DEFAULT 0, "id_feed" SMALLINT, "tags" varchar(1023), - FOREIGN KEY (id_feed) REFERENCES "%1$sfeed" (id) ON DELETE CASCADE ON UPDATE CASCADE + FOREIGN KEY ("id_feed") REFERENCES "%1$sfeed" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + UNIQUE ("id_feed","guid") );', 'CREATE INDEX is_favorite_index ON "%1$sentry" ("is_favorite");', 'CREATE INDEX is_read_index ON "%1$sentry" ("is_read");', 'CREATE INDEX entry_lastSeen_index ON "%1$sentry" ("lastSeen");', - -'CREATE OR REPLACE RULE check_constraints_on_entry AS ON INSERT TO "%1$sentry" WHERE EXISTS(SELECT 1 FROM "%1$sentry" WHERE guid=NEW.guid) DO INSTEAD NOTHING;', -'CREATE OR REPLACE RULE check_constraints_on_feed AS ON INSERT TO "%1$sfeed" WHERE EXISTS(SELECT 1 FROM "%1$sfeed" WHERE url=NEW.url) DO INSTEAD NOTHING;', -'CREATE OR REPLACE RULE check_constraints_on_category AS ON INSERT TO "%1$scategory" WHERE EXISTS(SELECT 1 FROM "%1$scategory" WHERE name=NEW.name) DO INSTEAD NOTHING;', -'CREATE OR REPLACE RULE check_constraints_on_category as on update to "%1$scategory" WHERE EXISTS(SELECT 1 FROM "%1$scategory" WHERE name=NEW.name) DO INSTEAD NOTHING;', - - 'INSERT INTO "%1$scategory" (id, name) VALUES(1, \'%2$s\');', 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) VALUES(\'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400);', 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) VALUES(\'https://github.com/FreshRSS/FreshRSS/releases.atom\', 1, \'FreshRSS @ GitHub\', \'https://github.com/FreshRSS/FreshRSS/\', \'FreshRSS releases @ GitHub\', 86400);', -- cgit v1.2.3 From e979b67ceb31c42955843b2c0c70cd4dfb9a4e95 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 1 Oct 2016 09:34:09 +0200 Subject: PostgreSQL uppercase keywords --- app/SQL/install.sql.pgsql.php | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index c2c32ef59..e60c5a07b 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -5,25 +5,25 @@ global $SQL_CREATE_TABLES; $SQL_CREATE_TABLES = array( 'CREATE TABLE IF NOT EXISTS "%1$scategory" ( "id" SERIAL PRIMARY KEY, - "name" varchar(255) UNIQUE NOT NULL + "name" VARCHAR(255) UNIQUE NOT NULL );', 'CREATE TABLE IF NOT EXISTS "%1$sfeed" ( "id" SERIAL PRIMARY KEY, "url" varchar(511) UNIQUE NOT NULL, "category" SMALLINT DEFAULT 0, - "name" varchar(255) NOT NULL, - "website" varchar(255), + "name" VARCHAR(255) NOT NULL, + "website" VARCHAR(255), "description" text, - "lastUpdate" int DEFAULT 0, - "priority" smallint NOT NULL DEFAULT 10, - "pathEntries" varchar(511) DEFAULT NULL, - "httpAuth" varchar(511) DEFAULT NULL, + "lastUpdate" INT DEFAULT 0, + "priority" SMALLINT NOT NULL DEFAULT 10, + "pathEntries" VARCHAR(511) DEFAULT NULL, + "httpAuth" VARCHAR(511) DEFAULT NULL, "error" smallint DEFAULT 0, "keep_history" INT NOT NULL DEFAULT -2, "ttl" INT NOT NULL DEFAULT -2, - "cache_nbEntries" int DEFAULT 0, - "cache_nbUnreads" int DEFAULT 0, + "cache_nbEntries" INT DEFAULT 0, + "cache_nbUnreads" INT DEFAULT 0, FOREIGN KEY ("category") REFERENCES "%1$scategory" ("id") ON DELETE SET NULL ON UPDATE CASCADE );', 'CREATE INDEX name_index ON "%1$sfeed" ("name");', @@ -31,19 +31,19 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX keep_history_index ON "%1$sfeed" ("keep_history");', 'CREATE TABLE IF NOT EXISTS "%1$sentry" ( - "id" bigint NOT NULL PRIMARY KEY, - "guid" varchar(760) UNIQUE NOT NULL, - "title" varchar(255) NOT NULL, - "author" varchar(255), - "content" text, - "link" varchar(1023) NOT NULL, + "id" BIGINT NOT NULL PRIMARY KEY, + "guid" VARCHAR(760) UNIQUE NOT NULL, + "title" VARCHAR(255) NOT NULL, + "author" VARCHAR(255), + "content" TEXT, + "link" VARCHAR(1023) NOT NULL, "date" INT, "lastSeen" INT DEFAULT 0, "hash" BYTEA, - "is_read" smallint NOT NULL DEFAULT 0, - "is_favorite" smallint NOT NULL DEFAULT 0, + "is_read" SMALLINT NOT NULL DEFAULT 0, + "is_favorite" SMALLINT NOT NULL DEFAULT 0, "id_feed" SMALLINT, - "tags" varchar(1023), + "tags" VARCHAR(1023), FOREIGN KEY ("id_feed") REFERENCES "%1$sfeed" ("id") ON DELETE CASCADE ON UPDATE CASCADE, UNIQUE ("id_feed","guid") );', -- cgit v1.2.3 From 9b22272eee6e37963f6ed5f60b9f82382b23adc2 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 1 Oct 2016 20:54:08 +0200 Subject: PostgreSQL multiuser bug https://github.com/FreshRSS/FreshRSS/pull/1195 --- app/Models/UserDAO.php | 5 +++-- app/SQL/install.sql.pgsql.php | 12 ++++++------ app/SQL/install.sql.sqlite.php | 32 ++++++++++++++++---------------- 3 files changed, 25 insertions(+), 24 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/Models/UserDAO.php b/app/Models/UserDAO.php index b55766ab4..e35be848c 100644 --- a/app/Models/UserDAO.php +++ b/app/Models/UserDAO.php @@ -8,8 +8,9 @@ class FreshRSS_UserDAO extends Minz_ModelPdo { $userPDO = new Minz_ModelPdo($username); $ok = false; + $bd_prefix_user = $db['prefix'] . $username . '_'; if (defined('SQL_CREATE_TABLES')) { //E.g. MySQL - $sql = sprintf(SQL_CREATE_TABLES, $db['prefix'] . $username . '_', _t('gen.short.default_category')); + $sql = sprintf(SQL_CREATE_TABLES, $bd_prefix_user, _t('gen.short.default_category')); $stm = $userPDO->bd->prepare($sql); $ok = $stm && $stm->execute(); } else { //E.g. SQLite @@ -17,7 +18,7 @@ class FreshRSS_UserDAO extends Minz_ModelPdo { if (is_array($SQL_CREATE_TABLES)) { $ok = true; foreach ($SQL_CREATE_TABLES as $instruction) { - $sql = sprintf($instruction, '', _t('gen.short.default_category')); + $sql = sprintf($instruction, $bd_prefix_user, _t('gen.short.default_category')); $stm = $userPDO->bd->prepare($sql); $ok &= ($stm && $stm->execute()); } diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index e60c5a07b..841003def 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -26,9 +26,9 @@ $SQL_CREATE_TABLES = array( "cache_nbUnreads" INT DEFAULT 0, FOREIGN KEY ("category") REFERENCES "%1$scategory" ("id") ON DELETE SET NULL ON UPDATE CASCADE );', -'CREATE INDEX name_index ON "%1$sfeed" ("name");', -'CREATE INDEX priority_index ON "%1$sfeed" ("priority");', -'CREATE INDEX keep_history_index ON "%1$sfeed" ("keep_history");', +'CREATE INDEX %1$sname_index ON "%1$sfeed" ("name");', +'CREATE INDEX %1$spriority_index ON "%1$sfeed" ("priority");', +'CREATE INDEX %1$skeep_history_index ON "%1$sfeed" ("keep_history");', 'CREATE TABLE IF NOT EXISTS "%1$sentry" ( "id" BIGINT NOT NULL PRIMARY KEY, @@ -47,9 +47,9 @@ $SQL_CREATE_TABLES = array( FOREIGN KEY ("id_feed") REFERENCES "%1$sfeed" ("id") ON DELETE CASCADE ON UPDATE CASCADE, UNIQUE ("id_feed","guid") );', -'CREATE INDEX is_favorite_index ON "%1$sentry" ("is_favorite");', -'CREATE INDEX is_read_index ON "%1$sentry" ("is_read");', -'CREATE INDEX entry_lastSeen_index ON "%1$sentry" ("lastSeen");', +'CREATE INDEX %1$sis_favorite_index ON "%1$sentry" ("is_favorite");', +'CREATE INDEX %1$sis_read_index ON "%1$sentry" ("is_read");', +'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");', 'INSERT INTO "%1$scategory" (id, name) VALUES(1, \'%2$s\');', 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) VALUES(\'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400);', diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 87d5cf286..9d146d6f0 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -1,16 +1,16 @@ Date: Sat, 15 Oct 2016 00:04:51 +0200 Subject: PostgreSQL avoid duplicate category 1 https://github.com/FreshRSS/FreshRSS/issues/1312 --- app/SQL/install.sql.pgsql.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 841003def..6e1d4a103 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -51,7 +51,7 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX %1$sis_read_index ON "%1$sentry" ("is_read");', 'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");', -'INSERT INTO "%1$scategory" (id, name) VALUES(1, \'%2$s\');', +'INSERT INTO "%1$scategory" (id, name) SELECT 1, \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);', 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) VALUES(\'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400);', 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) VALUES(\'https://github.com/FreshRSS/FreshRSS/releases.atom\', 1, \'FreshRSS @ GitHub\', \'https://github.com/FreshRSS/FreshRSS/\', \'FreshRSS releases @ GitHub\', 86400);', ); -- cgit v1.2.3 From c24d60372ad59932638964e37cb84ba0de8b1661 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Oct 2016 00:49:10 +0200 Subject: PostgreSQL emulate "INSERT IGNORE" Would have liked to use UPSERT, but it is first available from PostgreSQL 9.5. --- app/SQL/install.sql.pgsql.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 6e1d4a103..91c5b257d 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -52,8 +52,8 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");', 'INSERT INTO "%1$scategory" (id, name) SELECT 1, \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);', -'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) VALUES(\'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400);', -'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) VALUES(\'https://github.com/FreshRSS/FreshRSS/releases.atom\', 1, \'FreshRSS @ GitHub\', \'https://github.com/FreshRSS/FreshRSS/\', \'FreshRSS releases @ GitHub\', 86400);', +'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'http://freshrss.org/feeds/all.atom.xml\');', +'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'https://github.com/FreshRSS/FreshRSS/releases.atom\', 1, \'FreshRSS @ GitHub\', \'https://github.com/FreshRSS/FreshRSS/\', \'FreshRSS releases @ GitHub\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'https://github.com/FreshRSS/FreshRSS/releases.atom\');', ); define('SQL_DROP_TABLES', 'DROP TABLES "%1$sentry", "%1$sfeed", "%1$scategory"'); -- cgit v1.2.3 From 34e38e3a97814eae27174839dd04bcf79c3d62b5 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Oct 2016 20:01:48 +0200 Subject: SQL DROP TABLE https://github.com/FreshRSS/FreshRSS/issues/1320 --- app/SQL/install.sql.mysql.php | 2 +- app/SQL/install.sql.pgsql.php | 2 +- app/SQL/install.sql.sqlite.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index 92a00aecc..ca181303e 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -63,7 +63,7 @@ INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) V INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS @ GitHub", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400); '); -define('SQL_DROP_TABLES', 'DROP TABLES %1$sentry, %1$sfeed, %1$scategory'); +define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS `%1$sentry`, `%1$sfeed`, `%1$scategory`'); define('SQL_UPDATE_UTF8MB4', ' ALTER DATABASE `%2$s` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 91c5b257d..0b86908cd 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -56,4 +56,4 @@ $SQL_CREATE_TABLES = array( 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'https://github.com/FreshRSS/FreshRSS/releases.atom\', 1, \'FreshRSS @ GitHub\', \'https://github.com/FreshRSS/FreshRSS/\', \'FreshRSS releases @ GitHub\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'https://github.com/FreshRSS/FreshRSS/releases.atom\');', ); -define('SQL_DROP_TABLES', 'DROP TABLES "%1$sentry", "%1$sfeed", "%1$scategory"'); +define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS "%1$sentry", "%1$sfeed", "%1$scategory"'); diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 9d146d6f0..1d3a5d92f 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -59,4 +59,4 @@ $SQL_CREATE_TABLES = array( 'INSERT OR IGNORE INTO `feed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS releases", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400);', ); -define('SQL_DROP_TABLES', 'DROP TABLES entry, feed, category'); +define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS entry, feed, category'); -- cgit v1.2.3 From f1eaf02a0c323c401950bca6189a795915193c3a Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Oct 2016 23:09:21 +0200 Subject: PostgreSQL workaround for SERIAL strange behaviour http://stackoverflow.com/questions/18389537/does-postgresql-serial-work-differently --- app/SQL/install.sql.pgsql.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 0b86908cd..b343bda86 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -51,7 +51,7 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX %1$sis_read_index ON "%1$sentry" ("is_read");', 'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");', -'INSERT INTO "%1$scategory" (id, name) SELECT 1, \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);', +'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);', 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'http://freshrss.org/feeds/all.atom.xml\');', 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'https://github.com/FreshRSS/FreshRSS/releases.atom\', 1, \'FreshRSS @ GitHub\', \'https://github.com/FreshRSS/FreshRSS/\', \'FreshRSS releases @ GitHub\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'https://github.com/FreshRSS/FreshRSS/releases.atom\');', ); -- cgit v1.2.3 From 1182129ce5f07892afed190ffbb2ea4c7fc28967 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 24 Oct 2016 20:29:08 +0200 Subject: CLI option no-default-feeds https://github.com/FreshRSS/FreshRSS/issues/1095 --- app/Controllers/userController.php | 4 ++-- app/Models/UserDAO.php | 18 +++++++++++++++++- app/SQL/install.sql.mysql.php | 3 +++ app/SQL/install.sql.pgsql.php | 4 ++++ app/SQL/install.sql.sqlite.php | 4 ++++ app/install.php | 15 +++++++++++++++ cli/create-user.php | 7 +++++-- 7 files changed, 50 insertions(+), 5 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/Controllers/userController.php b/app/Controllers/userController.php index 9dee16e8c..9d6ae18e6 100644 --- a/app/Controllers/userController.php +++ b/app/Controllers/userController.php @@ -99,7 +99,7 @@ class FreshRSS_user_Controller extends Minz_ActionController { $this->view->size_user = $entryDAO->size(); } - public static function createUser($new_user_name, $passwordPlain, $apiPasswordPlain, $userConfig = array()) { + public static function createUser($new_user_name, $passwordPlain, $apiPasswordPlain, $userConfig = array(), $insertDefaultFeeds = true) { if (!is_array($userConfig)) { $userConfig = array(); } @@ -138,7 +138,7 @@ class FreshRSS_user_Controller extends Minz_ActionController { } if ($ok) { $userDAO = new FreshRSS_UserDAO(); - $ok &= $userDAO->createUser($new_user_name, $userConfig['language']); + $ok &= $userDAO->createUser($new_user_name, $userConfig['language'], $insertDefaultFeeds); } return $ok; } diff --git a/app/Models/UserDAO.php b/app/Models/UserDAO.php index 597182693..a95ee6bc4 100644 --- a/app/Models/UserDAO.php +++ b/app/Models/UserDAO.php @@ -1,7 +1,7 @@ db; require_once(APP_PATH . '/SQL/install.sql.' . $db['type'] . '.php'); @@ -28,6 +28,22 @@ class FreshRSS_UserDAO extends Minz_ModelPdo { } } } + if ($insertDefaultFeeds) { + if (defined('SQL_INSERT_FEEDS')) { //E.g. MySQL + $sql = sprintf(SQL_INSERT_FEEDS, $bd_prefix_user); + $stm = $userPDO->bd->prepare($sql); + $ok &= $stm && $stm->execute(); + } else { //E.g. SQLite + global $SQL_INSERT_FEEDS; + if (is_array($SQL_INSERT_FEEDS)) { + foreach ($SQL_INSERT_FEEDS as $instruction) { + $sql = sprintf($instruction, $bd_prefix_user); + $stm = $userPDO->bd->prepare($sql); + $ok &= ($stm && $stm->execute()); + } + } + } + } } catch (Exception $e) { Minz_Log::error('Error while creating user: ' . $e->getMessage()); } diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index ca181303e..a454829d5 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -59,6 +59,9 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( ENGINE = INNODB; INSERT IGNORE INTO `%1$scategory` (id, name) VALUES(1, "%2$s"); +'); + +define('SQL_INSERT_FEEDS', ' INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("http://freshrss.org/feeds/all.atom.xml", 1, "FreshRSS.org", "http://freshrss.org/", "FreshRSS, a free, self-hostable aggregator…", 86400); INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS @ GitHub", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400); '); diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index b343bda86..9f4240b98 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -52,6 +52,10 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");', 'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);', +); + +global $SQL_INSERT_FEEDS; +$SQL_INSERT_FEEDS = array( 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'http://freshrss.org/feeds/all.atom.xml\');', 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'https://github.com/FreshRSS/FreshRSS/releases.atom\', 1, \'FreshRSS @ GitHub\', \'https://github.com/FreshRSS/FreshRSS/\', \'FreshRSS releases @ GitHub\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'https://github.com/FreshRSS/FreshRSS/releases.atom\');', ); diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 1d3a5d92f..68d93ba92 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -55,6 +55,10 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `entry`(`lastSeen`);', //v1.1.1 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");', +); + +global $SQL_INSERT_FEEDS; +$SQL_INSERT_FEEDS = array( 'INSERT OR IGNORE INTO `feed` (url, category, name, website, description, ttl) VALUES("http://freshrss.org/feeds/all.atom.xml", 1, "FreshRSS.org", "http://freshrss.org/", "FreshRSS, a free, self-hostable aggregator…", 86400);', 'INSERT OR IGNORE INTO `feed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS releases", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400);', ); diff --git a/app/install.php b/app/install.php index 6956761c7..0daa02b1b 100644 --- a/app/install.php +++ b/app/install.php @@ -357,6 +357,21 @@ function checkDbUser(&$dbOptions) { } } } + + if (defined('SQL_INSERT_FEEDS')) { + $sql = sprintf(SQL_INSERT_FEEDS, $dbOptions['bd_prefix_user']); + $stm = $c->prepare($sql); + $ok &= $stm->execute(); + } else { + global $SQL_INSERT_FEEDS; + if (is_array($SQL_INSERT_FEEDS)) { + foreach ($SQL_INSERT_FEEDS as $instruction) { + $sql = sprintf($instruction, $dbOptions['bd_prefix_user']); + $stm = $c->prepare($sql); + $ok &= $stm->execute(); + } + } + } } catch (PDOException $e) { $ok = false; $dbOptions['bd_error'] = $e->getMessage(); diff --git a/cli/create-user.php b/cli/create-user.php index 5e93d4605..c790acb59 100755 --- a/cli/create-user.php +++ b/cli/create-user.php @@ -9,11 +9,12 @@ $options = getopt('', array( 'language:', 'email:', 'token:', + 'no-default-feeds', )); if (empty($options['user'])) { fail('Usage: ' . basename(__FILE__) . " --user username ( --password 'password' --api-password 'api_password'" . - " --language en --email user@example.net --token 'longRandomString' )"); + " --language en --email user@example.net --token 'longRandomString --no-default-feeds' )"); } $username = $options['user']; if (!ctype_alnum($username)) { @@ -33,7 +34,9 @@ $ok = FreshRSS_user_Controller::createUser($username, array( 'language' => empty($options['language']) ? '' : $options['language'], 'token' => empty($options['token']) ? '' : $options['token'], - )); + ), + !isset($options['no-default-feeds']) + ); if (!$ok) { fail('FreshRSS could not create user!'); -- cgit v1.2.3 From 22b41f3bfcbd5a54d59789c2cebfda6dc23b7dde Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 26 Mar 2017 00:01:11 +0100 Subject: Candidate implementation of defered insertion https://github.com/FreshRSS/FreshRSS/issues/530 --- app/Controllers/feedController.php | 8 +- app/Controllers/importExportController.php | 1 + app/Models/EntryDAO.php | 115 +++++++++++++++++++++-------- app/Models/EntryDAOPGSQL.php | 26 +++++++ app/Models/EntryDAOSQLite.php | 39 +++++++--- app/Models/UserDAO.php | 10 ++- app/SQL/install.sql.mysql.php | 23 ++++++ app/SQL/install.sql.pgsql.php | 24 +++++- app/SQL/install.sql.sqlite.php | 24 ++++++ app/install.php | 36 +++------ p/scripts/main.js | 1 + 11 files changed, 234 insertions(+), 73 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index f71f26a4e..a2d9d5c35 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -226,7 +226,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } } - public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush = null, $isNewFeed = false) { + public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush = null, $isNewFeed = false, $noCommit = false) { @set_time_limit(300); $feedDAO = FreshRSS_Factory::createFeedDao(); @@ -434,6 +434,9 @@ class FreshRSS_feed_Controller extends Minz_ActionController { break; } } + if (!$noCommit) { + $entryDAO->commitNewEntries(); + } return array($updated_feeds, reset($feeds)); } @@ -452,8 +455,9 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $id = Minz_Request::param('id'); $url = Minz_Request::param('url'); $force = Minz_Request::param('force'); + $noCommit = Minz_Session::param('isLastFeed', 1) != 1; - list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force); + list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force, null, false, $noCommit); if (Minz_Request::param('ajax')) { // Most of the time, ajax request is for only one feed. But since diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index 6ae89defb..ededfe506 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -475,6 +475,7 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { } } $this->entryDAO->commit(); + $entryDAO->commitNewEntries(); return !$error; } diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index afcde3d7f..decae9307 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -88,6 +88,38 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return false; } + protected function createEntryTempTable() { + $ok = false; + $hadTransaction = $this->bd->inTransaction(); + if ($hadTransaction) { + $this->bd->commit(); + } + try { + $db = FreshRSS_Context::$system_conf->db; + require_once(APP_PATH . '/SQL/install.sql.' . $db['type'] . '.php'); + Minz_Log::warning('SQL CREATE TABLE entrytmp...'); + if (defined('SQL_CREATE_TABLE_ENTRYTMP')) { + $sql = sprintf(SQL_CREATE_TABLE_ENTRYTMP, $this->prefix); + $stm = $this->bd->prepare($sql); + $ok = $stm && $stm->execute(); + } else { + global $SQL_CREATE_TABLE_ENTRYTMP; + $ok = !empty($SQL_CREATE_TABLE_ENTRYTMP); + foreach ($SQL_CREATE_TABLE_ENTRYTMP as $instruction) { + $sql = sprintf($instruction, $this->prefix); + $stm = $this->bd->prepare($sql); + $ok &= $stm && $stm->execute(); + } + } + } catch (Exception $e) { + Minz_Log::error('FreshRSS_EntryDAO::createEntryTempTable error: ' . $e->getMessage()); + } + if ($hadTransaction) { + $this->bd->beginTransaction(); + } + return $ok; + } + protected function autoUpdateDb($errorInfo) { if (isset($errorInfo[0])) { if ($errorInfo[0] === '42S22') { //ER_BAD_FIELD_ERROR @@ -97,6 +129,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $this->addColumn($column); } } + } elseif ($errorInfo[0] === '42S02' && stripos($errorInfo[2], 'entrytmp') !== false) { //ER_BAD_TABLE_ERROR + return $this->createEntryTempTable(); //v1.7 } } if (isset($errorInfo[1])) { @@ -110,8 +144,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { private $addEntryPrepared = null; public function addEntry($valuesTmp) { - if ($this->addEntryPrepared === null) { - $sql = 'INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, ' + if ($this->addEntryPrepared == null) { + $sql = 'INSERT INTO `' . $this->prefix . 'entrytmp` (id, guid, title, author, ' . ($this->isCompressed() ? 'content_bin' : 'content') . ', link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' . 'VALUES(:id, :guid, :title, :author, ' @@ -121,41 +155,43 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { . ', :is_read, :is_favorite, :id_feed, :tags)'; $this->addEntryPrepared = $this->bd->prepare($sql); } - $this->addEntryPrepared->bindParam(':id', $valuesTmp['id']); - $valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760); - $valuesTmp['guid'] = safe_ascii($valuesTmp['guid']); - $this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']); - $valuesTmp['title'] = substr($valuesTmp['title'], 0, 255); - $this->addEntryPrepared->bindParam(':title', $valuesTmp['title']); - $valuesTmp['author'] = substr($valuesTmp['author'], 0, 255); - $this->addEntryPrepared->bindParam(':author', $valuesTmp['author']); - $this->addEntryPrepared->bindParam(':content', $valuesTmp['content']); - $valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023); - $valuesTmp['link'] = safe_ascii($valuesTmp['link']); - $this->addEntryPrepared->bindParam(':link', $valuesTmp['link']); - $this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT); - $valuesTmp['lastSeen'] = time(); - $this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT); - $valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0; - $this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT); - $valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0; - $this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT); - $this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT); - $valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023); - $this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']); - - if ($this->hasNativeHex()) { - $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']); - } else { - $valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']); //hex2bin() is PHP5.4+ - $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']); + if ($this->addEntryPrepared) { + $this->addEntryPrepared->bindParam(':id', $valuesTmp['id']); + $valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760); + $valuesTmp['guid'] = safe_ascii($valuesTmp['guid']); + $this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']); + $valuesTmp['title'] = substr($valuesTmp['title'], 0, 255); + $this->addEntryPrepared->bindParam(':title', $valuesTmp['title']); + $valuesTmp['author'] = substr($valuesTmp['author'], 0, 255); + $this->addEntryPrepared->bindParam(':author', $valuesTmp['author']); + $this->addEntryPrepared->bindParam(':content', $valuesTmp['content']); + $valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023); + $valuesTmp['link'] = safe_ascii($valuesTmp['link']); + $this->addEntryPrepared->bindParam(':link', $valuesTmp['link']); + $this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT); + $valuesTmp['lastSeen'] = time(); + $this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT); + $valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0; + $this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT); + $valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0; + $this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT); + $this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT); + $valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023); + $this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']); + + if ($this->hasNativeHex()) { + $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']); + } else { + $valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']); //hex2bin() is PHP5.4+ + $this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']); + } } - if ($this->addEntryPrepared && $this->addEntryPrepared->execute()) { return $this->bd->lastInsertId(); } else { $info = $this->addEntryPrepared == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $this->addEntryPrepared->errorInfo(); if ($this->autoUpdateDb($info)) { + $this->addEntryPrepared = null; return $this->addEntry($valuesTmp); } elseif ((int)($info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] @@ -165,6 +201,23 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } + public function commitNewEntries() { + $sql = 'SET @rank=SELECT MAX(id) - COUNT(*) FROM `' . $this->prefix . 'entrytmp`; ' . //MySQL-specific + 'INSERT IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' . + 'SELECT @rank:=@rank+1 AS id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date; ' . + 'DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= @rank;'; + $stm = $this->bd->prepare($sql); + $hadTransaction = $this->bd->inTransaction(); + if (!$hadTransaction) { + $this->bd->beginTransaction(); + } + $result = $stm ? $stm->execute() : false; + if (!$hadTransaction) { + $this->bd->commit(); + } + return $result; + } + private $updateEntryPrepared = null; public function updateEntry($valuesTmp) { diff --git a/app/Models/EntryDAOPGSQL.php b/app/Models/EntryDAOPGSQL.php index b96a62ebc..b25993c47 100644 --- a/app/Models/EntryDAOPGSQL.php +++ b/app/Models/EntryDAOPGSQL.php @@ -11,6 +11,11 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite { } protected function autoUpdateDb($errorInfo) { + if (isset($errorInfo[0])) { + if ($errorInfo[0] === '42P01' && stripos($errorInfo[2], 'entrytmp') !== false) { //undefined_table + return $this->createEntryTempTable(); + } + } return false; } @@ -18,6 +23,27 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite { return false; } + public function commitNewEntries() { + $sql = 'DO $$ +DECLARE +maxrank bigint := (SELECT MAX(id) FROM `' . $this->prefix . 'entrytmp`); +rank bigint := (SELECT maxrank - COUNT(*) FROM `' . $this->prefix . 'entrytmp`); +BEGIN + INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) + (SELECT rank + row_number() OVER(ORDER BY date) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date); + DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= maxrank; +END $$;'; + $hadTransaction = $this->bd->inTransaction(); + if (!$hadTransaction) { + $this->bd->beginTransaction(); + } + $result = $this->bd->exec($sql) !== false; + if (!$hadTransaction) { + $this->bd->commit(); + } + return $result; + } + public function size($all = true) { $db = FreshRSS_Context::$system_conf->db; $sql = 'SELECT pg_size_pretty(pg_database_size(?))'; diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php index 34e854608..ad7bcd865 100644 --- a/app/Models/EntryDAOSQLite.php +++ b/app/Models/EntryDAOSQLite.php @@ -7,21 +7,42 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO { } protected function autoUpdateDb($errorInfo) { - if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR - //autoAddColumn - if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) { - $showCreate = $tableInfo->fetchColumn(); - Minz_Log::debug('FreshRSS_EntryDAOSQLite::autoUpdateDb: ' . $showCreate); - foreach (array('lastSeen', 'hash') as $column) { - if (stripos($showCreate, $column) === false) { - return $this->addColumn($column); - } + Minz_Log::error('FreshRSS_EntryDAO::autoUpdateDb error: ' . print_r($errorInfo, true)); + if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entrytmp'")) { + $showCreate = $tableInfo->fetchColumn(); + if (stripos($showCreate, 'entrytmp') === false) { + return $this->createEntryTempTable(); + } + } + if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) { + $showCreate = $tableInfo->fetchColumn(); + foreach (array('lastSeen', 'hash') as $column) { + if (stripos($showCreate, $column) === false) { + return $this->addColumn($column); } } } return false; } + public function commitNewEntries() { + $sql = ' +CREATE TEMP TABLE `tmp` AS SELECT id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date; +INSERT OR IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) + SELECT rowid + (SELECT MAX(id) - COUNT(*) FROM `tmp`) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `tmp` ORDER BY date; +DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= (SELECT MAX(id) FROM `tmp`); +DROP TABLE `tmp`;'; + $hadTransaction = $this->bd->inTransaction(); + if (!$hadTransaction) { + $this->bd->beginTransaction(); + } + $result = $this->bd->exec($sql) !== false; + if (!$hadTransaction) { + $this->bd->commit(); + } + return $result; + } + protected function sqlConcat($s1, $s2) { return $s1 . '||' . $s2; } diff --git a/app/Models/UserDAO.php b/app/Models/UserDAO.php index a60caf395..310c7c096 100644 --- a/app/Models/UserDAO.php +++ b/app/Models/UserDAO.php @@ -14,21 +14,23 @@ class FreshRSS_UserDAO extends Minz_ModelPdo { $ok = false; $bd_prefix_user = $db['prefix'] . $username . '_'; if (defined('SQL_CREATE_TABLES')) { //E.g. MySQL - $sql = sprintf(SQL_CREATE_TABLES, $bd_prefix_user, _t('gen.short.default_category')); + $sql = sprintf(SQL_CREATE_TABLES . SQL_CREATE_TABLE_ENTRYTMP, $bd_prefix_user, _t('gen.short.default_category')); $stm = $userPDO->bd->prepare($sql); $ok = $stm && $stm->execute(); } else { //E.g. SQLite global $SQL_CREATE_TABLES; + global $SQL_CREATE_TABLE_ENTRYTMP; if (is_array($SQL_CREATE_TABLES)) { - $ok = true; - foreach ($SQL_CREATE_TABLES as $instruction) { + $instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP); + $ok = !empty($instructions); + foreach ($instructions as $instruction) { $sql = sprintf($instruction, $bd_prefix_user, _t('gen.short.default_category')); $stm = $userPDO->bd->prepare($sql); $ok &= ($stm && $stm->execute()); } } } - if ($insertDefaultFeeds) { + if ($ok && $insertDefaultFeeds) { if (defined('SQL_INSERT_FEEDS')) { //E.g. MySQL $sql = sprintf(SQL_INSERT_FEEDS, $bd_prefix_user); $stm = $userPDO->bd->prepare($sql); diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index a454829d5..ceca07f93 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -61,6 +61,29 @@ ENGINE = INNODB; INSERT IGNORE INTO `%1$scategory` (id, name) VALUES(1, "%2$s"); '); +define('SQL_CREATE_TABLE_ENTRYTMP', ' +CREATE TABLE IF NOT EXISTS `%1$sentrytmp` ( -- v1.7 + `id` bigint NOT NULL, + `guid` varchar(760) CHARACTER SET latin1 NOT NULL, + `title` varchar(255) NOT NULL, + `author` varchar(255), + `content_bin` blob, + `link` varchar(1023) CHARACTER SET latin1 NOT NULL, + `date` int(11), + `lastSeen` INT(11) DEFAULT 0, + `hash` BINARY(16), + `is_read` boolean NOT NULL DEFAULT 0, + `is_favorite` boolean NOT NULL DEFAULT 0, + `id_feed` SMALLINT, + `tags` varchar(1023), + PRIMARY KEY (`id`), + FOREIGN KEY (`id_feed`) REFERENCES `%1$sfeed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE, + UNIQUE KEY (`id_feed`,`guid`), + INDEX (`date`), +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci +ENGINE = INNODB; +'); + define('SQL_INSERT_FEEDS', ' INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("http://freshrss.org/feeds/all.atom.xml", 1, "FreshRSS.org", "http://freshrss.org/", "FreshRSS, a free, self-hostable aggregator…", 86400); INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS @ GitHub", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400); diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 9f4240b98..1a666cb8a 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -32,7 +32,7 @@ $SQL_CREATE_TABLES = array( 'CREATE TABLE IF NOT EXISTS "%1$sentry" ( "id" BIGINT NOT NULL PRIMARY KEY, - "guid" VARCHAR(760) UNIQUE NOT NULL, + "guid" VARCHAR(760) NOT NULL, "title" VARCHAR(255) NOT NULL, "author" VARCHAR(255), "content" TEXT, @@ -54,6 +54,28 @@ $SQL_CREATE_TABLES = array( 'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);', ); +global $SQL_CREATE_TABLE_ENTRYTMP; +$SQL_CREATE_TABLE_ENTRYTMP = array( +'CREATE TABLE IF NOT EXISTS "%1$sentrytmp" ( -- v1.7 + "id" BIGINT NOT NULL PRIMARY KEY, + "guid" VARCHAR(760) NOT NULL, + "title" VARCHAR(255) NOT NULL, + "author" VARCHAR(255), + "content" TEXT, + "link" VARCHAR(1023) NOT NULL, + "date" INT, + "lastSeen" INT DEFAULT 0, + "hash" BYTEA, + "is_read" SMALLINT NOT NULL DEFAULT 0, + "is_favorite" SMALLINT NOT NULL DEFAULT 0, + "id_feed" SMALLINT, + "tags" VARCHAR(1023), + FOREIGN KEY ("id_feed") REFERENCES "%1$sfeed" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + UNIQUE ("id_feed","guid") +);', +'CREATE INDEX %1$sentrytmp_date_index ON "%1$sentrytmp" ("date");', +); + global $SQL_INSERT_FEEDS; $SQL_INSERT_FEEDS = array( 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'http://freshrss.org/feeds/all.atom.xml\');', diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 68d93ba92..ad7d525fd 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -57,6 +57,30 @@ $SQL_CREATE_TABLES = array( 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");', ); +global $SQL_CREATE_TABLE_ENTRYTMP; +$SQL_CREATE_TABLE_ENTRYTMP = array( +'CREATE TABLE IF NOT EXISTS `entrytmp` ( -- v1.7 + `id` bigint NOT NULL, + `guid` varchar(760) NOT NULL, + `title` varchar(255) NOT NULL, + `author` varchar(255), + `content` text, + `link` varchar(1023) NOT NULL, + `date` int(11), + `lastSeen` INT(11) DEFAULT 0, + `hash` BINARY(16), + `is_read` boolean NOT NULL DEFAULT 0, + `is_favorite` boolean NOT NULL DEFAULT 0, + `id_feed` SMALLINT, + `tags` varchar(1023), + PRIMARY KEY (`id`), + FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE, + UNIQUE (`id_feed`,`guid`) +);', + +'CREATE INDEX IF NOT EXISTS entrytmp_date_index ON `entrytmp`(`date`);', +); + global $SQL_INSERT_FEEDS; $SQL_INSERT_FEEDS = array( 'INSERT OR IGNORE INTO `feed` (url, category, name, website, description, ttl) VALUES("http://freshrss.org/feeds/all.atom.xml", 1, "FreshRSS.org", "http://freshrss.org/", "FreshRSS, a free, self-hostable aggregator…", 86400);', diff --git a/app/install.php b/app/install.php index 9a88e0f37..17037c384 100644 --- a/app/install.php +++ b/app/install.php @@ -342,35 +342,19 @@ function checkDbUser(&$dbOptions) { $driver_options = $dbOptions['options']; try { $c = new PDO($str, $dbOptions['user'], $dbOptions['password'], $driver_options); - if (defined('SQL_CREATE_TABLES')) { - $sql = sprintf(SQL_CREATE_TABLES, $dbOptions['prefix_user'], _t('gen.short.default_category')); - $stm = $c->prepare($sql); - $ok = $stm->execute(); - } else { - global $SQL_CREATE_TABLES; - if (is_array($SQL_CREATE_TABLES)) { - $ok = true; - foreach ($SQL_CREATE_TABLES as $instruction) { - $sql = sprintf($instruction, $dbOptions['prefix_user'], _t('gen.short.default_category')); - $stm = $c->prepare($sql); - $ok &= $stm->execute(); - } - } - } - - if (defined('SQL_INSERT_FEEDS')) { - $sql = sprintf(SQL_INSERT_FEEDS, $dbOptions['prefix_user']); + $sql = sprintf(SQL_CREATE_TABLES . SQL_CREATE_TABLE_ENTRYTMP . SQL_INSERT_FEEDS, + $dbOptions['prefix_user'], _t('gen.short.default_category')); $stm = $c->prepare($sql); - $ok &= $stm->execute(); + $ok = $stm && $stm->execute(); } else { - global $SQL_INSERT_FEEDS; - if (is_array($SQL_INSERT_FEEDS)) { - foreach ($SQL_INSERT_FEEDS as $instruction) { - $sql = sprintf($instruction, $dbOptions['prefix_user']); - $stm = $c->prepare($sql); - $ok &= $stm->execute(); - } + global $SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_INSERT_FEEDS; + $instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_INSERT_FEEDS); + $ok = !empty($instructions); + foreach ($instructions as $instruction) { + $sql = sprintf($instruction, $dbOptions['prefix_user'], _t('gen.short.default_category')); + $stm = $c->prepare($sql); + $ok &= $stm && $stm->execute(); } } } catch (PDOException $e) { diff --git a/p/scripts/main.js b/p/scripts/main.js index 5dbb95c91..c2f60bf7f 100644 --- a/p/scripts/main.js +++ b/p/scripts/main.js @@ -809,6 +809,7 @@ function updateFeed(feeds, feeds_count) { url: feed.url, data : { _csrf: context.csrf, + isLastFeed: feeds.length <= 0 ? 1 : 0, }, }).always(function (data) { feed_processed++; -- cgit v1.2.3 From e956aee53d561fbdc11a78a50ad7cc041108e5b5 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 26 Mar 2017 14:07:06 +0200 Subject: More defered insertion. New SQL index New index entry_feed_read_index TODO: Auto add this index to existing version --- app/Controllers/feedController.php | 8 +++++- app/Models/EntryDAO.php | 1 - app/Models/Factory.php | 9 +----- app/Models/FeedDAO.php | 56 ++++++++++++++++++-------------------- app/Models/FeedDAOSQLite.php | 19 ------------- app/SQL/install.sql.mysql.php | 1 + app/SQL/install.sql.pgsql.php | 1 + app/SQL/install.sql.sqlite.php | 1 + 8 files changed, 37 insertions(+), 59 deletions(-) delete mode 100644 app/Models/FeedDAOSQLite.php (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index b565d0439..bfc8b2045 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -393,7 +393,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } } - $feedDAO->updateLastUpdate($feed->id(), false, $entryDAO->inTransaction(), $mtime); + $feedDAO->updateLastUpdate($feed->id(), false, $mtime); if ($entryDAO->inTransaction()) { $entryDAO->commit(); } @@ -435,8 +435,14 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } } if (!$noCommit) { + if (!$entryDAO->inTransaction()) { + $entryDAO->beginTransaction(); + } $entryDAO->commitNewEntries(); $feedDAO->updateCachedValues(); //TODO: Optimize + if ($entryDAO->inTransaction()) { + $entryDAO->commit(); + } } return array($updated_feeds, reset($feeds)); } diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 9d11cec6a..39c00f01c 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -121,7 +121,6 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } protected function autoUpdateDb($errorInfo) { - Minz_Log::warning('FreshRSS_EntryDAO::autoUpdateDb: ' . print_r($errorInfo, true)); if (isset($errorInfo[0])) { if ($errorInfo[0] === '42S22') { //ER_BAD_FIELD_ERROR //autoAddColumn diff --git a/app/Models/Factory.php b/app/Models/Factory.php index 6502c38b7..dfccc883e 100644 --- a/app/Models/Factory.php +++ b/app/Models/Factory.php @@ -3,14 +3,7 @@ class FreshRSS_Factory { public static function createFeedDao($username = null) { - $conf = Minz_Configuration::get('system'); - switch ($conf->db['type']) { - case 'sqlite': - case 'pgsql': - return new FreshRSS_FeedDAOSQLite($username); - default: - return new FreshRSS_FeedDAO($username); - } + return new FreshRSS_FeedDAO($username); } public static function createEntryDao($username = null) { diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 0168aebd9..d278122e3 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -92,29 +92,15 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } } - public function updateLastUpdate($id, $inError = false, $updateCache = true, $mtime = 0) { - if ($updateCache) { - $sql = 'UPDATE `' . $this->prefix . 'feed` ' //2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE - . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' - . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0),' - . '`lastUpdate`=?, error=? ' - . 'WHERE id=?'; - } else { - $sql = 'UPDATE `' . $this->prefix . 'feed` ' - . 'SET `lastUpdate`=?, error=? ' - . 'WHERE id=?'; - } - - if ($mtime <= 0) { - $mtime = time(); - } - + public function updateLastUpdate($id, $inError = false, $mtime = 0) { //See also updateCachedValue() + $sql = 'UPDATE `' . $this->prefix . 'feed` ' + . 'SET `lastUpdate`=?, error=? ' + . 'WHERE id=?'; $values = array( - $mtime, + $mtime <= 0 ? time() : $mtime, $inError ? 1 : 0, $id, ); - $stm = $this->bd->prepare($sql); if ($stm && $stm->execute($values)) { @@ -294,18 +280,28 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $res[0]['count']; } - public function updateCachedValues() { //For one single feed, call updateLastUpdate($id) - $sql = 'UPDATE `' . $this->prefix . 'feed` f ' - . 'INNER JOIN (' - . 'SELECT e.id_feed, ' - . 'COUNT(CASE WHEN e.is_read = 0 THEN 1 END) AS nbUnreads, ' - . 'COUNT(e.id) AS nbEntries ' - . 'FROM `' . $this->prefix . 'entry` e ' - . 'GROUP BY e.id_feed' - . ') x ON x.id_feed=f.id ' - . 'SET f.`cache_nbEntries`=x.nbEntries, f.`cache_nbUnreads`=x.nbUnreads'; + public function updateCachedValue($id) { //For multiple feeds, call updateCachedValues() + $sql = 'UPDATE `' . $this->prefix . 'feed` ' //2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE + . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' + . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0) ' + . 'WHERE id=?'; + $values = array($id); $stm = $this->bd->prepare($sql); + if ($stm && $stm->execute($values)) { + return $stm->rowCount(); + } else { + $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); + Minz_Log::error('SQL error updateCachedValue: ' . $info[2]); + return false; + } + } + + public function updateCachedValues() { //For one single feed, call updateCachedValue($id) + $sql = 'UPDATE `' . $this->prefix . 'feed` ' + . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' + . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0)'; + $stm = $this->bd->prepare($sql); if ($stm && $stm->execute()) { return $stm->rowCount(); } else { @@ -343,7 +339,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $affected; } - public function cleanOldEntries($id, $date_min, $keep = 15) { //Remember to call updateLastUpdate($id) or updateCachedValues() just after + public function cleanOldEntries($id, $date_min, $keep = 15) { //Remember to call updateCachedValue($id) or updateCachedValues() just after $sql = 'DELETE FROM `' . $this->prefix . 'entry` ' . 'WHERE id_feed=:id_feed AND id<=:id_max ' . 'AND is_favorite=0 ' //Do not remove favourites diff --git a/app/Models/FeedDAOSQLite.php b/app/Models/FeedDAOSQLite.php deleted file mode 100644 index 440ae74da..000000000 --- a/app/Models/FeedDAOSQLite.php +++ /dev/null @@ -1,19 +0,0 @@ -prefix . 'feed` ' - . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),' - . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0)'; - $stm = $this->bd->prepare($sql); - if ($stm && $stm->execute()) { - return $stm->rowCount(); - } else { - $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); - Minz_Log::error('SQL error updateCachedValues: ' . $info[2]); - return false; - } - } - -} diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index f42e08ad3..3ad68173f 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -55,6 +55,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( INDEX (`is_favorite`), -- v0.7 INDEX (`is_read`), -- v0.7 INDEX `entry_lastSeen_index` (`lastSeen`) -- v1.1.1 + INDEX `entry_feed_read_index` (`id_feed`,`is_read`) -- v1.7 //TODO: Auto add this index to existing version ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ENGINE = INNODB; diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 1a666cb8a..ca4dfa925 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -50,6 +50,7 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX %1$sis_favorite_index ON "%1$sentry" ("is_favorite");', 'CREATE INDEX %1$sis_read_index ON "%1$sentry" ("is_read");', 'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");', +'CREATE INDEX %1$sentry_feed_read_index ON "%1$sentry" ("id_feed","is_read");', //v1.7 //TODO: Auto add this index to existing version 'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);', ); diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index ad7d525fd..dcb7a351a 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -53,6 +53,7 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX IF NOT EXISTS entry_is_favorite_index ON `entry`(`is_favorite`);', 'CREATE INDEX IF NOT EXISTS entry_is_read_index ON `entry`(`is_read`);', 'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `entry`(`lastSeen`);', //v1.1.1 +'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);', //v1.7 //TODO: Auto add this index to existing version 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");', ); -- cgit v1.2.3 From bc580c5905522143cb3d73a37641f7bf21225545 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 26 Mar 2017 22:08:17 +0200 Subject: SQL index auto added At the same time as the new temp table --- app/SQL/install.sql.mysql.php | 4 +++- app/SQL/install.sql.pgsql.php | 3 ++- app/SQL/install.sql.sqlite.php | 7 ++----- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index 3ad68173f..6b07f5965 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -55,7 +55,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( INDEX (`is_favorite`), -- v0.7 INDEX (`is_read`), -- v0.7 INDEX `entry_lastSeen_index` (`lastSeen`) -- v1.1.1 - INDEX `entry_feed_read_index` (`id_feed`,`is_read`) -- v1.7 //TODO: Auto add this index to existing version + -- INDEX `entry_feed_read_index` (`id_feed`,`is_read`) -- v1.7 Located futher down ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ENGINE = INNODB; @@ -83,6 +83,8 @@ CREATE TABLE IF NOT EXISTS `%1$sentrytmp` ( -- v1.7 INDEX (`date`) ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ENGINE = INNODB; + +CREATE INDEX `entry_feed_read_index` ON `%1$sentry`(`id_feed`,`is_read`); -- v1.7 Located here to be auto-added '); define('SQL_INSERT_FEEDS', ' diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index ca4dfa925..74c6d1b79 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -50,7 +50,6 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX %1$sis_favorite_index ON "%1$sentry" ("is_favorite");', 'CREATE INDEX %1$sis_read_index ON "%1$sentry" ("is_read");', 'CREATE INDEX %1$sentry_lastSeen_index ON "%1$sentry" ("lastSeen");', -'CREATE INDEX %1$sentry_feed_read_index ON "%1$sentry" ("id_feed","is_read");', //v1.7 //TODO: Auto add this index to existing version 'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);', ); @@ -75,6 +74,8 @@ $SQL_CREATE_TABLE_ENTRYTMP = array( UNIQUE ("id_feed","guid") );', 'CREATE INDEX %1$sentrytmp_date_index ON "%1$sentrytmp" ("date");', + +'CREATE INDEX %1$sentry_feed_read_index ON "%1$sentry" ("id_feed","is_read");', //v1.7 ); global $SQL_INSERT_FEEDS; diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 1fac5fcf7..bce9f5a95 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -26,7 +26,6 @@ $SQL_CREATE_TABLES = array( FOREIGN KEY (`category`) REFERENCES `category`(`id`) ON DELETE SET NULL ON UPDATE CASCADE, UNIQUE (`url`) );', - 'CREATE INDEX IF NOT EXISTS feed_name_index ON `feed`(`name`);', 'CREATE INDEX IF NOT EXISTS feed_priority_index ON `feed`(`priority`);', 'CREATE INDEX IF NOT EXISTS feed_keep_history_index ON `feed`(`keep_history`);', @@ -49,11 +48,9 @@ $SQL_CREATE_TABLES = array( FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE, UNIQUE (`id_feed`,`guid`) );', - 'CREATE INDEX IF NOT EXISTS entry_is_favorite_index ON `entry`(`is_favorite`);', 'CREATE INDEX IF NOT EXISTS entry_is_read_index ON `entry`(`is_read`);', 'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `entry`(`lastSeen`);', //v1.1.1 -'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);', //v1.7 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");', ); @@ -78,9 +75,9 @@ $SQL_CREATE_TABLE_ENTRYTMP = array( FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE, UNIQUE (`id_feed`,`guid`) );', - 'CREATE INDEX IF NOT EXISTS entrytmp_date_index ON `entrytmp`(`date`);', -'CREATE INDEX IF NOT EXISTS entry_feed_read_index ON `entry`(`id_feed`,`is_read`);', //v1.7 //This line is used for auto-update + +'CREATE INDEX IF NOT EXISTS `entry_feed_read_index` ON `entry`(`id_feed`,`is_read`);', //v1.7 ); global $SQL_INSERT_FEEDS; -- cgit v1.2.3 From 5d54e47ff4065167a07085df07a590372ac5d636 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 10 Apr 2017 00:43:18 +0200 Subject: Fix DROP tables for deferred insertion https://github.com/FreshRSS/FreshRSS/pull/1470 --- app/SQL/install.sql.mysql.php | 2 +- app/SQL/install.sql.pgsql.php | 2 +- app/SQL/install.sql.sqlite.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/SQL/install.sql.pgsql.php') diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index 6b07f5965..09defd452 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -92,7 +92,7 @@ INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) V INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS @ GitHub", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400); '); -define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS `%1$sentry`, `%1$sfeed`, `%1$scategory`'); +define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS `%1$sentrytmp`, `%1$sentry`, `%1$sfeed`, `%1$scategory`'); define('SQL_UPDATE_UTF8MB4', ' ALTER DATABASE `%2$s` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/app/SQL/install.sql.pgsql.php b/app/SQL/install.sql.pgsql.php index 74c6d1b79..4cfeb2517 100644 --- a/app/SQL/install.sql.pgsql.php +++ b/app/SQL/install.sql.pgsql.php @@ -84,4 +84,4 @@ $SQL_INSERT_FEEDS = array( 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'https://github.com/FreshRSS/FreshRSS/releases.atom\', 1, \'FreshRSS @ GitHub\', \'https://github.com/FreshRSS/FreshRSS/\', \'FreshRSS releases @ GitHub\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'https://github.com/FreshRSS/FreshRSS/releases.atom\');', ); -define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS "%1$sentry", "%1$sfeed", "%1$scategory"'); +define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS "%1$sentrytmp", "%1$sentry", "%1$sfeed", "%1$scategory"'); diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index bce9f5a95..c4e4af006 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -86,4 +86,4 @@ $SQL_INSERT_FEEDS = array( 'INSERT OR IGNORE INTO `feed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS releases", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400);', ); -define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS entry, feed, category'); +define('SQL_DROP_TABLES', 'DROP TABLE IF EXISTS `entrytmp`, `entry`, `feed`, `category`'); -- cgit v1.2.3