From ec611d7d543e27fe8c794cd98177d00cf636c58f Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 16 Mar 2015 21:37:28 +0100 Subject: SimplePie: decode special chars for MAYBE_HTML https://github.com/FreshRSS/FreshRSS/issues/754 Needs to check with many feeds to see if this does not introduce incompatibilities with some valid feeds. --- lib/SimplePie/SimplePie/Sanitize.php | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Sanitize.php b/lib/SimplePie/SimplePie/Sanitize.php index 168a5e2e8..3da7239be 100644 --- a/lib/SimplePie/SimplePie/Sanitize.php +++ b/lib/SimplePie/SimplePie/Sanitize.php @@ -249,6 +249,11 @@ class SimplePie_Sanitize { if ($type & SIMPLEPIE_CONSTRUCT_MAYBE_HTML) { + if (preg_match('/&#(x[0-9a-fA-F]+|[0-9]+);/', $data)) //FreshRSS + { + $data = htmlspecialchars_decode($data, ENT_QUOTES); //FreshRSS + //syslog(LOG_DEBUG, 'SimplePie sanitize MAYBE_HTML htmlspecialchars_decode: ' . $data); //FreshRSS + } if (preg_match('/(&(#(x[0-9a-fA-F]+|[0-9]+)|[a-zA-Z0-9]+)|<\/[A-Za-z][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E]*' . SIMPLEPIE_PCRE_HTML_ATTRIBUTE . '>)/', $data)) { $type |= SIMPLEPIE_CONSTRUCT_HTML; -- cgit v1.2.3 From 0e08b5ba56b2117e0f3d9bea6ae725295e8700d5 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 21 Mar 2015 15:13:08 +0100 Subject: htmlspecialchars_decode for all SIMPLEPIE_CONSTRUCT_MAYBE_HTML https://github.com/FreshRSS/FreshRSS/issues/754 --- lib/SimplePie/SimplePie/Sanitize.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Sanitize.php b/lib/SimplePie/SimplePie/Sanitize.php index 3da7239be..e7c9f925f 100644 --- a/lib/SimplePie/SimplePie/Sanitize.php +++ b/lib/SimplePie/SimplePie/Sanitize.php @@ -249,11 +249,7 @@ class SimplePie_Sanitize { if ($type & SIMPLEPIE_CONSTRUCT_MAYBE_HTML) { - if (preg_match('/&#(x[0-9a-fA-F]+|[0-9]+);/', $data)) //FreshRSS - { - $data = htmlspecialchars_decode($data, ENT_QUOTES); //FreshRSS - //syslog(LOG_DEBUG, 'SimplePie sanitize MAYBE_HTML htmlspecialchars_decode: ' . $data); //FreshRSS - } + $data = htmlspecialchars_decode($data, ENT_QUOTES); //FreshRSS if (preg_match('/(&(#(x[0-9a-fA-F]+|[0-9]+)|[a-zA-Z0-9]+)|<\/[A-Za-z][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E]*' . SIMPLEPIE_PCRE_HTML_ATTRIBUTE . '>)/', $data)) { $type |= SIMPLEPIE_CONSTRUCT_HTML; -- cgit v1.2.3 From 1a35e2271d3b9383e882371d37d5fef16abd745d Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 21 Mar 2015 18:20:36 +0100 Subject: SimplePie option to restaure syslog of HTTP requests https://github.com/FreshRSS/FreshRSS/issues/711 --- app/Models/Feed.php | 4 ++-- data/config.default.php | 1 + lib/SimplePie/SimplePie.php | 41 ++++++++++++++++++++++++++++++++++------ lib/SimplePie/SimplePie/File.php | 7 +++++-- lib/lib_rss.php | 1 + 5 files changed, 44 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 5ce03be5d..5f67ea6ce 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -246,10 +246,10 @@ class FreshRSS_Feed extends Minz_Model { } if (($mtime === true) ||($mtime > $this->lastUpdate)) { - Minz_Log::notice('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url); + //Minz_Log::debug('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url); $this->loadEntries($feed); // et on charge les articles du flux } else { - Minz_Log::notice('FreshRSS use cache for ' . $clean_url); + //Minz_Log::debug('FreshRSS use cache for ' . $clean_url); $this->entries = array(); } diff --git a/data/config.default.php b/data/config.default.php index 97df3a299..839bd1687 100644 --- a/data/config.default.php +++ b/data/config.default.php @@ -12,6 +12,7 @@ return array( 'auth_type' => 'none', 'api_enabled' => false, 'unsafe_autologin_enabled' => false, + 'simplepie_syslog_enabled' => true, 'limits' => array( 'cache_duration' => 800, 'timeout' => 10, diff --git a/lib/SimplePie/SimplePie.php b/lib/SimplePie/SimplePie.php index c4872b5be..bb8ce4191 100644 --- a/lib/SimplePie/SimplePie.php +++ b/lib/SimplePie/SimplePie.php @@ -74,6 +74,12 @@ define('SIMPLEPIE_USERAGENT', SIMPLEPIE_NAME . '/' . SIMPLEPIE_VERSION . ' (Feed */ define('SIMPLEPIE_LINKBACK', '' . SIMPLEPIE_NAME . ''); +/** + * Use syslog to report HTTP requests done by SimplePie. + * @see SimplePie::set_syslog() + */ +define('SIMPLEPIE_SYSLOG', true); //FreshRSS + /** * No Autodiscovery * @see SimplePie::set_autodiscovery_level() @@ -622,6 +628,12 @@ class SimplePie */ public $strip_htmltags = array('base', 'blink', 'body', 'doctype', 'embed', 'font', 'form', 'frame', 'frameset', 'html', 'iframe', 'input', 'marquee', 'meta', 'noscript', 'object', 'param', 'script', 'style'); + /** + * Use syslog to report HTTP requests done by SimplePie. + * @see SimplePie::set_syslog() + */ + public $syslog_enabled = SIMPLEPIE_SYSLOG; + /** * The SimplePie class contains feed level data and options * @@ -1136,7 +1148,7 @@ class SimplePie $this->sanitize->strip_attributes($attribs); } - public function add_attributes($attribs = '') + public function add_attributes($attribs = '') //FreshRSS { if ($attribs === '') { @@ -1145,6 +1157,14 @@ class SimplePie $this->sanitize->add_attributes($attribs); } + /** + * Use syslog to report HTTP requests done by SimplePie. + */ + public function set_syslog($value = SIMPLEPIE_SYSLOG) //FreshRSS + { + $this->syslog_enabled = $value == true; + } + /** * Set the output encoding * @@ -1231,7 +1251,8 @@ class SimplePie $this->enable_exceptions = $enable; } - function cleanMd5($rss) { //FreshRSS + function cleanMd5($rss) //FreshRSS + { return md5(preg_replace(array('#<(lastBuildDate|pubDate|updated|feedDate|dc:date|slash:comments)>[^<]+#', '##s'), '', $rss)); } @@ -1329,7 +1350,8 @@ class SimplePie list($headers, $sniffed) = $fetched; - if (isset($this->data['md5'])) { //FreshRSS + if (isset($this->data['md5'])) //FreshRSS + { $md5 = $this->data['md5']; } } @@ -1455,7 +1477,8 @@ class SimplePie { // Load the Cache $this->data = $cache->load(); - if ($cache->mtime() + $this->cache_duration > time()) { //FreshRSS + if ($cache->mtime() + $this->cache_duration > time()) //FreshRSS + { $this->raw_data = false; return true; // If the cache is still valid, just return true } @@ -1529,11 +1552,17 @@ class SimplePie { //FreshRSS $md5 = $this->cleanMd5($file->body); if ($this->data['md5'] === $md5) { - // syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . $this->feed_url); + if ($this->syslog_enabled) + { + syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . $this->feed_url); + } $cache->touch(); return true; //Content unchanged even though server did not send a 304 } else { - // syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . $this->feed_url); + if ($this->syslog_enabled) + { + syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . $this->feed_url); + } $this->data['md5'] = $md5; } } diff --git a/lib/SimplePie/SimplePie/File.php b/lib/SimplePie/SimplePie/File.php index 9625af2a9..56fe72196 100644 --- a/lib/SimplePie/SimplePie/File.php +++ b/lib/SimplePie/SimplePie/File.php @@ -66,7 +66,7 @@ class SimplePie_File var $method = SIMPLEPIE_FILE_SOURCE_NONE; var $permanent_url; //FreshRSS - public function __construct($url, $timeout = 10, $redirects = 5, $headers = null, $useragent = null, $force_fsockopen = false) + public function __construct($url, $timeout = 10, $redirects = 5, $headers = null, $useragent = null, $force_fsockopen = false, $syslog_enabled = SIMPLEPIE_SYSLOG) { if (class_exists('idna_convert')) { @@ -79,7 +79,10 @@ class SimplePie_File $this->useragent = $useragent; if (preg_match('/^http(s)?:\/\//i', $url)) { - // syslog(LOG_INFO, 'SimplePie GET ' . $url); //FreshRSS + if ($syslog_enabled) + { + syslog(LOG_INFO, 'SimplePie GET ' . $url); //FreshRSS + } if ($useragent === null) { $useragent = ini_get('user_agent'); diff --git a/lib/lib_rss.php b/lib/lib_rss.php index e5fe73041..16ae3097f 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -123,6 +123,7 @@ function customSimplePie() { $limits = $system_conf->limits; $simplePie = new SimplePie(); $simplePie->set_useragent(_t('gen.freshrss') . '/' . FRESHRSS_VERSION . ' (' . PHP_OS . '; ' . FRESHRSS_WEBSITE . ') ' . SIMPLEPIE_NAME . '/' . SIMPLEPIE_VERSION); + $simplePie->set_syslog($system_conf->simplepie_syslog_enabled); $simplePie->set_cache_location(CACHE_PATH); $simplePie->set_cache_duration($limits['cache_duration']); $simplePie->set_timeout($limits['timeout']); -- cgit v1.2.3 From ad9fe52f5a76faf58d13fcf7bde8f58e85abe82b Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 22 Mar 2015 22:54:29 +0100 Subject: SimplePie sanitize URLs for syslog https://github.com/FreshRSS/FreshRSS/issues/711 https://github.com/FreshRSS/FreshRSS/pull/715 --- app/Models/Feed.php | 2 +- lib/SimplePie/SimplePie.php | 4 ++-- lib/SimplePie/SimplePie/File.php | 2 +- lib/SimplePie/SimplePie/Misc.php | 10 ++++++++++ lib/lib_rss.php | 12 +----------- 5 files changed, 15 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 5f67ea6ce..15cbb7d0a 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -240,7 +240,7 @@ class FreshRSS_Feed extends Minz_Model { $subscribe_url = $feed->subscribe_url(true); } - $clean_url = url_remove_credentials($subscribe_url); + $clean_url = SimplePie_Misc::url_remove_credentials($subscribe_url); if ($subscribe_url !== null && $subscribe_url !== $url) { $this->_url($clean_url); } diff --git a/lib/SimplePie/SimplePie.php b/lib/SimplePie/SimplePie.php index bb8ce4191..54f4c5770 100644 --- a/lib/SimplePie/SimplePie.php +++ b/lib/SimplePie/SimplePie.php @@ -1554,14 +1554,14 @@ class SimplePie if ($this->data['md5'] === $md5) { if ($this->syslog_enabled) { - syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . $this->feed_url); + syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . SimplePie_Misc::url_remove_credentials($this->feed_url)); } $cache->touch(); return true; //Content unchanged even though server did not send a 304 } else { if ($this->syslog_enabled) { - syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . $this->feed_url); + syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . SimplePie_Misc::url_remove_credentials($this->feed_url)); } $this->data['md5'] = $md5; } diff --git a/lib/SimplePie/SimplePie/File.php b/lib/SimplePie/SimplePie/File.php index 56fe72196..1f9e3d502 100644 --- a/lib/SimplePie/SimplePie/File.php +++ b/lib/SimplePie/SimplePie/File.php @@ -81,7 +81,7 @@ class SimplePie_File { if ($syslog_enabled) { - syslog(LOG_INFO, 'SimplePie GET ' . $url); //FreshRSS + syslog(LOG_INFO, 'SimplePie GET ' . SimplePie_Misc::url_remove_credentials($url)); //FreshRSS } if ($useragent === null) { diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index 5a263a2e5..de50d37b8 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2240,5 +2240,15 @@ function embed_wmedia(width, height, link) { { // No-op } + + /** + * Sanitize a URL by removing HTTP credentials. + * @param $url the URL to sanitize. + * @return the same URL without HTTP credentials. + */ + function url_remove_credentials($url) //FreshRSS + { + return preg_replace('#(?<=//)[^/:@]+:[^/:@]+@#', '', $url); + } } diff --git a/lib/lib_rss.php b/lib/lib_rss.php index 16ae3097f..65a1a8e04 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -181,7 +181,7 @@ function sanitizeHTML($data, $base = '') { function get_content_by_parsing ($url, $path) { require_once (LIB_PATH . '/lib_phpQuery.php'); - Minz_Log::notice('FreshRSS GET ' . url_remove_credentials($url)); + Minz_Log::notice('FreshRSS GET ' . SimplePie_Misc::url_remove_credentials($url)); $html = file_get_contents ($url); if ($html) { @@ -430,13 +430,3 @@ function array_push_unique(&$array, $value) { function array_remove(&$array, $value) { $array = array_diff($array, array($value)); } - - -/** - * Sanitize a URL by removing HTTP credentials. - * @param $url the URL to sanitize. - * @return the same URL without HTTP credentials. - */ -function url_remove_credentials($url) { - return preg_replace('/[^\/]*:[^:]*@/', '', $url); -} -- cgit v1.2.3 From 1c38e646c3223571988b33e6a0d481426e3b6931 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 22 Mar 2015 23:53:59 +0100 Subject: SimplePie seems to only supports HTTP schemes Can simplify the regex (faster because anchored) for cleaning URLs based on the fact that we only have to deal with HTTP or HTTPS --- lib/SimplePie/SimplePie/Misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index de50d37b8..90f01f737 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2248,7 +2248,7 @@ function embed_wmedia(width, height, link) { */ function url_remove_credentials($url) //FreshRSS { - return preg_replace('#(?<=//)[^/:@]+:[^/:@]+@#', '', $url); + return preg_replace('#(?<=^https?://)[^/:@]+:[^/:@]+@#', '', $url); } } -- cgit v1.2.3 From 0ed213d97f566d7f46747d28a318cf58cdb5d699 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 22 Mar 2015 23:56:47 +0100 Subject: Revert "SimplePie seems to only supports HTTP schemes" This reverts commit 1c38e646c3223571988b33e6a0d481426e3b6931. --- lib/SimplePie/SimplePie/Misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index 90f01f737..de50d37b8 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2248,7 +2248,7 @@ function embed_wmedia(width, height, link) { */ function url_remove_credentials($url) //FreshRSS { - return preg_replace('#(?<=^https?://)[^/:@]+:[^/:@]+@#', '', $url); + return preg_replace('#(?<=//)[^/:@]+:[^/:@]+@#', '', $url); } } -- cgit v1.2.3 From 7735471140e1c6087a177192a09c7fb849decd77 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 23 Mar 2015 21:58:37 +0100 Subject: SimplePie faster regex for sanitizing URLs Can simplify the regex (faster because anchored) for cleaning URLs based on the fact that we only have to deal with HTTP or HTTPS https://github.com/FreshRSS/FreshRSS/pull/715 --- lib/SimplePie/SimplePie/Misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index de50d37b8..91549f93a 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2248,7 +2248,7 @@ function embed_wmedia(width, height, link) { */ function url_remove_credentials($url) //FreshRSS { - return preg_replace('#(?<=//)[^/:@]+:[^/:@]+@#', '', $url); + return preg_replace('#^(https?://)[^/:@]+:[^/:@]+@#i', '$1', $url); } } -- cgit v1.2.3 From 2bfc4dbf8b19cd2b892526ef65a966ff3bfef718 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Tue, 24 Mar 2015 21:58:00 +0100 Subject: SimplePie forgot static keyword https://github.com/FreshRSS/FreshRSS/issues/711 --- lib/SimplePie/SimplePie/Misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index 91549f93a..956a284cb 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2246,7 +2246,7 @@ function embed_wmedia(width, height, link) { * @param $url the URL to sanitize. * @return the same URL without HTTP credentials. */ - function url_remove_credentials($url) //FreshRSS + public static function url_remove_credentials($url) //FreshRSS { return preg_replace('#^(https?://)[^/:@]+:[^/:@]+@#i', '$1', $url); } -- cgit v1.2.3 From 711530a512b370d79b079205ce1f8376174f7f03 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 4 Apr 2015 22:39:31 +0200 Subject: SQL: detection of updates, and preparation for better burge https://github.com/FreshRSS/FreshRSS/issues/798 https://github.com/FreshRSS/FreshRSS/issues/493 SQLite not yet tested. Only MySQL tested so far. --- app/Controllers/feedController.php | 98 ++++++++------ app/Controllers/importExportController.php | 3 +- app/Models/Entry.php | 16 +++ app/Models/EntryDAO.php | 198 +++++++++++++++++++++-------- app/Models/Feed.php | 1 + app/SQL/install.sql.mysql.php | 7 +- app/SQL/install.sql.sqlite.php | 7 +- lib/lib_rss.php | 2 +- 8 files changed, 231 insertions(+), 101 deletions(-) (limited to 'lib') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 6f544d834..08a0257a2 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -145,7 +145,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // Call the extension hook $name = $feed->name(); $feed = Minz_ExtensionManager::callHook('feed_before_insert', $feed); - if (is_null($feed)) { + if ($feed === null) { Minz_Request::bad(_t('feed_not_added', $name), $url_redirect); } @@ -181,7 +181,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // Use a shared statement and a transaction to improve a LOT the // performances. - $prepared_statement = $entryDAO->addEntryPrepare(); $feedDAO->beginTransaction(); foreach ($entries as $entry) { // Entries are added without any verification. @@ -190,13 +189,13 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $entry->_isRead($is_read); $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); - if (is_null($entry)) { + if ($entry === null) { // An extension has returned a null value, there is nothing to insert. continue; } $values = $entry->toArray(); - $entryDAO->addEntry($values, $prepared_statement); + $entryDAO->addEntry($values); } $feedDAO->updateLastUpdate($feed->id()); $feedDAO->commit(); @@ -307,7 +306,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $feed->load(false); } catch (FreshRSS_Feed_Exception $e) { Minz_Log::notice($e->getMessage()); - $feedDAO->updateLastUpdate($feed->id(), 1); + $feedDAO->updateLastUpdate($feed->id(), true); $feed->unlock(); continue; } @@ -323,50 +322,69 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // We want chronological order and SimplePie uses reverse order. $entries = array_reverse($feed->entries()); if (count($entries) > 0) { - // For this feed, check last n entry GUIDs already in database. - $existing_guids = array_fill_keys($entryDAO->listLastGuidsByFeed( - $feed->id(), count($entries) + 10 - ), 1); - $use_declared_date = empty($existing_guids); + $newGuids = array(); + foreach ($entries as $entry) { + $newGuids[] = $entry->guid(); + } + // For this feed, check existing GUIDs already in database. + $existingHashForGuids = $entryDAO->listHashForFeedGuids($feed->id(), $newGuids); + unset($newGuids); + $use_declared_date = empty($existingHashForGuids); + $oldGuids = array(); // Add entries in database if possible. - $prepared_statement = $entryDAO->addEntryPrepare(); - $feedDAO->beginTransaction(); foreach ($entries as $entry) { $entry_date = $entry->date(true); - if (isset($existing_guids[$entry->guid()]) || - ($feed_history == 0 && $entry_date < $date_min)) { - // This entry already exists in DB or should not be added - // considering configuration and date. - continue; - } - - $id = uTimeString(); - if ($use_declared_date || $entry_date < $date_min) { - // Use declared date at first import. - $id = min(time(), $entry_date) . uSecString(); + if (isset($existingHashForGuids[$entry->guid()])) { + $existingHash = $existingHashForGuids[$entry->guid()]; + if (strcasecmp($existingHash, $entry->hash()) === 0 || $existingHash === '00000000000000000000000000000000') { + //This entry already exists and is unchanged. TODO: Remove the test with the zero'ed hash in FreshRSS v1.3 + $oldGuids[] = $entry->guid(); + } else { //This entry already exists but has been updated + Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->id() . + ', old hash ' . $existingHash . ', new hash ' . $entry->hash()); + $entry->_isRead($is_read); //Reset is_read + if (!$entryDAO->hasTransaction()) { + $entryDAO->beginTransaction(); + } + $entryDAO->updateEntry($entry->toArray()); + } + } elseif ($feed_history == 0 && $entry_date < $date_min) { + // This entry should not be added considering configuration and date. + $oldGuids[] = $entry->guid(); + } else { + $id = uTimeString(); + if ($use_declared_date || $entry_date < $date_min) { + // Use declared date at first import. + $id = min(time(), $entry_date) . uSecString(); + } + + $entry->_id($id); + $entry->_isRead($is_read); + + $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); + if ($entry === null) { + // An extension has returned a null value, there is nothing to insert. + continue; + } + + if (!$entryDAO->hasTransaction()) { + $entryDAO->beginTransaction(); + } + $entryDAO->addEntry($entry->toArray()); } - - $entry->_id($id); - $entry->_isRead($is_read); - - $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); - if (is_null($entry)) { - // An extension has returned a null value, there is nothing to insert. - continue; - } - - $values = $entry->toArray(); - $entryDAO->addEntry($values, $prepared_statement); } + $entryDAO->updateLastSeen($feed->id(), $oldGuids); } + //TODO: updateLastSeen old GUIDS once in a while, in the case of caching (i.e. the whole feed content has not changed) if ($feed_history >= 0 && rand(0, 30) === 1) { // TODO: move this function in web cron when available (see entry::purge) // Remove old entries once in 30. - if (!$feedDAO->hasTransaction()) { - $feedDAO->beginTransaction(); + if (!$entryDAO->hasTransaction()) { + $entryDAO->beginTransaction(); } + //TODO: more robust system based on entry.lastSeen to avoid cleaning entries that are still published in the RSS feed. $nb = $feedDAO->cleanOldEntries($feed->id(), $date_min, @@ -377,9 +395,9 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } } - $feedDAO->updateLastUpdate($feed->id(), 0, $feedDAO->hasTransaction()); - if ($feedDAO->hasTransaction()) { - $feedDAO->commit(); + $feedDAO->updateLastUpdate($feed->id(), 0, $entryDAO->hasTransaction()); + if ($entryDAO->hasTransaction()) { + $entryDAO->commit(); } if ($feed->url() !== $url) { diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index 589777b2a..26b163e43 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -361,7 +361,6 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { } // Then, articles are imported. - $prepared_statement = $this->entryDAO->addEntryPrepare(); $this->entryDAO->beginTransaction(); foreach ($article_object['items'] as $item) { if (!isset($article_to_feed[$item['id']])) { @@ -396,7 +395,7 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { } $values = $entry->toArray(); - $id = $this->entryDAO->addEntry($values, $prepared_statement); + $id = $this->entryDAO->addEntry($values); if (!$error && ($id === false)) { $error = true; diff --git a/app/Models/Entry.php b/app/Models/Entry.php index 346c98a92..6931c9f25 100644 --- a/app/Models/Entry.php +++ b/app/Models/Entry.php @@ -14,6 +14,7 @@ class FreshRSS_Entry extends Minz_Model { private $content; private $link; private $date; + private $hash = null; private $is_read; private $is_favorite; private $feed; @@ -88,6 +89,14 @@ class FreshRSS_Entry extends Minz_Model { } } + public function hash() { + if ($this->hash === null) { + //Do not include $this->date because it may be automatically generated when lacking + $this->hash = md5($this->link . $this->title . $this->author . $this->content . $this->tags(true)); + } + return $this->hash; + } + public function _id($value) { $this->id = $value; } @@ -95,18 +104,23 @@ class FreshRSS_Entry extends Minz_Model { $this->guid = $value; } public function _title($value) { + $this->hash = null; $this->title = $value; } public function _author($value) { + $this->hash = null; $this->author = $value; } public function _content($value) { + $this->hash = null; $this->content = $value; } public function _link($value) { + $this->hash = null; $this->link = $value; } public function _date($value) { + $this->hash = null; $value = intval($value); $this->date = $value > 1 ? $value : time(); } @@ -120,6 +134,7 @@ class FreshRSS_Entry extends Minz_Model { $this->feed = $value; } public function _tags($value) { + $this->hash = null; if (!is_array($value)) { $value = array($value); } @@ -182,6 +197,7 @@ class FreshRSS_Entry extends Minz_Model { 'content' => $this->content(), 'link' => $this->link(), 'date' => $this->date(true), + 'hash' => $this->hash(), 'is_read' => $this->isRead(), 'is_favorite' => $this->isFavorite(), 'id_feed' => $this->feed(), diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 9736d5cd3..5b4b85547 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -6,20 +6,57 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return parent::$sharedDbType !== 'sqlite'; } - public function addEntryPrepare() { - $sql = 'INSERT INTO `' . $this->prefix . 'entry`(id, guid, title, author, ' - . ($this->isCompressed() ? 'content_bin' : 'content') - . ', link, date, is_read, is_favorite, id_feed, tags) ' - . 'VALUES(?, ?, ?, ?, ' - . ($this->isCompressed() ? 'COMPRESS(?)' : '?') - . ', ?, ?, ?, ?, ?, ?)'; - return $this->bd->prepare($sql); + protected function autoAddColumn($errorInfo) { + if (isset($errorInfo[0])) { + if ($errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR + $hasTransaction = false; + try { + $stm = null; + if (stripos($errorInfo[2], 'lastSeen') !== false) { //v1.2 + if (!$this->bd->inTransaction()) { + $this->bd->beginTransaction(); + $hasTransaction = true; + } + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN lastSeen INT(11) NOT NULL'); + if ($stm && $stm->execute()) { + $stm = $this->bd->prepare('CREATE INDEX entry_lastSeen_index ON `' . $this->prefix . 'entry`(`lastSeen`);'); //"IF NOT EXISTS" does not exist in MySQL 5.7 + if ($stm && $stm->execute()) { + if ($hasTransaction) { + $this->bd->commit(); + } + return true; + } + } + if ($hasTransaction) { + $this->bd->rollBack(); + } + } elseif (stripos($errorInfo[2], 'hash') !== false) { //v1.2 + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN hash BINARY(16) NOT NULL'); + return $stm && $stm->execute(); + } + } catch (Exception $e) { + Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn error: ' . $e->getMessage()); + if ($hasTransaction) { + $this->bd->rollBack(); + } + } + } + } + return false; } - public function addEntry($valuesTmp, $preparedStatement = null) { - $stm = $preparedStatement === null ? - FreshRSS_EntryDAO::addEntryPrepare() : - $preparedStatement; + private $addEntryPrepared = null; + + public function addEntry($valuesTmp) { + if ($this->addEntryPrepared === null) { + $sql = 'INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, ' + . ($this->isCompressed() ? 'content_bin' : 'content') + . ', link, date, lastSeen, hash, is_read, is_favorite, id_feed, tags) ' + . 'VALUES(?, ?, ?, ?, ' + . ($this->isCompressed() ? 'COMPRESS(?)' : '?') + . ', ?, ?, ?, X?, ?, ?, ?, ?)'; + $this->addEntryPrepared = $this->bd->prepare($sql); + } $values = array( $valuesTmp['id'], @@ -29,55 +66,65 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $valuesTmp['content'], substr($valuesTmp['link'], 0, 1023), $valuesTmp['date'], + time(), + $valuesTmp['hash'], $valuesTmp['is_read'] ? 1 : 0, $valuesTmp['is_favorite'] ? 1 : 0, $valuesTmp['id_feed'], substr($valuesTmp['tags'], 0, 1023), ); - if ($stm && $stm->execute($values)) { + if ($this->addEntryPrepared && $this->addEntryPrepared->execute($values)) { return $this->bd->lastInsertId(); } else { - $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); - if ((int)($info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries + $info = $this->addEntryPrepared == null ? array(2 => 'syntax error') : $this->addEntryPrepared->errorInfo(); + if ($this->autoAddColumn($info)) { + return $this->addEntry($valuesTmp); + } elseif ((int)($info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']); - } /*else { - Minz_Log::debug('SQL error ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] - . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']); - }*/ + } return false; } } - public function addEntryObject($entry, $conf, $feedHistory) { - $existingGuids = array_fill_keys( - $this->listLastGuidsByFeed($entry->feed(), 20), 1 - ); - - $nb_month_old = max($conf->old_entries, 1); - $date_min = time() - (3600 * 24 * 30 * $nb_month_old); + private $updateEntryPrepared = null; - $eDate = $entry->date(true); - - if ($feedHistory == -2) { - $feedHistory = $conf->keep_history_default; + public function updateEntry($valuesTmp) { + if ($this->updateEntryPrepared === null) { + $sql = 'UPDATE `' . $this->prefix . 'entry` ' + . 'SET title=?, author=?, ' + . ($this->isCompressed() ? 'content_bin=COMPRESS(?)' : 'content=?') + . ', link=?, date=?, lastSeen=?, hash=X?, is_read=?, tags=? ' + . 'WHERE id_feed=? AND guid=?'; + $this->updateEntryPrepared = $this->bd->prepare($sql); } - if (!isset($existingGuids[$entry->guid()]) && - ($feedHistory != 0 || $eDate >= $date_min || $entry->isFavorite())) { - $values = $entry->toArray(); - - $useDeclaredDate = empty($existingGuids); - $values['id'] = ($useDeclaredDate || $eDate < $date_min) ? - min(time(), $eDate) . uSecString() : - uTimeString(); + $values = array( + substr($valuesTmp['title'], 0, 255), + substr($valuesTmp['author'], 0, 255), + $valuesTmp['content'], + substr($valuesTmp['link'], 0, 1023), + $valuesTmp['date'], + time(), + $valuesTmp['hash'], + $valuesTmp['is_read'] ? 1 : 0, + substr($valuesTmp['tags'], 0, 1023), + $valuesTmp['id_feed'], + substr($valuesTmp['guid'], 0, 760), + ); - return $this->addEntry($values); + if ($this->updateEntryPrepared && $this->updateEntryPrepared->execute($values)) { + return $this->bd->lastInsertId(); + } else { + $info = $this->updateEntryPrepared == null ? array(2 => 'syntax error') : $this->updateEntryPrepared->errorInfo(); + if ($this->autoAddColumn($info)) { + return $this->updateEntry($valuesTmp); + } + Minz_Log::error('SQL error updateEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] + . ' while updating entry with GUID ' . $valuesTmp['guid'] . ' in feed ' . $valuesTmp['id_feed']); + return false; } - - // We don't return Entry object to avoid a research in DB - return -1; } /** @@ -94,6 +141,9 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { if (!is_array($ids)) { $ids = array($ids); } + if (count($ids) < 1) { + return 0; + } $sql = 'UPDATE `' . $this->prefix . 'entry` ' . 'SET is_favorite=? ' . 'WHERE id IN (' . str_repeat('?,', count($ids) - 1). '?)'; @@ -296,11 +346,11 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { * * If $idMax equals 0, a deprecated debug message is logged * - * @param integer $id feed ID + * @param integer $id_feed feed ID * @param integer $idMax fail safe article ID * @return integer affected rows */ - public function markReadFeed($id, $idMax = 0) { + public function markReadFeed($id_feed, $idMax = 0) { if ($idMax == 0) { $idMax = time() . '000000'; Minz_Log::debug('Calling markReadFeed(0) is deprecated!'); @@ -310,7 +360,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $sql = 'UPDATE `' . $this->prefix . 'entry` ' . 'SET is_read=1 ' . 'WHERE id_feed=? AND is_read=0 AND id <= ?'; - $values = array($id, $idMax); + $values = array($id_feed, $idMax); $stm = $this->bd->prepare($sql); if (!($stm && $stm->execute($values))) { $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); @@ -324,7 +374,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $sql = 'UPDATE `' . $this->prefix . 'feed` ' . 'SET cache_nbUnreads=cache_nbUnreads-' . $affected . ' WHERE id=?'; - $values = array($id); + $values = array($id_feed); $stm = $this->bd->prepare($sql); if (!($stm && $stm->execute($values))) { $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); @@ -338,7 +388,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $affected; } - public function searchByGuid($feed_id, $id) { + public function searchByGuid($id_feed, $guid) { // un guid est unique pour un flux donné $sql = 'SELECT id, guid, title, author, ' . ($this->isCompressed() ? 'UNCOMPRESS(content_bin) AS content' : 'content') @@ -347,8 +397,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $stm = $this->bd->prepare($sql); $values = array( - $feed_id, - $id + $id_feed, + $guid, ); $stm->execute($values); @@ -519,12 +569,52 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $stm->fetchAll(PDO::FETCH_COLUMN, 0); } - public function listLastGuidsByFeed($id, $n) { - $sql = 'SELECT guid FROM `' . $this->prefix . 'entry` WHERE id_feed=? ORDER BY id DESC LIMIT ' . intval($n); + public function listHashForFeedGuids($id_feed, $guids) { + if (count($guids) < 1) { + return array(); + } + $sql = 'SELECT guid, hex(hash) AS hexHash FROM `' . $this->prefix . 'entry` WHERE id_feed=? AND guid IN (' . str_repeat('?,', count($guids) - 1). '?)'; $stm = $this->bd->prepare($sql); - $values = array($id); - $stm->execute($values); - return $stm->fetchAll(PDO::FETCH_COLUMN, 0); + $values = array($id_feed); + $values = array_merge($values, $guids); + if ($stm && $stm->execute($values)) { + $result = array(); + $rows = $stm->fetchAll(PDO::FETCH_ASSOC); + foreach ($rows as $row) { + $result[$row['guid']] = $row['hexHash']; + } + return $result; + } else { + + $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); + if ($this->autoAddColumn($info)) { + return $this->listHashForFeedGuids($id_feed, $guids); + } + Minz_Log::error('SQL error listHashForFeedGuids: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] + . ' while querying feed ' . $id_feed); + return false; + } + } + + public function updateLastSeen($id_feed, $guids) { + if (count($guids) < 1) { + return 0; + } + $sql = 'UPDATE `' . $this->prefix . 'entry` SET lastSeen=? WHERE id_feed=? AND guid IN (' . str_repeat('?,', count($guids) - 1). '?)'; + $stm = $this->bd->prepare($sql); + $values = array(time(), $id_feed); + $values = array_merge($values, $guids); + if ($stm && $stm->execute($values)) { + return $stm->rowCount(); + } else { + $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); + if ($this->autoAddColumn($info)) { + return $this->updateLastSeen($id_feed, $guids); + } + Minz_Log::error('SQL error updateLastSeen: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] + . ' while updating feed ' . $id_feed); + return false; + } } public function countUnreadRead() { diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 5ce03be5d..27c83ffd5 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -255,6 +255,7 @@ class FreshRSS_Feed extends Minz_Model { $feed->__destruct(); //http://simplepie.org/wiki/faq/i_m_getting_memory_leaks unset($feed); + //TODO: Return a different information in case of cache/no-cache, and give access to the GUIDs in case of cache } } } diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index cf0159199..afdd821b2 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS `%1$sfeed` ( `name` varchar(255) NOT NULL, `website` varchar(255) CHARACTER SET latin1, `description` text, - `lastUpdate` int(11) DEFAULT 0, + `lastUpdate` int(11) DEFAULT 0, -- Until year 2038 `priority` tinyint(2) NOT NULL DEFAULT 10, `pathEntries` varchar(511) DEFAULT NULL, `httpAuth` varchar(511) DEFAULT NULL, @@ -40,7 +40,9 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( `author` varchar(255), `content_bin` blob, -- v0.7 `link` varchar(1023) CHARACTER SET latin1 NOT NULL, - `date` int(11), + `date` int(11), -- Until year 2038 + `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, `id_feed` SMALLINT, -- v0.7 @@ -50,6 +52,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( UNIQUE KEY (`id_feed`,`guid`), -- v0.7 INDEX (`is_favorite`), -- v0.7 INDEX (`is_read`) -- v0.7 + INDEX entry_lastSeen_index (`lastSeen`) -- v1.2 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci ENGINE = INNODB; diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 30bca2810..7517ead45 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -14,7 +14,7 @@ $SQL_CREATE_TABLES = array( `name` varchar(255) NOT NULL, `website` varchar(255), `description` text, - `lastUpdate` int(11) DEFAULT 0, + `lastUpdate` int(11) DEFAULT 0, -- Until year 2038 `priority` tinyint(2) NOT NULL DEFAULT 10, `pathEntries` varchar(511) DEFAULT NULL, `httpAuth` varchar(511) DEFAULT NULL, @@ -38,7 +38,9 @@ $SQL_CREATE_TABLES = array( `author` varchar(255), `content` text, `link` varchar(1023) NOT NULL, - `date` int(11), + `date` int(11), -- Until year 2038 + `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, `id_feed` SMALLINT, @@ -50,6 +52,7 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX IF NOT EXISTS entry_is_favorite_index ON `%1$sentry`(`is_favorite`);', 'CREATE INDEX IF NOT EXISTS entry_is_read_index ON `%1$sentry`(`is_read`);', +'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `%1$sentry`(`lastSeen`);', //v1.2 'INSERT OR IGNORE INTO `%1$scategory` (id, name) VALUES(1, "%2$s");', ); diff --git a/lib/lib_rss.php b/lib/lib_rss.php index e5fe73041..c6bdfde0e 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -38,7 +38,7 @@ function classAutoloader($class) { include(APP_PATH . '/Models/' . $components[1] . '.php'); return; case 3: //Controllers, Exceptions - @include(APP_PATH . '/' . $components[2] . 's/' . $components[1] . $components[2] . '.php'); + include(APP_PATH . '/' . $components[2] . 's/' . $components[1] . $components[2] . '.php'); return; } } elseif (strpos($class, 'Minz') === 0) { -- cgit v1.2.3 From 2d18910d02d92098257b96766e5b89a780daab0b Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 5 Apr 2015 12:41:16 +0200 Subject: Support for Internationalized Domain Names (IDN) https://github.com/FreshRSS/FreshRSS/issues/819 Add explicit conversion from IDN to Punycode. Requires PHP 5.3 IDN extension http://php.net/intl.idn (php5-idn package on Debian/Ubuntu). For systems without PHP 5.3+ IDN extension, we may consider adding a dependency (322 kB) to the third-party library https://phlymail.com/en/downloads/idna-convert.html See PHP bug 53474 FILTER_VALIDATE_URL should not fail URL's that use IDNhttps://bugs.php.net/bug.php?id=53474 --- README.fr.md | 2 +- README.md | 2 +- lib/lib_rss.php | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/README.fr.md b/README.fr.md index 380d7bc1e..ea5b07a9b 100644 --- a/README.fr.md +++ b/README.fr.md @@ -32,7 +32,7 @@ Privilégiez pour cela des demandes sur GitHub * Fonctionne même sur un Raspberry Pi avec des temps de réponse < 1s (testé sur 150 flux, 22k articles, soit 32Mo de données partiellement compressées) * Serveur Web Apache2 (recommandé), ou nginx, lighttpd (non testé sur les autres) * PHP 5.2.1+ (PHP 5.3.7+ recommandé) - * Requis : [PDO_MySQL](http://php.net/pdo-mysql) ou [PDO_SQLite](http://php.net/pdo-sqlite), [cURL](http://php.net/curl), [GMP](http://php.net/gmp) (seulement pour accès API sur platformes < 64 bits) + * Requis : [PDO_MySQL](http://php.net/pdo-mysql) ou [PDO_SQLite](http://php.net/pdo-sqlite), [cURL](http://php.net/curl), [GMP](http://php.net/gmp) (pour accès API sur platformes < 64 bits), [IDN](http://php.net/intl.idn) (pour les noms de domaines internationalisés) * Recommandés : [JSON](http://php.net/json), [mbstring](http://php.net/mbstring), [zlib](http://php.net/zlib), [Zip](http://php.net/zip) * MySQL 5.0.3+ (recommandé) ou SQLite 3.7.4+ * Un navigateur Web récent tel Firefox 4+, Chrome, Opera, Safari, Internet Explorer 9+ diff --git a/README.md b/README.md index a09a64639..92072b07d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The best way is to open issues on GitHub * It even works on Raspberry Pi with response time under a second (tested with 150 feeds, 22k articles, or 32Mo of compressed data) * A web server: Apache2 (recommanded), nginx, lighttpd (not tested on others) * PHP 5.2.1+ (PHP 5.3.7+ recommanded) - * Required extensions: [PDO_MySQL](http://php.net/pdo-mysql) or [PDO_SQLite](http://php.net/pdo-sqlite), [cURL](http://php.net/curl), [GMP](http://php.net/gmp) (only for API access on platforms under 64 bits) + * Required extensions: [PDO_MySQL](http://php.net/pdo-mysql) or [PDO_SQLite](http://php.net/pdo-sqlite), [cURL](http://php.net/curl), [GMP](http://php.net/gmp) (for API access on platforms < 64 bits), [IDN](http://php.net/intl.idn) (for Internationalized Domain Names) * Recommanded extensions : [JSON](http://php.net/json), [mbstring](http://php.net/mbstring), [zlib](http://php.net/zlib), [Zip](http://php.net/zip) * MySQL 5.0.3+ (recommanded) or SQLite 3.7.4+ * A recent browser like Firefox 4+, Chrome, Opera, Safari, Internet Explorer 9+ diff --git a/lib/lib_rss.php b/lib/lib_rss.php index e5fe73041..bc5d6fc5b 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -51,6 +51,21 @@ function classAutoloader($class) { spl_autoload_register('classAutoloader'); // +function idn_to_punny($url) { + if (function_exists('idn_to_ascii')) { + $parts = parse_url($url); + if (!empty($parts['host'])) { + $idn = $parts['host']; + $punny = idn_to_ascii($idn); + $pos = strpos($url, $idn); + if ($pos !== false) { + return substr_replace($url, $punny, $pos, strlen($idn)); + } + } + } + return $url; +} + function checkUrl($url) { if (empty ($url)) { return ''; @@ -58,6 +73,7 @@ function checkUrl($url) { if (!preg_match ('#^https?://#i', $url)) { $url = 'http://' . $url; } + $url = idn_to_punny($url); //PHP bug #53474 IDN if (filter_var($url, FILTER_VALIDATE_URL) || (version_compare(PHP_VERSION, '5.3.3', '<') && (strpos($url, '-') > 0) && //PHP bug #51192 ($url === filter_var($url, FILTER_SANITIZE_URL)))) { -- cgit v1.2.3 From 06b76831dece01f836c9d0a3cc32c3f59910fe60 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 5 Apr 2015 13:07:34 +0200 Subject: Punycode spelling mistake https://github.com/FreshRSS/FreshRSS/pull/820 --- data/subscriptions/.gitignore | 1 + lib/lib_rss.php | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 data/subscriptions/.gitignore (limited to 'lib') diff --git a/data/subscriptions/.gitignore b/data/subscriptions/.gitignore new file mode 100644 index 000000000..150f68c80 --- /dev/null +++ b/data/subscriptions/.gitignore @@ -0,0 +1 @@ +*/* diff --git a/lib/lib_rss.php b/lib/lib_rss.php index bc5d6fc5b..c4f6a6011 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -51,15 +51,15 @@ function classAutoloader($class) { spl_autoload_register('classAutoloader'); // -function idn_to_punny($url) { +function idn_to_puny($url) { if (function_exists('idn_to_ascii')) { $parts = parse_url($url); if (!empty($parts['host'])) { $idn = $parts['host']; - $punny = idn_to_ascii($idn); + $puny = idn_to_ascii($idn); $pos = strpos($url, $idn); if ($pos !== false) { - return substr_replace($url, $punny, $pos, strlen($idn)); + return substr_replace($url, $puny, $pos, strlen($idn)); } } } @@ -73,7 +73,7 @@ function checkUrl($url) { if (!preg_match ('#^https?://#i', $url)) { $url = 'http://' . $url; } - $url = idn_to_punny($url); //PHP bug #53474 IDN + $url = idn_to_puny($url); //PHP bug #53474 IDN if (filter_var($url, FILTER_VALIDATE_URL) || (version_compare(PHP_VERSION, '5.3.3', '<') && (strpos($url, '-') > 0) && //PHP bug #51192 ($url === filter_var($url, FILTER_SANITIZE_URL)))) { -- cgit v1.2.3 From 217c191a1ba3ac03b847d261a32e19975380fcad Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 11 May 2015 22:42:41 +0200 Subject: More SQLite compatibility Additional changes to add compatibility with SQLite for the new hash/lastSeen mode of updating articles. --- app/Models/EntryDAO.php | 71 ++++++++++++++++++++++++------------------ app/Models/EntryDAOSQLite.php | 15 +++++++++ app/Models/FeedDAO.php | 11 ++++--- app/SQL/install.sql.mysql.php | 2 +- app/SQL/install.sql.sqlite.php | 2 +- app/install.php | 2 ++ lib/Minz/ModelPdo.php | 5 +++ 7 files changed, 70 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 172eac897..eae9683ad 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -6,38 +6,48 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return parent::$sharedDbType !== 'sqlite'; } - protected function autoAddColumn($errorInfo) { - if (isset($errorInfo[0])) { - if ($errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR - $hasTransaction = false; - try { - $stm = null; - if (stripos($errorInfo[2], 'lastSeen') !== false) { //v1.2 - if (!$this->bd->inTransaction()) { - $this->bd->beginTransaction(); - $hasTransaction = true; - } - $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN lastSeen INT(11) NOT NULL'); - if ($stm && $stm->execute()) { - $stm = $this->bd->prepare('CREATE INDEX entry_lastSeen_index ON `' . $this->prefix . 'entry`(`lastSeen`);'); //"IF NOT EXISTS" does not exist in MySQL 5.7 - if ($stm && $stm->execute()) { - if ($hasTransaction) { - $this->bd->commit(); - } - return true; - } - } + protected function addColumn($name) { + Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn: ' . $name); + $hasTransaction = false; + try { + $stm = null; + if ($name === 'lastSeen') { //v1.2 + if (!$this->bd->inTransaction()) { + $this->bd->beginTransaction(); + $hasTransaction = true; + } + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN lastSeen INT(11) DEFAULT 0'); + if ($stm && $stm->execute()) { + $stm = $this->bd->prepare('CREATE INDEX entry_lastSeen_index ON `' . $this->prefix . 'entry`(`lastSeen`);'); //"IF NOT EXISTS" does not exist in MySQL 5.7 + if ($stm && $stm->execute()) { if ($hasTransaction) { - $this->bd->rollBack(); + $this->bd->commit(); } - } elseif (stripos($errorInfo[2], 'hash') !== false) { //v1.2 - $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN hash BINARY(16) NOT NULL'); - return $stm && $stm->execute(); + return true; } - } catch (Exception $e) { - Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn error: ' . $e->getMessage()); - if ($hasTransaction) { - $this->bd->rollBack(); + } + if ($hasTransaction) { + $this->bd->rollBack(); + } + } elseif ($name === 'hash') { //v1.2 + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN hash BINARY(16)'); + return $stm && $stm->execute(); + } + } catch (Exception $e) { + Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn error: ' . $e->getMessage()); + if ($hasTransaction) { + $this->bd->rollBack(); + } + } + return false; + } + + protected function autoAddColumn($errorInfo) { + if (isset($errorInfo[0])) { + if ($errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR + foreach (array('lastSeen', 'hash') as $column) { + if (stripos($errorInfo[2], $column) !== false) { + return $this->addColumn($column); } } } @@ -82,7 +92,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $this->addEntry($valuesTmp); } elseif ((int)($info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] - . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']); + . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']. ' ' . $this->addEntryPrepared); } return false; } @@ -597,7 +607,6 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } return $result; } else { - $info = $stm == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $stm->errorInfo(); if ($this->autoAddColumn($info)) { return $this->listHashForFeedGuids($id_feed, $guids); diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php index ffe0f037c..ff049d813 100644 --- a/app/Models/EntryDAOSQLite.php +++ b/app/Models/EntryDAOSQLite.php @@ -2,6 +2,21 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO { + protected function autoAddColumn($errorInfo) { + if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR + if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) { + $showCreate = $tableInfo->fetchColumn(); + Minz_Log::debug('FreshRSS_EntryDAOSQLite::autoAddColumn: ' . $showCreate); + foreach (array('lastSeen', 'hash') as $column) { + if (stripos($showCreate, $column) === false) { + return $this->addColumn($column); + } + } + } + } + return false; + } + protected function sqlConcat($s1, $s2) { return $s1 . '||' . $s2; } diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 76025ff53..475d39286 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -330,11 +330,12 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { . 'AND id NOT IN (SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=:id_feed ORDER BY id DESC LIMIT :keep) keep)'; //Double select: MySQL doesn't support 'LIMIT & IN/ALL/ANY/SOME subquery' $stm = $this->bd->prepare($sql); - $id_max = intval($date_min) . '000000'; - - $stm->bindParam(':id_feed', $id, PDO::PARAM_INT); - $stm->bindParam(':id_max', $id_max, PDO::PARAM_STR); - $stm->bindParam(':keep', $keep, PDO::PARAM_INT); + if ($stm) { + $id_max = intval($date_min) . '000000'; + $stm->bindParam(':id_feed', $id, PDO::PARAM_INT); + $stm->bindParam(':id_max', $id_max, PDO::PARAM_STR); + $stm->bindParam(':keep', $keep, PDO::PARAM_INT); + } if ($stm && $stm->execute()) { return $stm->rowCount(); diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index afdd821b2..9c6af405d 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( `content_bin` blob, -- v0.7 `link` varchar(1023) CHARACTER SET latin1 NOT NULL, `date` int(11), -- Until year 2038 - `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `lastSeen` INT(11) DEFAULT 0, -- v1.2, Until year 2038 `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 7517ead45..77e8e094c 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -39,7 +39,7 @@ $SQL_CREATE_TABLES = array( `content` text, `link` varchar(1023) NOT NULL, `date` int(11), -- Until year 2038 - `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `lastSeen` INT(11) DEFAULT 0, -- v1.2, Until year 2038 `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, diff --git a/app/install.php b/app/install.php index 177173fdb..86afb9318 100644 --- a/app/install.php +++ b/app/install.php @@ -168,8 +168,10 @@ function saveStep3() { $_SESSION['bd_prefix_user'] = $_SESSION['bd_prefix'] .(empty($_SESSION['default_user']) ? '' :($_SESSION['default_user'] . '_')); } + //TODO: load `config.default.php` as default $config_array = array( 'environment' => 'production', + 'simplepie_syslog_enabled' => true, 'salt' => $_SESSION['salt'], 'title' => $_SESSION['title'], 'default_user' => $_SESSION['default_user'], diff --git a/lib/Minz/ModelPdo.php b/lib/Minz/ModelPdo.php index ac7a1bed7..3e8ec1f43 100644 --- a/lib/Minz/ModelPdo.php +++ b/lib/Minz/ModelPdo.php @@ -134,4 +134,9 @@ class MinzPDO extends PDO { MinzPDO::check($statement); return parent::exec($statement); } + + public function query($statement) { + MinzPDO::check($statement); + return parent::query($statement); + } } -- cgit v1.2.3 From 27d2b88a19345dfc665dc086d3c2b2e4547e1b7f Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 23 May 2015 02:23:38 +0200 Subject: Minz getBaseUrl correction and RSS template bug https://github.com/FreshRSS/FreshRSS/issues/848 Corrections in Minz (HTTP_HOST was not sanitized, getURI() was never used and not working anyway with absolute base_url) $this->url was not defined in rss.phtml --- app/Controllers/indexController.php | 1 + constants.php | 3 ++- lib/Minz/Request.php | 46 +++++++++++-------------------------- lib/Minz/Url.php | 10 +------- 4 files changed, 18 insertions(+), 42 deletions(-) (limited to 'lib') diff --git a/app/Controllers/indexController.php b/app/Controllers/indexController.php index c1aaca53f..baaf99065 100755 --- a/app/Controllers/indexController.php +++ b/app/Controllers/indexController.php @@ -137,6 +137,7 @@ class FreshRSS_index_Controller extends Minz_ActionController { } // No layout for RSS output. + $this->view->url = empty($_SERVER['QUERY_STRING']) ? '' : '?' . $_SERVER['QUERY_STRING']; $this->view->rss_title = FreshRSS_Context::$name . ' | ' . Minz_View::title(); $this->view->_useLayout(false); header('Content-Type: application/rss+xml; charset=utf-8'); diff --git a/constants.php b/constants.php index b20bf0710..d32fdfa9b 100644 --- a/constants.php +++ b/constants.php @@ -11,7 +11,8 @@ define('PHP_COMPRESSION', false); define('FRESHRSS_PATH', dirname(__FILE__)); define('PUBLIC_PATH', FRESHRSS_PATH . '/p'); - define('INDEX_PATH', PUBLIC_PATH . '/i'); + define('PUBLIC_TO_INDEX_PATH', '/i'); + define('INDEX_PATH', PUBLIC_PATH . PUBLIC_TO_INDEX_PATH); define('PUBLIC_RELATIVE', '..'); define('DATA_PATH', FRESHRSS_PATH . '/data'); diff --git a/lib/Minz/Request.php b/lib/Minz/Request.php index 6db2e9c7a..b9eda82a5 100644 --- a/lib/Minz/Request.php +++ b/lib/Minz/Request.php @@ -84,45 +84,27 @@ class Minz_Request { self::magicQuotesOff(); } - /** - * Retourn le nom de domaine du site - */ - public static function getDomainName() { - return $_SERVER['HTTP_HOST']; - } - /** * Détermine la base de l'url * @return la base de l'url */ - public static function getBaseUrl() { + public static function getBaseUrl($baseUrlSuffix = '') { $conf = Minz_Configuration::get('system'); - $defaultBaseUrl = $conf->base_url; - if (!empty($defaultBaseUrl)) { - return $defaultBaseUrl; - } elseif (isset($_SERVER['REQUEST_URI'])) { - return dirname($_SERVER['REQUEST_URI']) . '/'; - } else { - return '/'; - } - } - - /** - * Récupère l'URI de la requête - * @return l'URI - */ - public static function getURI() { - if (isset($_SERVER['REQUEST_URI'])) { - $base_url = self::getBaseUrl(); - $uri = $_SERVER['REQUEST_URI']; - - $len_base_url = strlen($base_url); - $real_uri = substr($uri, $len_base_url); + $url = $conf->base_url; + if ($url == '' || !preg_match('%^https?://%i', $url)) { + $url = 'http'; + $host = empty($_SERVER['HTTP_HOST']) ? $_SERVER['SERVER_NAME'] : $_SERVER['HTTP_HOST']; + $port = empty($_SERVER['SERVER_PORT']) ? 80 : $_SERVER['SERVER_PORT']; + if (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on') { + $url .= 's://' . $host . ($port == 443 ? '' : ':' . $port); + } else { + $url .= '://' . $host . ($port == 80 ? '' : ':' . $port); + } + $url .= isset($_SERVER['REQUEST_URI']) ? dirname($_SERVER['REQUEST_URI']) : ''; } else { - $real_uri = ''; + $url = rtrim($url, '/\\') . $baseUrlSuffix; } - - return $real_uri; + return filter_var($url . '/', FILTER_SANITIZE_URL); } /** diff --git a/lib/Minz/Url.php b/lib/Minz/Url.php index af555a277..a47d8f1a6 100644 --- a/lib/Minz/Url.php +++ b/lib/Minz/Url.php @@ -10,7 +10,6 @@ class Minz_Url { * $url['c'] = controller * $url['a'] = action * $url['params'] = tableau des paramètres supplémentaires - * $url['protocol'] = protocole à utiliser (http par défaut) * ou comme une chaîne de caractère * @param $encodage pour indiquer comment encoder les & (& ou & pour html) * @return l'url formatée @@ -25,14 +24,7 @@ class Minz_Url { $url_string = ''; if ($absolute) { - if ($isArray && isset ($url['protocol'])) { - $protocol = $url['protocol']; - } elseif (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on') { - $protocol = 'https:'; - } else { - $protocol = 'http:'; - } - $url_string = $protocol . '//' . Minz_Request::getDomainName () . Minz_Request::getBaseUrl (); + $url_string = Minz_Request::getBaseUrl(PUBLIC_TO_INDEX_PATH); } else { $url_string = $isArray ? '.' : PUBLIC_RELATIVE; } -- cgit v1.2.3 From a5a24108ef923c47422f6b10db57863739a996cd Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 30 May 2015 19:25:20 +0200 Subject: Minz URL encoding bug See also https://github.com/FreshRSS/FreshRSS/pull/849 --- lib/Minz/Url.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/Minz/Url.php b/lib/Minz/Url.php index a47d8f1a6..879077d0f 100644 --- a/lib/Minz/Url.php +++ b/lib/Minz/Url.php @@ -18,7 +18,7 @@ class Minz_Url { $isArray = is_array($url); if ($isArray) { - $url = self::checkUrl ($url); + $url = self::checkUrl($url); } $url_string = ''; @@ -30,9 +30,9 @@ class Minz_Url { } if ($isArray) { - $url_string .= self::printUri ($url, $encodage); + $url_string .= self::printUri($url, $encodage); } else { - $url_string .= $url; + $url_string = Minz_Helper::htmlspecialchars_utf8($url_string . $url); } return $url_string; -- cgit v1.2.3