From ec611d7d543e27fe8c794cd98177d00cf636c58f Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 16 Mar 2015 21:37:28 +0100 Subject: SimplePie: decode special chars for MAYBE_HTML https://github.com/FreshRSS/FreshRSS/issues/754 Needs to check with many feeds to see if this does not introduce incompatibilities with some valid feeds. --- lib/SimplePie/SimplePie/Sanitize.php | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Sanitize.php b/lib/SimplePie/SimplePie/Sanitize.php index 168a5e2e8..3da7239be 100644 --- a/lib/SimplePie/SimplePie/Sanitize.php +++ b/lib/SimplePie/SimplePie/Sanitize.php @@ -249,6 +249,11 @@ class SimplePie_Sanitize { if ($type & SIMPLEPIE_CONSTRUCT_MAYBE_HTML) { + if (preg_match('/&#(x[0-9a-fA-F]+|[0-9]+);/', $data)) //FreshRSS + { + $data = htmlspecialchars_decode($data, ENT_QUOTES); //FreshRSS + //syslog(LOG_DEBUG, 'SimplePie sanitize MAYBE_HTML htmlspecialchars_decode: ' . $data); //FreshRSS + } if (preg_match('/(&(#(x[0-9a-fA-F]+|[0-9]+)|[a-zA-Z0-9]+)|<\/[A-Za-z][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E]*' . SIMPLEPIE_PCRE_HTML_ATTRIBUTE . '>)/', $data)) { $type |= SIMPLEPIE_CONSTRUCT_HTML; -- cgit v1.2.3 From 0e08b5ba56b2117e0f3d9bea6ae725295e8700d5 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 21 Mar 2015 15:13:08 +0100 Subject: htmlspecialchars_decode for all SIMPLEPIE_CONSTRUCT_MAYBE_HTML https://github.com/FreshRSS/FreshRSS/issues/754 --- lib/SimplePie/SimplePie/Sanitize.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Sanitize.php b/lib/SimplePie/SimplePie/Sanitize.php index 3da7239be..e7c9f925f 100644 --- a/lib/SimplePie/SimplePie/Sanitize.php +++ b/lib/SimplePie/SimplePie/Sanitize.php @@ -249,11 +249,7 @@ class SimplePie_Sanitize { if ($type & SIMPLEPIE_CONSTRUCT_MAYBE_HTML) { - if (preg_match('/&#(x[0-9a-fA-F]+|[0-9]+);/', $data)) //FreshRSS - { - $data = htmlspecialchars_decode($data, ENT_QUOTES); //FreshRSS - //syslog(LOG_DEBUG, 'SimplePie sanitize MAYBE_HTML htmlspecialchars_decode: ' . $data); //FreshRSS - } + $data = htmlspecialchars_decode($data, ENT_QUOTES); //FreshRSS if (preg_match('/(&(#(x[0-9a-fA-F]+|[0-9]+)|[a-zA-Z0-9]+)|<\/[A-Za-z][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E]*' . SIMPLEPIE_PCRE_HTML_ATTRIBUTE . '>)/', $data)) { $type |= SIMPLEPIE_CONSTRUCT_HTML; -- cgit v1.2.3 From 1a35e2271d3b9383e882371d37d5fef16abd745d Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 21 Mar 2015 18:20:36 +0100 Subject: SimplePie option to restaure syslog of HTTP requests https://github.com/FreshRSS/FreshRSS/issues/711 --- app/Models/Feed.php | 4 ++-- data/config.default.php | 1 + lib/SimplePie/SimplePie.php | 41 ++++++++++++++++++++++++++++++++++------ lib/SimplePie/SimplePie/File.php | 7 +++++-- lib/lib_rss.php | 1 + 5 files changed, 44 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 5ce03be5d..5f67ea6ce 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -246,10 +246,10 @@ class FreshRSS_Feed extends Minz_Model { } if (($mtime === true) ||($mtime > $this->lastUpdate)) { - Minz_Log::notice('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url); + //Minz_Log::debug('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url); $this->loadEntries($feed); // et on charge les articles du flux } else { - Minz_Log::notice('FreshRSS use cache for ' . $clean_url); + //Minz_Log::debug('FreshRSS use cache for ' . $clean_url); $this->entries = array(); } diff --git a/data/config.default.php b/data/config.default.php index 97df3a299..839bd1687 100644 --- a/data/config.default.php +++ b/data/config.default.php @@ -12,6 +12,7 @@ return array( 'auth_type' => 'none', 'api_enabled' => false, 'unsafe_autologin_enabled' => false, + 'simplepie_syslog_enabled' => true, 'limits' => array( 'cache_duration' => 800, 'timeout' => 10, diff --git a/lib/SimplePie/SimplePie.php b/lib/SimplePie/SimplePie.php index c4872b5be..bb8ce4191 100644 --- a/lib/SimplePie/SimplePie.php +++ b/lib/SimplePie/SimplePie.php @@ -74,6 +74,12 @@ define('SIMPLEPIE_USERAGENT', SIMPLEPIE_NAME . '/' . SIMPLEPIE_VERSION . ' (Feed */ define('SIMPLEPIE_LINKBACK', '' . SIMPLEPIE_NAME . ''); +/** + * Use syslog to report HTTP requests done by SimplePie. + * @see SimplePie::set_syslog() + */ +define('SIMPLEPIE_SYSLOG', true); //FreshRSS + /** * No Autodiscovery * @see SimplePie::set_autodiscovery_level() @@ -622,6 +628,12 @@ class SimplePie */ public $strip_htmltags = array('base', 'blink', 'body', 'doctype', 'embed', 'font', 'form', 'frame', 'frameset', 'html', 'iframe', 'input', 'marquee', 'meta', 'noscript', 'object', 'param', 'script', 'style'); + /** + * Use syslog to report HTTP requests done by SimplePie. + * @see SimplePie::set_syslog() + */ + public $syslog_enabled = SIMPLEPIE_SYSLOG; + /** * The SimplePie class contains feed level data and options * @@ -1136,7 +1148,7 @@ class SimplePie $this->sanitize->strip_attributes($attribs); } - public function add_attributes($attribs = '') + public function add_attributes($attribs = '') //FreshRSS { if ($attribs === '') { @@ -1145,6 +1157,14 @@ class SimplePie $this->sanitize->add_attributes($attribs); } + /** + * Use syslog to report HTTP requests done by SimplePie. + */ + public function set_syslog($value = SIMPLEPIE_SYSLOG) //FreshRSS + { + $this->syslog_enabled = $value == true; + } + /** * Set the output encoding * @@ -1231,7 +1251,8 @@ class SimplePie $this->enable_exceptions = $enable; } - function cleanMd5($rss) { //FreshRSS + function cleanMd5($rss) //FreshRSS + { return md5(preg_replace(array('#<(lastBuildDate|pubDate|updated|feedDate|dc:date|slash:comments)>[^<]+#', '##s'), '', $rss)); } @@ -1329,7 +1350,8 @@ class SimplePie list($headers, $sniffed) = $fetched; - if (isset($this->data['md5'])) { //FreshRSS + if (isset($this->data['md5'])) //FreshRSS + { $md5 = $this->data['md5']; } } @@ -1455,7 +1477,8 @@ class SimplePie { // Load the Cache $this->data = $cache->load(); - if ($cache->mtime() + $this->cache_duration > time()) { //FreshRSS + if ($cache->mtime() + $this->cache_duration > time()) //FreshRSS + { $this->raw_data = false; return true; // If the cache is still valid, just return true } @@ -1529,11 +1552,17 @@ class SimplePie { //FreshRSS $md5 = $this->cleanMd5($file->body); if ($this->data['md5'] === $md5) { - // syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . $this->feed_url); + if ($this->syslog_enabled) + { + syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . $this->feed_url); + } $cache->touch(); return true; //Content unchanged even though server did not send a 304 } else { - // syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . $this->feed_url); + if ($this->syslog_enabled) + { + syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . $this->feed_url); + } $this->data['md5'] = $md5; } } diff --git a/lib/SimplePie/SimplePie/File.php b/lib/SimplePie/SimplePie/File.php index 9625af2a9..56fe72196 100644 --- a/lib/SimplePie/SimplePie/File.php +++ b/lib/SimplePie/SimplePie/File.php @@ -66,7 +66,7 @@ class SimplePie_File var $method = SIMPLEPIE_FILE_SOURCE_NONE; var $permanent_url; //FreshRSS - public function __construct($url, $timeout = 10, $redirects = 5, $headers = null, $useragent = null, $force_fsockopen = false) + public function __construct($url, $timeout = 10, $redirects = 5, $headers = null, $useragent = null, $force_fsockopen = false, $syslog_enabled = SIMPLEPIE_SYSLOG) { if (class_exists('idna_convert')) { @@ -79,7 +79,10 @@ class SimplePie_File $this->useragent = $useragent; if (preg_match('/^http(s)?:\/\//i', $url)) { - // syslog(LOG_INFO, 'SimplePie GET ' . $url); //FreshRSS + if ($syslog_enabled) + { + syslog(LOG_INFO, 'SimplePie GET ' . $url); //FreshRSS + } if ($useragent === null) { $useragent = ini_get('user_agent'); diff --git a/lib/lib_rss.php b/lib/lib_rss.php index e5fe73041..16ae3097f 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -123,6 +123,7 @@ function customSimplePie() { $limits = $system_conf->limits; $simplePie = new SimplePie(); $simplePie->set_useragent(_t('gen.freshrss') . '/' . FRESHRSS_VERSION . ' (' . PHP_OS . '; ' . FRESHRSS_WEBSITE . ') ' . SIMPLEPIE_NAME . '/' . SIMPLEPIE_VERSION); + $simplePie->set_syslog($system_conf->simplepie_syslog_enabled); $simplePie->set_cache_location(CACHE_PATH); $simplePie->set_cache_duration($limits['cache_duration']); $simplePie->set_timeout($limits['timeout']); -- cgit v1.2.3 From ad9fe52f5a76faf58d13fcf7bde8f58e85abe82b Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 22 Mar 2015 22:54:29 +0100 Subject: SimplePie sanitize URLs for syslog https://github.com/FreshRSS/FreshRSS/issues/711 https://github.com/FreshRSS/FreshRSS/pull/715 --- app/Models/Feed.php | 2 +- lib/SimplePie/SimplePie.php | 4 ++-- lib/SimplePie/SimplePie/File.php | 2 +- lib/SimplePie/SimplePie/Misc.php | 10 ++++++++++ lib/lib_rss.php | 12 +----------- 5 files changed, 15 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 5f67ea6ce..15cbb7d0a 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -240,7 +240,7 @@ class FreshRSS_Feed extends Minz_Model { $subscribe_url = $feed->subscribe_url(true); } - $clean_url = url_remove_credentials($subscribe_url); + $clean_url = SimplePie_Misc::url_remove_credentials($subscribe_url); if ($subscribe_url !== null && $subscribe_url !== $url) { $this->_url($clean_url); } diff --git a/lib/SimplePie/SimplePie.php b/lib/SimplePie/SimplePie.php index bb8ce4191..54f4c5770 100644 --- a/lib/SimplePie/SimplePie.php +++ b/lib/SimplePie/SimplePie.php @@ -1554,14 +1554,14 @@ class SimplePie if ($this->data['md5'] === $md5) { if ($this->syslog_enabled) { - syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . $this->feed_url); + syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . SimplePie_Misc::url_remove_credentials($this->feed_url)); } $cache->touch(); return true; //Content unchanged even though server did not send a 304 } else { if ($this->syslog_enabled) { - syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . $this->feed_url); + syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . SimplePie_Misc::url_remove_credentials($this->feed_url)); } $this->data['md5'] = $md5; } diff --git a/lib/SimplePie/SimplePie/File.php b/lib/SimplePie/SimplePie/File.php index 56fe72196..1f9e3d502 100644 --- a/lib/SimplePie/SimplePie/File.php +++ b/lib/SimplePie/SimplePie/File.php @@ -81,7 +81,7 @@ class SimplePie_File { if ($syslog_enabled) { - syslog(LOG_INFO, 'SimplePie GET ' . $url); //FreshRSS + syslog(LOG_INFO, 'SimplePie GET ' . SimplePie_Misc::url_remove_credentials($url)); //FreshRSS } if ($useragent === null) { diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index 5a263a2e5..de50d37b8 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2240,5 +2240,15 @@ function embed_wmedia(width, height, link) { { // No-op } + + /** + * Sanitize a URL by removing HTTP credentials. + * @param $url the URL to sanitize. + * @return the same URL without HTTP credentials. + */ + function url_remove_credentials($url) //FreshRSS + { + return preg_replace('#(?<=//)[^/:@]+:[^/:@]+@#', '', $url); + } } diff --git a/lib/lib_rss.php b/lib/lib_rss.php index 16ae3097f..65a1a8e04 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -181,7 +181,7 @@ function sanitizeHTML($data, $base = '') { function get_content_by_parsing ($url, $path) { require_once (LIB_PATH . '/lib_phpQuery.php'); - Minz_Log::notice('FreshRSS GET ' . url_remove_credentials($url)); + Minz_Log::notice('FreshRSS GET ' . SimplePie_Misc::url_remove_credentials($url)); $html = file_get_contents ($url); if ($html) { @@ -430,13 +430,3 @@ function array_push_unique(&$array, $value) { function array_remove(&$array, $value) { $array = array_diff($array, array($value)); } - - -/** - * Sanitize a URL by removing HTTP credentials. - * @param $url the URL to sanitize. - * @return the same URL without HTTP credentials. - */ -function url_remove_credentials($url) { - return preg_replace('/[^\/]*:[^:]*@/', '', $url); -} -- cgit v1.2.3 From 1c38e646c3223571988b33e6a0d481426e3b6931 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 22 Mar 2015 23:53:59 +0100 Subject: SimplePie seems to only supports HTTP schemes Can simplify the regex (faster because anchored) for cleaning URLs based on the fact that we only have to deal with HTTP or HTTPS --- lib/SimplePie/SimplePie/Misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index de50d37b8..90f01f737 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2248,7 +2248,7 @@ function embed_wmedia(width, height, link) { */ function url_remove_credentials($url) //FreshRSS { - return preg_replace('#(?<=//)[^/:@]+:[^/:@]+@#', '', $url); + return preg_replace('#(?<=^https?://)[^/:@]+:[^/:@]+@#', '', $url); } } -- cgit v1.2.3 From 0ed213d97f566d7f46747d28a318cf58cdb5d699 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 22 Mar 2015 23:56:47 +0100 Subject: Revert "SimplePie seems to only supports HTTP schemes" This reverts commit 1c38e646c3223571988b33e6a0d481426e3b6931. --- lib/SimplePie/SimplePie/Misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index 90f01f737..de50d37b8 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2248,7 +2248,7 @@ function embed_wmedia(width, height, link) { */ function url_remove_credentials($url) //FreshRSS { - return preg_replace('#(?<=^https?://)[^/:@]+:[^/:@]+@#', '', $url); + return preg_replace('#(?<=//)[^/:@]+:[^/:@]+@#', '', $url); } } -- cgit v1.2.3 From 7735471140e1c6087a177192a09c7fb849decd77 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 23 Mar 2015 21:58:37 +0100 Subject: SimplePie faster regex for sanitizing URLs Can simplify the regex (faster because anchored) for cleaning URLs based on the fact that we only have to deal with HTTP or HTTPS https://github.com/FreshRSS/FreshRSS/pull/715 --- lib/SimplePie/SimplePie/Misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index de50d37b8..91549f93a 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2248,7 +2248,7 @@ function embed_wmedia(width, height, link) { */ function url_remove_credentials($url) //FreshRSS { - return preg_replace('#(?<=//)[^/:@]+:[^/:@]+@#', '', $url); + return preg_replace('#^(https?://)[^/:@]+:[^/:@]+@#i', '$1', $url); } } -- cgit v1.2.3 From 2bfc4dbf8b19cd2b892526ef65a966ff3bfef718 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Tue, 24 Mar 2015 21:58:00 +0100 Subject: SimplePie forgot static keyword https://github.com/FreshRSS/FreshRSS/issues/711 --- lib/SimplePie/SimplePie/Misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index 91549f93a..956a284cb 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -2246,7 +2246,7 @@ function embed_wmedia(width, height, link) { * @param $url the URL to sanitize. * @return the same URL without HTTP credentials. */ - function url_remove_credentials($url) //FreshRSS + public static function url_remove_credentials($url) //FreshRSS { return preg_replace('#^(https?://)[^/:@]+:[^/:@]+@#i', '$1', $url); } -- cgit v1.2.3 From 711530a512b370d79b079205ce1f8376174f7f03 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 4 Apr 2015 22:39:31 +0200 Subject: SQL: detection of updates, and preparation for better burge https://github.com/FreshRSS/FreshRSS/issues/798 https://github.com/FreshRSS/FreshRSS/issues/493 SQLite not yet tested. Only MySQL tested so far. --- app/Controllers/feedController.php | 98 ++++++++------ app/Controllers/importExportController.php | 3 +- app/Models/Entry.php | 16 +++ app/Models/EntryDAO.php | 198 +++++++++++++++++++++-------- app/Models/Feed.php | 1 + app/SQL/install.sql.mysql.php | 7 +- app/SQL/install.sql.sqlite.php | 7 +- lib/lib_rss.php | 2 +- 8 files changed, 231 insertions(+), 101 deletions(-) (limited to 'lib') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 6f544d834..08a0257a2 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -145,7 +145,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // Call the extension hook $name = $feed->name(); $feed = Minz_ExtensionManager::callHook('feed_before_insert', $feed); - if (is_null($feed)) { + if ($feed === null) { Minz_Request::bad(_t('feed_not_added', $name), $url_redirect); } @@ -181,7 +181,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // Use a shared statement and a transaction to improve a LOT the // performances. - $prepared_statement = $entryDAO->addEntryPrepare(); $feedDAO->beginTransaction(); foreach ($entries as $entry) { // Entries are added without any verification. @@ -190,13 +189,13 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $entry->_isRead($is_read); $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); - if (is_null($entry)) { + if ($entry === null) { // An extension has returned a null value, there is nothing to insert. continue; } $values = $entry->toArray(); - $entryDAO->addEntry($values, $prepared_statement); + $entryDAO->addEntry($values); } $feedDAO->updateLastUpdate($feed->id()); $feedDAO->commit(); @@ -307,7 +306,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $feed->load(false); } catch (FreshRSS_Feed_Exception $e) { Minz_Log::notice($e->getMessage()); - $feedDAO->updateLastUpdate($feed->id(), 1); + $feedDAO->updateLastUpdate($feed->id(), true); $feed->unlock(); continue; } @@ -323,50 +322,69 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // We want chronological order and SimplePie uses reverse order. $entries = array_reverse($feed->entries()); if (count($entries) > 0) { - // For this feed, check last n entry GUIDs already in database. - $existing_guids = array_fill_keys($entryDAO->listLastGuidsByFeed( - $feed->id(), count($entries) + 10 - ), 1); - $use_declared_date = empty($existing_guids); + $newGuids = array(); + foreach ($entries as $entry) { + $newGuids[] = $entry->guid(); + } + // For this feed, check existing GUIDs already in database. + $existingHashForGuids = $entryDAO->listHashForFeedGuids($feed->id(), $newGuids); + unset($newGuids); + $use_declared_date = empty($existingHashForGuids); + $oldGuids = array(); // Add entries in database if possible. - $prepared_statement = $entryDAO->addEntryPrepare(); - $feedDAO->beginTransaction(); foreach ($entries as $entry) { $entry_date = $entry->date(true); - if (isset($existing_guids[$entry->guid()]) || - ($feed_history == 0 && $entry_date < $date_min)) { - // This entry already exists in DB or should not be added - // considering configuration and date. - continue; - } - - $id = uTimeString(); - if ($use_declared_date || $entry_date < $date_min) { - // Use declared date at first import. - $id = min(time(), $entry_date) . uSecString(); + if (isset($existingHashForGuids[$entry->guid()])) { + $existingHash = $existingHashForGuids[$entry->guid()]; + if (strcasecmp($existingHash, $entry->hash()) === 0 || $existingHash === '00000000000000000000000000000000') { + //This entry already exists and is unchanged. TODO: Remove the test with the zero'ed hash in FreshRSS v1.3 + $oldGuids[] = $entry->guid(); + } else { //This entry already exists but has been updated + Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->id() . + ', old hash ' . $existingHash . ', new hash ' . $entry->hash()); + $entry->_isRead($is_read); //Reset is_read + if (!$entryDAO->hasTransaction()) { + $entryDAO->beginTransaction(); + } + $entryDAO->updateEntry($entry->toArray()); + } + } elseif ($feed_history == 0 && $entry_date < $date_min) { + // This entry should not be added considering configuration and date. + $oldGuids[] = $entry->guid(); + } else { + $id = uTimeString(); + if ($use_declared_date || $entry_date < $date_min) { + // Use declared date at first import. + $id = min(time(), $entry_date) . uSecString(); + } + + $entry->_id($id); + $entry->_isRead($is_read); + + $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); + if ($entry === null) { + // An extension has returned a null value, there is nothing to insert. + continue; + } + + if (!$entryDAO->hasTransaction()) { + $entryDAO->beginTransaction(); + } + $entryDAO->addEntry($entry->toArray()); } - - $entry->_id($id); - $entry->_isRead($is_read); - - $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); - if (is_null($entry)) { - // An extension has returned a null value, there is nothing to insert. - continue; - } - - $values = $entry->toArray(); - $entryDAO->addEntry($values, $prepared_statement); } + $entryDAO->updateLastSeen($feed->id(), $oldGuids); } + //TODO: updateLastSeen old GUIDS once in a while, in the case of caching (i.e. the whole feed content has not changed) if ($feed_history >= 0 && rand(0, 30) === 1) { // TODO: move this function in web cron when available (see entry::purge) // Remove old entries once in 30. - if (!$feedDAO->hasTransaction()) { - $feedDAO->beginTransaction(); + if (!$entryDAO->hasTransaction()) { + $entryDAO->beginTransaction(); } + //TODO: more robust system based on entry.lastSeen to avoid cleaning entries that are still published in the RSS feed. $nb = $feedDAO->cleanOldEntries($feed->id(), $date_min, @@ -377,9 +395,9 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } } - $feedDAO->updateLastUpdate($feed->id(), 0, $feedDAO->hasTransaction()); - if ($feedDAO->hasTransaction()) { - $feedDAO->commit(); + $feedDAO->updateLastUpdate($feed->id(), 0, $entryDAO->hasTransaction()); + if ($entryDAO->hasTransaction()) { + $entryDAO->commit(); } if ($feed->url() !== $url) { diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index 589777b2a..26b163e43 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -361,7 +361,6 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { } // Then, articles are imported. - $prepared_statement = $this->entryDAO->addEntryPrepare(); $this->entryDAO->beginTransaction(); foreach ($article_object['items'] as $item) { if (!isset($article_to_feed[$item['id']])) { @@ -396,7 +395,7 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { } $values = $entry->toArray(); - $id = $this->entryDAO->addEntry($values, $prepared_statement); + $id = $this->entryDAO->addEntry($values); if (!$error && ($id === false)) { $error = true; diff --git a/app/Models/Entry.php b/app/Models/Entry.php index 346c98a92..6931c9f25 100644 --- a/app/Models/Entry.php +++ b/app/Models/Entry.php @@ -14,6 +14,7 @@ class FreshRSS_Entry extends Minz_Model { private $content; private $link; private $date; + private $hash = null; private $is_read; private $is_favorite; private $feed; @@ -88,6 +89,14 @@ class FreshRSS_Entry extends Minz_Model { } } + public function hash() { + if ($this->hash === null) { + //Do not include $this->date because it may be automatically generated when lacking + $this->hash = md5($this->link . $this->title . $this->author . $this->content . $this->tags(true)); + } + return $this->hash; + } + public function _id($value) { $this->id = $value; } @@ -95,18 +104,23 @@ class FreshRSS_Entry extends Minz_Model { $this->guid = $value; } public function _title($value) { + $this->hash = null; $this->title = $value; } public function _author($value) { + $this->hash = null; $this->author = $value; } public function _content($value) { + $this->hash = null; $this->content = $value; } public function _link($value) { + $this->hash = null; $this->link = $value; } public function _date($value) { + $this->hash = null; $value = intval($value); $this->date = $value > 1 ? $value : time(); } @@ -120,6 +134,7 @@ class FreshRSS_Entry extends Minz_Model { $this->feed = $value; } public function _tags($value) { + $this->hash = null; if (!is_array($value)) { $value = array($value); } @@ -182,6 +197,7 @@ class FreshRSS_Entry extends Minz_Model { 'content' => $this->content(), 'link' => $this->link(), 'date' => $this->date(true), + 'hash' => $this->hash(), 'is_read' => $this->isRead(), 'is_favorite' => $this->isFavorite(), 'id_feed' => $this->feed(), diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 9736d5cd3..5b4b85547 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -6,20 +6,57 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return parent::$sharedDbType !== 'sqlite'; } - public function addEntryPrepare() { - $sql = 'INSERT INTO `' . $this->prefix . 'entry`(id, guid, title, author, ' - . ($this->isCompressed() ? 'content_bin' : 'content') - . ', link, date, is_read, is_favorite, id_feed, tags) ' - . 'VALUES(?, ?, ?, ?, ' - . ($this->isCompressed() ? 'COMPRESS(?)' : '?') - . ', ?, ?, ?, ?, ?, ?)'; - return $this->bd->prepare($sql); + protected function autoAddColumn($errorInfo) { + if (isset($errorInfo[0])) { + if ($errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR + $hasTransaction = false; + try { + $stm = null; + if (stripos($errorInfo[2], 'lastSeen') !== false) { //v1.2 + if (!$this->bd->inTransaction()) { + $this->bd->beginTransaction(); + $hasTransaction = true; + } + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN lastSeen INT(11) NOT NULL'); + if ($stm && $stm->execute()) { + $stm = $this->bd->prepare('CREATE INDEX entry_lastSeen_index ON `' . $this->prefix . 'entry`(`lastSeen`);'); //"IF NOT EXISTS" does not exist in MySQL 5.7 + if ($stm && $stm->execute()) { + if ($hasTransaction) { + $this->bd->commit(); + } + return true; + } + } + if ($hasTransaction) { + $this->bd->rollBack(); + } + } elseif (stripos($errorInfo[2], 'hash') !== false) { //v1.2 + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN hash BINARY(16) NOT NULL'); + return $stm && $stm->execute(); + } + } catch (Exception $e) { + Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn error: ' . $e->getMessage()); + if ($hasTransaction) { + $this->bd->rollBack(); + } + } + } + } + return false; } - public function addEntry($valuesTmp, $preparedStatement = null) { - $stm = $preparedStatement === null ? - FreshRSS_EntryDAO::addEntryPrepare() : - $preparedStatement; + private $addEntryPrepared = null; + + public function addEntry($valuesTmp) { + if ($this->addEntryPrepared === null) { + $sql = 'INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, ' + . ($this->isCompressed() ? 'content_bin' : 'content') + . ', link, date, lastSeen, hash, is_read, is_favorite, id_feed, tags) ' + . 'VALUES(?, ?, ?, ?, ' + . ($this->isCompressed() ? 'COMPRESS(?)' : '?') + . ', ?, ?, ?, X?, ?, ?, ?, ?)'; + $this->addEntryPrepared = $this->bd->prepare($sql); + } $values = array( $valuesTmp['id'], @@ -29,55 +66,65 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $valuesTmp['content'], substr($valuesTmp['link'], 0, 1023), $valuesTmp['date'], + time(), + $valuesTmp['hash'], $valuesTmp['is_read'] ? 1 : 0, $valuesTmp['is_favorite'] ? 1 : 0, $valuesTmp['id_feed'], substr($valuesTmp['tags'], 0, 1023), ); - if ($stm && $stm->execute($values)) { + if ($this->addEntryPrepared && $this->addEntryPrepared->execute($values)) { return $this->bd->lastInsertId(); } else { - $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); - if ((int)($info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries + $info = $this->addEntryPrepared == null ? array(2 => 'syntax error') : $this->addEntryPrepared->errorInfo(); + if ($this->autoAddColumn($info)) { + return $this->addEntry($valuesTmp); + } elseif ((int)($info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']); - } /*else { - Minz_Log::debug('SQL error ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] - . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']); - }*/ + } return false; } } - public function addEntryObject($entry, $conf, $feedHistory) { - $existingGuids = array_fill_keys( - $this->listLastGuidsByFeed($entry->feed(), 20), 1 - ); - - $nb_month_old = max($conf->old_entries, 1); - $date_min = time() - (3600 * 24 * 30 * $nb_month_old); + private $updateEntryPrepared = null; - $eDate = $entry->date(true); - - if ($feedHistory == -2) { - $feedHistory = $conf->keep_history_default; + public function updateEntry($valuesTmp) { + if ($this->updateEntryPrepared === null) { + $sql = 'UPDATE `' . $this->prefix . 'entry` ' + . 'SET title=?, author=?, ' + . ($this->isCompressed() ? 'content_bin=COMPRESS(?)' : 'content=?') + . ', link=?, date=?, lastSeen=?, hash=X?, is_read=?, tags=? ' + . 'WHERE id_feed=? AND guid=?'; + $this->updateEntryPrepared = $this->bd->prepare($sql); } - if (!isset($existingGuids[$entry->guid()]) && - ($feedHistory != 0 || $eDate >= $date_min || $entry->isFavorite())) { - $values = $entry->toArray(); - - $useDeclaredDate = empty($existingGuids); - $values['id'] = ($useDeclaredDate || $eDate < $date_min) ? - min(time(), $eDate) . uSecString() : - uTimeString(); + $values = array( + substr($valuesTmp['title'], 0, 255), + substr($valuesTmp['author'], 0, 255), + $valuesTmp['content'], + substr($valuesTmp['link'], 0, 1023), + $valuesTmp['date'], + time(), + $valuesTmp['hash'], + $valuesTmp['is_read'] ? 1 : 0, + substr($valuesTmp['tags'], 0, 1023), + $valuesTmp['id_feed'], + substr($valuesTmp['guid'], 0, 760), + ); - return $this->addEntry($values); + if ($this->updateEntryPrepared && $this->updateEntryPrepared->execute($values)) { + return $this->bd->lastInsertId(); + } else { + $info = $this->updateEntryPrepared == null ? array(2 => 'syntax error') : $this->updateEntryPrepared->errorInfo(); + if ($this->autoAddColumn($info)) { + return $this->updateEntry($valuesTmp); + } + Minz_Log::error('SQL error updateEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] + . ' while updating entry with GUID ' . $valuesTmp['guid'] . ' in feed ' . $valuesTmp['id_feed']); + return false; } - - // We don't return Entry object to avoid a research in DB - return -1; } /** @@ -94,6 +141,9 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { if (!is_array($ids)) { $ids = array($ids); } + if (count($ids) < 1) { + return 0; + } $sql = 'UPDATE `' . $this->prefix . 'entry` ' . 'SET is_favorite=? ' . 'WHERE id IN (' . str_repeat('?,', count($ids) - 1). '?)'; @@ -296,11 +346,11 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { * * If $idMax equals 0, a deprecated debug message is logged * - * @param integer $id feed ID + * @param integer $id_feed feed ID * @param integer $idMax fail safe article ID * @return integer affected rows */ - public function markReadFeed($id, $idMax = 0) { + public function markReadFeed($id_feed, $idMax = 0) { if ($idMax == 0) { $idMax = time() . '000000'; Minz_Log::debug('Calling markReadFeed(0) is deprecated!'); @@ -310,7 +360,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $sql = 'UPDATE `' . $this->prefix . 'entry` ' . 'SET is_read=1 ' . 'WHERE id_feed=? AND is_read=0 AND id <= ?'; - $values = array($id, $idMax); + $values = array($id_feed, $idMax); $stm = $this->bd->prepare($sql); if (!($stm && $stm->execute($values))) { $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); @@ -324,7 +374,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $sql = 'UPDATE `' . $this->prefix . 'feed` ' . 'SET cache_nbUnreads=cache_nbUnreads-' . $affected . ' WHERE id=?'; - $values = array($id); + $values = array($id_feed); $stm = $this->bd->prepare($sql); if (!($stm && $stm->execute($values))) { $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); @@ -338,7 +388,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $affected; } - public function searchByGuid($feed_id, $id) { + public function searchByGuid($id_feed, $guid) { // un guid est unique pour un flux donné $sql = 'SELECT id, guid, title, author, ' . ($this->isCompressed() ? 'UNCOMPRESS(content_bin) AS content' : 'content') @@ -347,8 +397,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { $stm = $this->bd->prepare($sql); $values = array( - $feed_id, - $id + $id_feed, + $guid, ); $stm->execute($values); @@ -519,12 +569,52 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $stm->fetchAll(PDO::FETCH_COLUMN, 0); } - public function listLastGuidsByFeed($id, $n) { - $sql = 'SELECT guid FROM `' . $this->prefix . 'entry` WHERE id_feed=? ORDER BY id DESC LIMIT ' . intval($n); + public function listHashForFeedGuids($id_feed, $guids) { + if (count($guids) < 1) { + return array(); + } + $sql = 'SELECT guid, hex(hash) AS hexHash FROM `' . $this->prefix . 'entry` WHERE id_feed=? AND guid IN (' . str_repeat('?,', count($guids) - 1). '?)'; $stm = $this->bd->prepare($sql); - $values = array($id); - $stm->execute($values); - return $stm->fetchAll(PDO::FETCH_COLUMN, 0); + $values = array($id_feed); + $values = array_merge($values, $guids); + if ($stm && $stm->execute($values)) { + $result = array(); + $rows = $stm->fetchAll(PDO::FETCH_ASSOC); + foreach ($rows as $row) { + $result[$row['guid']] = $row['hexHash']; + } + return $result; + } else { + + $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); + if ($this->autoAddColumn($info)) { + return $this->listHashForFeedGuids($id_feed, $guids); + } + Minz_Log::error('SQL error listHashForFeedGuids: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] + . ' while querying feed ' . $id_feed); + return false; + } + } + + public function updateLastSeen($id_feed, $guids) { + if (count($guids) < 1) { + return 0; + } + $sql = 'UPDATE `' . $this->prefix . 'entry` SET lastSeen=? WHERE id_feed=? AND guid IN (' . str_repeat('?,', count($guids) - 1). '?)'; + $stm = $this->bd->prepare($sql); + $values = array(time(), $id_feed); + $values = array_merge($values, $guids); + if ($stm && $stm->execute($values)) { + return $stm->rowCount(); + } else { + $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); + if ($this->autoAddColumn($info)) { + return $this->updateLastSeen($id_feed, $guids); + } + Minz_Log::error('SQL error updateLastSeen: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] + . ' while updating feed ' . $id_feed); + return false; + } } public function countUnreadRead() { diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 5ce03be5d..27c83ffd5 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -255,6 +255,7 @@ class FreshRSS_Feed extends Minz_Model { $feed->__destruct(); //http://simplepie.org/wiki/faq/i_m_getting_memory_leaks unset($feed); + //TODO: Return a different information in case of cache/no-cache, and give access to the GUIDs in case of cache } } } diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index cf0159199..afdd821b2 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS `%1$sfeed` ( `name` varchar(255) NOT NULL, `website` varchar(255) CHARACTER SET latin1, `description` text, - `lastUpdate` int(11) DEFAULT 0, + `lastUpdate` int(11) DEFAULT 0, -- Until year 2038 `priority` tinyint(2) NOT NULL DEFAULT 10, `pathEntries` varchar(511) DEFAULT NULL, `httpAuth` varchar(511) DEFAULT NULL, @@ -40,7 +40,9 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( `author` varchar(255), `content_bin` blob, -- v0.7 `link` varchar(1023) CHARACTER SET latin1 NOT NULL, - `date` int(11), + `date` int(11), -- Until year 2038 + `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, `id_feed` SMALLINT, -- v0.7 @@ -50,6 +52,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( UNIQUE KEY (`id_feed`,`guid`), -- v0.7 INDEX (`is_favorite`), -- v0.7 INDEX (`is_read`) -- v0.7 + INDEX entry_lastSeen_index (`lastSeen`) -- v1.2 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci ENGINE = INNODB; diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 30bca2810..7517ead45 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -14,7 +14,7 @@ $SQL_CREATE_TABLES = array( `name` varchar(255) NOT NULL, `website` varchar(255), `description` text, - `lastUpdate` int(11) DEFAULT 0, + `lastUpdate` int(11) DEFAULT 0, -- Until year 2038 `priority` tinyint(2) NOT NULL DEFAULT 10, `pathEntries` varchar(511) DEFAULT NULL, `httpAuth` varchar(511) DEFAULT NULL, @@ -38,7 +38,9 @@ $SQL_CREATE_TABLES = array( `author` varchar(255), `content` text, `link` varchar(1023) NOT NULL, - `date` int(11), + `date` int(11), -- Until year 2038 + `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, `id_feed` SMALLINT, @@ -50,6 +52,7 @@ $SQL_CREATE_TABLES = array( 'CREATE INDEX IF NOT EXISTS entry_is_favorite_index ON `%1$sentry`(`is_favorite`);', 'CREATE INDEX IF NOT EXISTS entry_is_read_index ON `%1$sentry`(`is_read`);', +'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `%1$sentry`(`lastSeen`);', //v1.2 'INSERT OR IGNORE INTO `%1$scategory` (id, name) VALUES(1, "%2$s");', ); diff --git a/lib/lib_rss.php b/lib/lib_rss.php index e5fe73041..c6bdfde0e 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -38,7 +38,7 @@ function classAutoloader($class) { include(APP_PATH . '/Models/' . $components[1] . '.php'); return; case 3: //Controllers, Exceptions - @include(APP_PATH . '/' . $components[2] . 's/' . $components[1] . $components[2] . '.php'); + include(APP_PATH . '/' . $components[2] . 's/' . $components[1] . $components[2] . '.php'); return; } } elseif (strpos($class, 'Minz') === 0) { -- cgit v1.2.3 From 2d18910d02d92098257b96766e5b89a780daab0b Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 5 Apr 2015 12:41:16 +0200 Subject: Support for Internationalized Domain Names (IDN) https://github.com/FreshRSS/FreshRSS/issues/819 Add explicit conversion from IDN to Punycode. Requires PHP 5.3 IDN extension http://php.net/intl.idn (php5-idn package on Debian/Ubuntu). For systems without PHP 5.3+ IDN extension, we may consider adding a dependency (322 kB) to the third-party library https://phlymail.com/en/downloads/idna-convert.html See PHP bug 53474 FILTER_VALIDATE_URL should not fail URL's that use IDNhttps://bugs.php.net/bug.php?id=53474 --- README.fr.md | 2 +- README.md | 2 +- lib/lib_rss.php | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/README.fr.md b/README.fr.md index 380d7bc1e..ea5b07a9b 100644 --- a/README.fr.md +++ b/README.fr.md @@ -32,7 +32,7 @@ Privilégiez pour cela des demandes sur GitHub * Fonctionne même sur un Raspberry Pi avec des temps de réponse < 1s (testé sur 150 flux, 22k articles, soit 32Mo de données partiellement compressées) * Serveur Web Apache2 (recommandé), ou nginx, lighttpd (non testé sur les autres) * PHP 5.2.1+ (PHP 5.3.7+ recommandé) - * Requis : [PDO_MySQL](http://php.net/pdo-mysql) ou [PDO_SQLite](http://php.net/pdo-sqlite), [cURL](http://php.net/curl), [GMP](http://php.net/gmp) (seulement pour accès API sur platformes < 64 bits) + * Requis : [PDO_MySQL](http://php.net/pdo-mysql) ou [PDO_SQLite](http://php.net/pdo-sqlite), [cURL](http://php.net/curl), [GMP](http://php.net/gmp) (pour accès API sur platformes < 64 bits), [IDN](http://php.net/intl.idn) (pour les noms de domaines internationalisés) * Recommandés : [JSON](http://php.net/json), [mbstring](http://php.net/mbstring), [zlib](http://php.net/zlib), [Zip](http://php.net/zip) * MySQL 5.0.3+ (recommandé) ou SQLite 3.7.4+ * Un navigateur Web récent tel Firefox 4+, Chrome, Opera, Safari, Internet Explorer 9+ diff --git a/README.md b/README.md index a09a64639..92072b07d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The best way is to open issues on GitHub * It even works on Raspberry Pi with response time under a second (tested with 150 feeds, 22k articles, or 32Mo of compressed data) * A web server: Apache2 (recommanded), nginx, lighttpd (not tested on others) * PHP 5.2.1+ (PHP 5.3.7+ recommanded) - * Required extensions: [PDO_MySQL](http://php.net/pdo-mysql) or [PDO_SQLite](http://php.net/pdo-sqlite), [cURL](http://php.net/curl), [GMP](http://php.net/gmp) (only for API access on platforms under 64 bits) + * Required extensions: [PDO_MySQL](http://php.net/pdo-mysql) or [PDO_SQLite](http://php.net/pdo-sqlite), [cURL](http://php.net/curl), [GMP](http://php.net/gmp) (for API access on platforms < 64 bits), [IDN](http://php.net/intl.idn) (for Internationalized Domain Names) * Recommanded extensions : [JSON](http://php.net/json), [mbstring](http://php.net/mbstring), [zlib](http://php.net/zlib), [Zip](http://php.net/zip) * MySQL 5.0.3+ (recommanded) or SQLite 3.7.4+ * A recent browser like Firefox 4+, Chrome, Opera, Safari, Internet Explorer 9+ diff --git a/lib/lib_rss.php b/lib/lib_rss.php index e5fe73041..bc5d6fc5b 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -51,6 +51,21 @@ function classAutoloader($class) { spl_autoload_register('classAutoloader'); // +function idn_to_punny($url) { + if (function_exists('idn_to_ascii')) { + $parts = parse_url($url); + if (!empty($parts['host'])) { + $idn = $parts['host']; + $punny = idn_to_ascii($idn); + $pos = strpos($url, $idn); + if ($pos !== false) { + return substr_replace($url, $punny, $pos, strlen($idn)); + } + } + } + return $url; +} + function checkUrl($url) { if (empty ($url)) { return ''; @@ -58,6 +73,7 @@ function checkUrl($url) { if (!preg_match ('#^https?://#i', $url)) { $url = 'http://' . $url; } + $url = idn_to_punny($url); //PHP bug #53474 IDN if (filter_var($url, FILTER_VALIDATE_URL) || (version_compare(PHP_VERSION, '5.3.3', '<') && (strpos($url, '-') > 0) && //PHP bug #51192 ($url === filter_var($url, FILTER_SANITIZE_URL)))) { -- cgit v1.2.3 From 06b76831dece01f836c9d0a3cc32c3f59910fe60 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 5 Apr 2015 13:07:34 +0200 Subject: Punycode spelling mistake https://github.com/FreshRSS/FreshRSS/pull/820 --- data/subscriptions/.gitignore | 1 + lib/lib_rss.php | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 data/subscriptions/.gitignore (limited to 'lib') diff --git a/data/subscriptions/.gitignore b/data/subscriptions/.gitignore new file mode 100644 index 000000000..150f68c80 --- /dev/null +++ b/data/subscriptions/.gitignore @@ -0,0 +1 @@ +*/* diff --git a/lib/lib_rss.php b/lib/lib_rss.php index bc5d6fc5b..c4f6a6011 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -51,15 +51,15 @@ function classAutoloader($class) { spl_autoload_register('classAutoloader'); // -function idn_to_punny($url) { +function idn_to_puny($url) { if (function_exists('idn_to_ascii')) { $parts = parse_url($url); if (!empty($parts['host'])) { $idn = $parts['host']; - $punny = idn_to_ascii($idn); + $puny = idn_to_ascii($idn); $pos = strpos($url, $idn); if ($pos !== false) { - return substr_replace($url, $punny, $pos, strlen($idn)); + return substr_replace($url, $puny, $pos, strlen($idn)); } } } @@ -73,7 +73,7 @@ function checkUrl($url) { if (!preg_match ('#^https?://#i', $url)) { $url = 'http://' . $url; } - $url = idn_to_punny($url); //PHP bug #53474 IDN + $url = idn_to_puny($url); //PHP bug #53474 IDN if (filter_var($url, FILTER_VALIDATE_URL) || (version_compare(PHP_VERSION, '5.3.3', '<') && (strpos($url, '-') > 0) && //PHP bug #51192 ($url === filter_var($url, FILTER_SANITIZE_URL)))) { -- cgit v1.2.3 From 217c191a1ba3ac03b847d261a32e19975380fcad Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 11 May 2015 22:42:41 +0200 Subject: More SQLite compatibility Additional changes to add compatibility with SQLite for the new hash/lastSeen mode of updating articles. --- app/Models/EntryDAO.php | 71 ++++++++++++++++++++++++------------------ app/Models/EntryDAOSQLite.php | 15 +++++++++ app/Models/FeedDAO.php | 11 ++++--- app/SQL/install.sql.mysql.php | 2 +- app/SQL/install.sql.sqlite.php | 2 +- app/install.php | 2 ++ lib/Minz/ModelPdo.php | 5 +++ 7 files changed, 70 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index 172eac897..eae9683ad 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -6,38 +6,48 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return parent::$sharedDbType !== 'sqlite'; } - protected function autoAddColumn($errorInfo) { - if (isset($errorInfo[0])) { - if ($errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR - $hasTransaction = false; - try { - $stm = null; - if (stripos($errorInfo[2], 'lastSeen') !== false) { //v1.2 - if (!$this->bd->inTransaction()) { - $this->bd->beginTransaction(); - $hasTransaction = true; - } - $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN lastSeen INT(11) NOT NULL'); - if ($stm && $stm->execute()) { - $stm = $this->bd->prepare('CREATE INDEX entry_lastSeen_index ON `' . $this->prefix . 'entry`(`lastSeen`);'); //"IF NOT EXISTS" does not exist in MySQL 5.7 - if ($stm && $stm->execute()) { - if ($hasTransaction) { - $this->bd->commit(); - } - return true; - } - } + protected function addColumn($name) { + Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn: ' . $name); + $hasTransaction = false; + try { + $stm = null; + if ($name === 'lastSeen') { //v1.2 + if (!$this->bd->inTransaction()) { + $this->bd->beginTransaction(); + $hasTransaction = true; + } + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN lastSeen INT(11) DEFAULT 0'); + if ($stm && $stm->execute()) { + $stm = $this->bd->prepare('CREATE INDEX entry_lastSeen_index ON `' . $this->prefix . 'entry`(`lastSeen`);'); //"IF NOT EXISTS" does not exist in MySQL 5.7 + if ($stm && $stm->execute()) { if ($hasTransaction) { - $this->bd->rollBack(); + $this->bd->commit(); } - } elseif (stripos($errorInfo[2], 'hash') !== false) { //v1.2 - $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN hash BINARY(16) NOT NULL'); - return $stm && $stm->execute(); + return true; } - } catch (Exception $e) { - Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn error: ' . $e->getMessage()); - if ($hasTransaction) { - $this->bd->rollBack(); + } + if ($hasTransaction) { + $this->bd->rollBack(); + } + } elseif ($name === 'hash') { //v1.2 + $stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'entry` ADD COLUMN hash BINARY(16)'); + return $stm && $stm->execute(); + } + } catch (Exception $e) { + Minz_Log::debug('FreshRSS_EntryDAO::autoAddColumn error: ' . $e->getMessage()); + if ($hasTransaction) { + $this->bd->rollBack(); + } + } + return false; + } + + protected function autoAddColumn($errorInfo) { + if (isset($errorInfo[0])) { + if ($errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR + foreach (array('lastSeen', 'hash') as $column) { + if (stripos($errorInfo[2], $column) !== false) { + return $this->addColumn($column); } } } @@ -82,7 +92,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return $this->addEntry($valuesTmp); } elseif ((int)($info[0] / 1000) !== 23) { //Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2] - . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']); + . ' while adding entry in feed ' . $valuesTmp['id_feed'] . ' with title: ' . $valuesTmp['title']. ' ' . $this->addEntryPrepared); } return false; } @@ -597,7 +607,6 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { } return $result; } else { - $info = $stm == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $stm->errorInfo(); if ($this->autoAddColumn($info)) { return $this->listHashForFeedGuids($id_feed, $guids); diff --git a/app/Models/EntryDAOSQLite.php b/app/Models/EntryDAOSQLite.php index ffe0f037c..ff049d813 100644 --- a/app/Models/EntryDAOSQLite.php +++ b/app/Models/EntryDAOSQLite.php @@ -2,6 +2,21 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO { + protected function autoAddColumn($errorInfo) { + if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') { //ER_BAD_FIELD_ERROR + if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) { + $showCreate = $tableInfo->fetchColumn(); + Minz_Log::debug('FreshRSS_EntryDAOSQLite::autoAddColumn: ' . $showCreate); + foreach (array('lastSeen', 'hash') as $column) { + if (stripos($showCreate, $column) === false) { + return $this->addColumn($column); + } + } + } + } + return false; + } + protected function sqlConcat($s1, $s2) { return $s1 . '||' . $s2; } diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 76025ff53..475d39286 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -330,11 +330,12 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable { . 'AND id NOT IN (SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=:id_feed ORDER BY id DESC LIMIT :keep) keep)'; //Double select: MySQL doesn't support 'LIMIT & IN/ALL/ANY/SOME subquery' $stm = $this->bd->prepare($sql); - $id_max = intval($date_min) . '000000'; - - $stm->bindParam(':id_feed', $id, PDO::PARAM_INT); - $stm->bindParam(':id_max', $id_max, PDO::PARAM_STR); - $stm->bindParam(':keep', $keep, PDO::PARAM_INT); + if ($stm) { + $id_max = intval($date_min) . '000000'; + $stm->bindParam(':id_feed', $id, PDO::PARAM_INT); + $stm->bindParam(':id_max', $id_max, PDO::PARAM_STR); + $stm->bindParam(':keep', $keep, PDO::PARAM_INT); + } if ($stm && $stm->execute()) { return $stm->rowCount(); diff --git a/app/SQL/install.sql.mysql.php b/app/SQL/install.sql.mysql.php index afdd821b2..9c6af405d 100644 --- a/app/SQL/install.sql.mysql.php +++ b/app/SQL/install.sql.mysql.php @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` ( `content_bin` blob, -- v0.7 `link` varchar(1023) CHARACTER SET latin1 NOT NULL, `date` int(11), -- Until year 2038 - `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `lastSeen` INT(11) DEFAULT 0, -- v1.2, Until year 2038 `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, diff --git a/app/SQL/install.sql.sqlite.php b/app/SQL/install.sql.sqlite.php index 7517ead45..77e8e094c 100644 --- a/app/SQL/install.sql.sqlite.php +++ b/app/SQL/install.sql.sqlite.php @@ -39,7 +39,7 @@ $SQL_CREATE_TABLES = array( `content` text, `link` varchar(1023) NOT NULL, `date` int(11), -- Until year 2038 - `lastSeen` INT(11) NOT NULL, -- v1.2, Until year 2038 + `lastSeen` INT(11) DEFAULT 0, -- v1.2, Until year 2038 `hash` BINARY(16), -- v1.2 `is_read` boolean NOT NULL DEFAULT 0, `is_favorite` boolean NOT NULL DEFAULT 0, diff --git a/app/install.php b/app/install.php index 177173fdb..86afb9318 100644 --- a/app/install.php +++ b/app/install.php @@ -168,8 +168,10 @@ function saveStep3() { $_SESSION['bd_prefix_user'] = $_SESSION['bd_prefix'] .(empty($_SESSION['default_user']) ? '' :($_SESSION['default_user'] . '_')); } + //TODO: load `config.default.php` as default $config_array = array( 'environment' => 'production', + 'simplepie_syslog_enabled' => true, 'salt' => $_SESSION['salt'], 'title' => $_SESSION['title'], 'default_user' => $_SESSION['default_user'], diff --git a/lib/Minz/ModelPdo.php b/lib/Minz/ModelPdo.php index ac7a1bed7..3e8ec1f43 100644 --- a/lib/Minz/ModelPdo.php +++ b/lib/Minz/ModelPdo.php @@ -134,4 +134,9 @@ class MinzPDO extends PDO { MinzPDO::check($statement); return parent::exec($statement); } + + public function query($statement) { + MinzPDO::check($statement); + return parent::query($statement); + } } -- cgit v1.2.3 From 256c8613a4046931dcd28ab22b6aebe8752a98c2 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Fri, 15 May 2015 03:21:36 +0200 Subject: First draft of PubSubHubbub https://github.com/FreshRSS/FreshRSS/issues/312 Requires setting base_url in config.php. Currently using the filesystem (no change to the database) --- app/Controllers/feedController.php | 55 +++++++++++------ app/Models/Feed.php | 69 ++++++++++++++++++++- constants.php | 1 + data/PubSubHubbub/feeds/.gitignore | 1 + data/PubSubHubbub/feeds/README.md | 12 ++++ data/PubSubHubbub/secrets/.gitignore | 1 + data/PubSubHubbub/secrets/README.md | 4 ++ data/config.default.php | 8 ++- lib/lib_rss.php | 9 +++ p/api/pshb.php | 116 +++++++++++++++++++++++++++++++++++ 10 files changed, 252 insertions(+), 24 deletions(-) create mode 100644 data/PubSubHubbub/feeds/.gitignore create mode 100644 data/PubSubHubbub/feeds/README.md create mode 100644 data/PubSubHubbub/secrets/.gitignore create mode 100644 data/PubSubHubbub/secrets/README.md create mode 100644 p/api/pshb.php (limited to 'lib') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 0443b4159..9117da639 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -168,6 +168,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { // Ok, feed has been added in database. Now we have to refresh entries. $feed->_id($id); $feed->faviconPrepare(); + $feed->pubSubHubbubPrepare(); $is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0; @@ -261,12 +262,13 @@ class FreshRSS_feed_Controller extends Minz_ActionController { * This action actualizes entries from one or several feeds. * * Parameters are: - * - id (default: false) + * - id (default: false): Feed ID + * - url (default: false): Feed URL * - force (default: false) - * If id is not specified, all the feeds are actualized. But if force is + * If id and url are not specified, all the feeds are actualized. But if force is * false, process stops at 10 feeds to avoid time execution problem. */ - public function actualizeAction() { + public function actualizeAction($simplePie = null) { @set_time_limit(300); $feedDAO = FreshRSS_Factory::createFeedDao(); @@ -274,14 +276,15 @@ class FreshRSS_feed_Controller extends Minz_ActionController { Minz_Session::_param('actualize_feeds', false); $id = Minz_Request::param('id'); + $url = Minz_Request::param('url'); $force = Minz_Request::param('force'); // Create a list of feeds to actualize. // If id is set and valid, corresponding feed is added to the list but // alone in order to automatize further process. $feeds = array(); - if ($id) { - $feed = $feedDAO->searchById($id); + if ($id || $url) { + $feed = $id ? $feedDAO->searchById($id) : $feedDAO->searchByUrl($url); if ($feed) { $feeds[] = $feed; } @@ -302,8 +305,11 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } try { - // Load entries - $feed->load(false); + if ($simplePie) { + $feed->loadEntries($simplePie); //Used by PubSubHubbub + } else { + $feed->load(false); + } } catch (FreshRSS_Feed_Exception $e) { Minz_Log::notice($e->getMessage()); $feedDAO->updateLastUpdate($feed->id(), true); @@ -404,7 +410,16 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $feedDAO->updateFeed($feed->id(), array('url' => $feed->url())); } - $feed->faviconPrepare(); + if ($simplePie === null) { + $feed->faviconPrepare(); + if ($feed->url() === 'http://push-pub.appspot.com/feed') { + $secret = $feed->pubSubHubbubPrepare(); + if ($secret != '') { + Minz_Log::debug('PubSubHubbub subscribe ' . $feed->url()); + $feed->pubSubHubbubSubscribe(true, $secret); + } + } + } $feed->unlock(); $updated_feeds++; unset($feed); @@ -427,20 +442,20 @@ class FreshRSS_feed_Controller extends Minz_ActionController { Minz_Session::_param('notification', $notif); // No layout in ajax request. $this->view->_useLayout(false); - return; - } - - // Redirect to the main page with correct notification. - if ($updated_feeds === 1) { - $feed = reset($feeds); - Minz_Request::good(_t('feedback.sub.feed.actualized', $feed->name()), array( - 'params' => array('get' => 'f_' . $feed->id()) - )); - } elseif ($updated_feeds > 1) { - Minz_Request::good(_t('feedback.sub.feed.n_actualized', $updated_feeds), array()); } else { - Minz_Request::good(_t('feedback.sub.feed.no_refresh'), array()); + // Redirect to the main page with correct notification. + if ($updated_feeds === 1) { + $feed = reset($feeds); + Minz_Request::good(_t('feedback.sub.feed.actualized', $feed->name()), array( + 'params' => array('get' => 'f_' . $feed->id()) + )); + } elseif ($updated_feeds > 1) { + Minz_Request::good(_t('feedback.sub.feed.n_actualized', $updated_feeds), array()); + } else { + Minz_Request::good(_t('feedback.sub.feed.no_refresh'), array()); + } } + return $updated_feeds; } /** diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 85fb173ec..dcf083ea8 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -19,6 +19,8 @@ class FreshRSS_Feed extends Minz_Model { private $ttl = -2; private $hash = null; private $lockPath = ''; + private $hubUrl = ''; + private $selfUrl = ''; public function __construct($url, $validate=true) { if ($validate) { @@ -226,6 +228,11 @@ class FreshRSS_Feed extends Minz_Model { throw new FreshRSS_Feed_Exception(($errorMessage == '' ? 'Feed error' : $errorMessage) . ' [' . $url . ']'); } + $links = $feed->get_links('self'); + $this->selfUrl = isset($links[0]) ? $links[0] : null; + $links = $feed->get_links('hub'); + $this->hubUrl = isset($links[0]) ? $links[0] : null; + if ($loadDetails) { // si on a utilisé l'auto-discover, notre url va avoir changé $subscribe_url = $feed->subscribe_url(false); @@ -259,7 +266,7 @@ class FreshRSS_Feed extends Minz_Model { } } - private function loadEntries($feed) { + public function loadEntries($feed) { $entries = array(); foreach ($feed->get_items() as $item) { @@ -333,4 +340,64 @@ class FreshRSS_Feed extends Minz_Model { function unlock() { @unlink($this->lockPath); } + + // + + function pubSubHubbubPrepare() { + $secret = ''; + if (FreshRSS_Context::$system_conf->base_url && $this->hubUrl && $this->selfUrl) { + $path = PSHB_PATH . '/feeds/' . base64url_encode($this->selfUrl); + if (!file_exists($path . '/hub.txt')) { + @mkdir($path, 0777, true); + file_put_contents($path . '/hub.txt', $this->hubUrl); + $secret = sha1(FreshRSS_Context::$system_conf->salt . uniqid(mt_rand(), true)); + file_put_contents($path . '/secret.txt', $secret); + @mkdir(PSHB_PATH . '/secrets/'); + file_put_contents(PSHB_PATH . '/secrets/' . $secret . '.txt', base64url_encode($this->selfUrl)); + Minz_Log::notice('PubSubHubbub prepared for ' . $this->url); + file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . + 'PubSubHubbub prepared for ' . $this->url . "\n", FILE_APPEND); + } + $path .= '/' . base64url_encode($this->url); + $currentUser = Minz_Session::param('currentUser'); + if (ctype_alnum($currentUser) && !file_exists($path . '/' . $currentUser . '.txt')) { + @mkdir($path, 0777, true); + touch($path . '/' . $currentUser . '.txt'); + } + } + return $secret; + } + + //Parameter true to subscribe, false to unsubscribe. + function pubSubHubbubSubscribe($state, $secret = '') { + if (FreshRSS_Context::$system_conf->base_url && $this->hubUrl && $this->selfUrl) { + $callbackUrl = checkUrl(FreshRSS_Context::$system_conf->base_url . 'api/pshb.php?s=' . $secret); + if ($callbackUrl == '') { + return false; + } + + $ch = curl_init(); + curl_setopt_array($ch, array( + CURLOPT_URL => $this->hubUrl, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_USERAGENT => _t('gen.freshrss') . '/' . FRESHRSS_VERSION . ' (' . PHP_OS . '; ' . FRESHRSS_WEBSITE . ')', + CURLOPT_POSTFIELDS => 'hub.verify=sync' + . '&hub.mode=' . ($state ? 'subscribe' : 'unsubscribe') + . '&hub.topic=' . urlencode($this->selfUrl) + . '&hub.callback=' . urlencode($callbackUrl) + ) + ); + $response = curl_exec($ch); + $info = curl_getinfo($ch); + + file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . + 'PubSubHubbub ' . ($state ? 'subscribe' : 'unsubscribe') . ' to ' . $this->selfUrl . + ' with callback ' . $callbackUrl . ': ' . $info['http_code'] . ' ' . $response . "\n", FILE_APPEND); + return substr($info['http_code'], 0, 1) == '2'; + } + return false; + } + + // } diff --git a/constants.php b/constants.php index b20bf0710..5bb410e29 100644 --- a/constants.php +++ b/constants.php @@ -18,6 +18,7 @@ define('FRESHRSS_PATH', dirname(__FILE__)); define('UPDATE_FILENAME', DATA_PATH . '/update.php'); define('USERS_PATH', DATA_PATH . '/users'); define('CACHE_PATH', DATA_PATH . '/cache'); + define('PSHB_PATH', DATA_PATH . '/PubSubHubbub'); define('LIB_PATH', FRESHRSS_PATH . '/lib'); define('APP_PATH', FRESHRSS_PATH . '/app'); diff --git a/data/PubSubHubbub/feeds/.gitignore b/data/PubSubHubbub/feeds/.gitignore new file mode 100644 index 000000000..150f68c80 --- /dev/null +++ b/data/PubSubHubbub/feeds/.gitignore @@ -0,0 +1 @@ +*/* diff --git a/data/PubSubHubbub/feeds/README.md b/data/PubSubHubbub/feeds/README.md new file mode 100644 index 000000000..15fa8e521 --- /dev/null +++ b/data/PubSubHubbub/feeds/README.md @@ -0,0 +1,12 @@ +List of canonical URLS of the various feeds users have subscribed to. +Several feeds can share the same canonical URL (rel="self"). +Several users can have subscribed to the same feed. + +* ./base64url(canonicalUrl)/ + * ./secret.txt + * ./base64url(feedUrl1)/ + * ./user1.txt + * ./user2.txt + * ./base64url(feedUrl2)/ + * ./user3.txt + * ./user4.txt diff --git a/data/PubSubHubbub/secrets/.gitignore b/data/PubSubHubbub/secrets/.gitignore new file mode 100644 index 000000000..2211df63d --- /dev/null +++ b/data/PubSubHubbub/secrets/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/data/PubSubHubbub/secrets/README.md b/data/PubSubHubbub/secrets/README.md new file mode 100644 index 000000000..ad8158839 --- /dev/null +++ b/data/PubSubHubbub/secrets/README.md @@ -0,0 +1,4 @@ +List of secrets given to PubSubHubbub hubs + +* ./sha1(random + salt).txt + * base64url(canonicalUrl) diff --git a/data/config.default.php b/data/config.default.php index 8be203d36..80d331df7 100644 --- a/data/config.default.php +++ b/data/config.default.php @@ -11,9 +11,11 @@ return array( # Used to make crypto more unique. Generated during install. 'salt' => '', - # Leave empty for most cases. - # Ability to override the address of the FreshRSS instance, - # used when building absolute URLs. + # Specify address of the FreshRSS instance, + # used when building absolute URLs, e.g. for PubSubHubbub. + # Examples: + # https://example.net/FreshRSS/p/ + # https://freshrss.example.net/ 'base_url' => '', # Natural language of the user interface, e.g. `en`, `fr`. diff --git a/lib/lib_rss.php b/lib/lib_rss.php index 6342011c8..191a58f35 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -446,3 +446,12 @@ function array_push_unique(&$array, $value) { function array_remove(&$array, $value) { $array = array_diff($array, array($value)); } + +//RFC 4648 +function base64url_encode($data) { + return strtr(rtrim(base64_encode($data), '='), '+/', '-_'); +} +//RFC 4648 +function base64url_decode($data) { + return base64_decode(strtr($data, '-_', '+/')); +} diff --git a/p/api/pshb.php b/p/api/pshb.php new file mode 100644 index 000000000..bcb8341b1 --- /dev/null +++ b/p/api/pshb.php @@ -0,0 +1,116 @@ + $_GET, '_POST' => $_POST, 'INPUT' => $ORIGINAL_INPUT), true)); + +$secret = isset($_GET['s']) ? substr($_GET['s'], 0, 128) : ''; +if (!ctype_xdigit($secret)) { + header('HTTP/1.1 422 Unprocessable Entity'); + die('Invalid feed secret format!'); +} +chdir(PSHB_PATH); +$canonical64 = @file_get_contents('secrets/' . $secret . '.txt'); +if ($canonical64 === false) { + header('HTTP/1.1 404 Not Found'); + logMe('Feed secret not found!: ' . $secret); + die('Feed secret not found!'); +} +$canonical64 = trim($canonical64); +if (!preg_match('/^[A-Za-z0-9_-]+$/D', $canonical64)) { + header('HTTP/1.1 500 Internal Server Error'); + logMe('Invalid secret reference!: ' . $canonical64); + die('Invalid secret reference!'); +} +$secret2 = @file_get_contents('feeds/' . $canonical64 . '/secret.txt'); +if ($secret2 === false) { + header('HTTP/1.1 404 Not Found'); + //@unlink('secrets/' . $secret . '.txt'); + logMe('Feed reverse secret not found!: ' . $canonical64); + die('Feed reverse secret not found!'); +} +if ($secret !== $secret2) { + header('HTTP/1.1 500 Internal Server Error'); + logMe('Invalid secret cross-check!: ' . $secret); + die('Invalid secret cross-check!'); +} +chdir('feeds/' . $canonical64); +$users = glob('*/*.txt', GLOB_NOSORT); +if (empty($users)) { + header('HTTP/1.1 410 Gone'); + logMe('Nobody is subscribed to this feed anymore!: ' . $canonical64); + die('Nobody is subscribed to this feed anymore!'); +} + +if (!empty($_REQUEST['hub_mode']) && $_REQUEST['hub_mode'] === 'subscribe') { + //TODO: hub_lease_seconds + exit(isset($_REQUEST['hub_challenge']) ? $_REQUEST['hub_challenge'] : ''); +} + +Minz_Configuration::register('system', DATA_PATH . '/config.php', DATA_PATH . '/config.default.php'); +$system_conf = Minz_Configuration::get('system'); +$system_conf->auth_type = 'none'; // avoid necessity to be logged in (not saved!) +Minz_Translate::init('en'); +Minz_Request::_param('ajax', true); +$feedController = new FreshRSS_feed_Controller(); + +$simplePie = customSimplePie(); +$simplePie->set_raw_data($ORIGINAL_INPUT); +$simplePie->init(); +unset($ORIGINAL_INPUT); + +$links = $simplePie->get_links('self'); +$self = isset($links[0]) ? $links[0] : null; + +if ($self !== base64url_decode($canonical64)) { + header('HTTP/1.1 422 Unprocessable Entity'); + logMe('Self URL does not match registered canonical URL!: ' . $self); + die('Self URL does not match registered canonical URL!'); +} +Minz_Request::_param('url', $self); + +$nb = 0; +foreach ($users as $userLine) { + $userLine = strtr($userLine, '\\', '/'); + $userInfos = explode('/', $userLine); + $feedUrl = isset($userInfos[0]) ? base64url_decode($userInfos[0]) : ''; + $username = isset($userInfos[1]) ? basename($userInfos[1], '.txt') : ''; + if (!file_exists(USERS_PATH . '/' . $username . '/config.php')) { + break; + } + + try { + Minz_Session::_param('currentUser', $username); + Minz_Configuration::register('user', + join_path(USERS_PATH, $username, 'config.php'), + join_path(USERS_PATH, '_', 'config.default.php')); + FreshRSS_Context::init(); + if ($feedController->actualizeAction($simplePie) > 0) { + $nb++; + } + } catch (Exception $e) { + logMe($e->getMessage()); + } +} + +$simplePie->__destruct(); +unset($simplePie); + +if ($nb === 0) { + header('HTTP/1.1 410 Gone'); + logMe('Nobody is subscribed to this feed anymore after all!: ' . $self); + die('Nobody is subscribed to this feed anymore after all!'); +} + +logMe($self . ' done: ' . $nb); +exit('Done: ' . $nb . "\n"); -- cgit v1.2.3 From 27d2b88a19345dfc665dc086d3c2b2e4547e1b7f Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 23 May 2015 02:23:38 +0200 Subject: Minz getBaseUrl correction and RSS template bug https://github.com/FreshRSS/FreshRSS/issues/848 Corrections in Minz (HTTP_HOST was not sanitized, getURI() was never used and not working anyway with absolute base_url) $this->url was not defined in rss.phtml --- app/Controllers/indexController.php | 1 + constants.php | 3 ++- lib/Minz/Request.php | 46 +++++++++++-------------------------- lib/Minz/Url.php | 10 +------- 4 files changed, 18 insertions(+), 42 deletions(-) (limited to 'lib') diff --git a/app/Controllers/indexController.php b/app/Controllers/indexController.php index c1aaca53f..baaf99065 100755 --- a/app/Controllers/indexController.php +++ b/app/Controllers/indexController.php @@ -137,6 +137,7 @@ class FreshRSS_index_Controller extends Minz_ActionController { } // No layout for RSS output. + $this->view->url = empty($_SERVER['QUERY_STRING']) ? '' : '?' . $_SERVER['QUERY_STRING']; $this->view->rss_title = FreshRSS_Context::$name . ' | ' . Minz_View::title(); $this->view->_useLayout(false); header('Content-Type: application/rss+xml; charset=utf-8'); diff --git a/constants.php b/constants.php index b20bf0710..d32fdfa9b 100644 --- a/constants.php +++ b/constants.php @@ -11,7 +11,8 @@ define('PHP_COMPRESSION', false); define('FRESHRSS_PATH', dirname(__FILE__)); define('PUBLIC_PATH', FRESHRSS_PATH . '/p'); - define('INDEX_PATH', PUBLIC_PATH . '/i'); + define('PUBLIC_TO_INDEX_PATH', '/i'); + define('INDEX_PATH', PUBLIC_PATH . PUBLIC_TO_INDEX_PATH); define('PUBLIC_RELATIVE', '..'); define('DATA_PATH', FRESHRSS_PATH . '/data'); diff --git a/lib/Minz/Request.php b/lib/Minz/Request.php index 6db2e9c7a..b9eda82a5 100644 --- a/lib/Minz/Request.php +++ b/lib/Minz/Request.php @@ -84,45 +84,27 @@ class Minz_Request { self::magicQuotesOff(); } - /** - * Retourn le nom de domaine du site - */ - public static function getDomainName() { - return $_SERVER['HTTP_HOST']; - } - /** * Détermine la base de l'url * @return la base de l'url */ - public static function getBaseUrl() { + public static function getBaseUrl($baseUrlSuffix = '') { $conf = Minz_Configuration::get('system'); - $defaultBaseUrl = $conf->base_url; - if (!empty($defaultBaseUrl)) { - return $defaultBaseUrl; - } elseif (isset($_SERVER['REQUEST_URI'])) { - return dirname($_SERVER['REQUEST_URI']) . '/'; - } else { - return '/'; - } - } - - /** - * Récupère l'URI de la requête - * @return l'URI - */ - public static function getURI() { - if (isset($_SERVER['REQUEST_URI'])) { - $base_url = self::getBaseUrl(); - $uri = $_SERVER['REQUEST_URI']; - - $len_base_url = strlen($base_url); - $real_uri = substr($uri, $len_base_url); + $url = $conf->base_url; + if ($url == '' || !preg_match('%^https?://%i', $url)) { + $url = 'http'; + $host = empty($_SERVER['HTTP_HOST']) ? $_SERVER['SERVER_NAME'] : $_SERVER['HTTP_HOST']; + $port = empty($_SERVER['SERVER_PORT']) ? 80 : $_SERVER['SERVER_PORT']; + if (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on') { + $url .= 's://' . $host . ($port == 443 ? '' : ':' . $port); + } else { + $url .= '://' . $host . ($port == 80 ? '' : ':' . $port); + } + $url .= isset($_SERVER['REQUEST_URI']) ? dirname($_SERVER['REQUEST_URI']) : ''; } else { - $real_uri = ''; + $url = rtrim($url, '/\\') . $baseUrlSuffix; } - - return $real_uri; + return filter_var($url . '/', FILTER_SANITIZE_URL); } /** diff --git a/lib/Minz/Url.php b/lib/Minz/Url.php index af555a277..a47d8f1a6 100644 --- a/lib/Minz/Url.php +++ b/lib/Minz/Url.php @@ -10,7 +10,6 @@ class Minz_Url { * $url['c'] = controller * $url['a'] = action * $url['params'] = tableau des paramètres supplémentaires - * $url['protocol'] = protocole à utiliser (http par défaut) * ou comme une chaîne de caractère * @param $encodage pour indiquer comment encoder les & (& ou & pour html) * @return l'url formatée @@ -25,14 +24,7 @@ class Minz_Url { $url_string = ''; if ($absolute) { - if ($isArray && isset ($url['protocol'])) { - $protocol = $url['protocol']; - } elseif (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on') { - $protocol = 'https:'; - } else { - $protocol = 'http:'; - } - $url_string = $protocol . '//' . Minz_Request::getDomainName () . Minz_Request::getBaseUrl (); + $url_string = Minz_Request::getBaseUrl(PUBLIC_TO_INDEX_PATH); } else { $url_string = $isArray ? '.' : PUBLIC_RELATIVE; } -- cgit v1.2.3 From a5a24108ef923c47422f6b10db57863739a996cd Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 30 May 2015 19:25:20 +0200 Subject: Minz URL encoding bug See also https://github.com/FreshRSS/FreshRSS/pull/849 --- lib/Minz/Url.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/Minz/Url.php b/lib/Minz/Url.php index a47d8f1a6..879077d0f 100644 --- a/lib/Minz/Url.php +++ b/lib/Minz/Url.php @@ -18,7 +18,7 @@ class Minz_Url { $isArray = is_array($url); if ($isArray) { - $url = self::checkUrl ($url); + $url = self::checkUrl($url); } $url_string = ''; @@ -30,9 +30,9 @@ class Minz_Url { } if ($isArray) { - $url_string .= self::printUri ($url, $encodage); + $url_string .= self::printUri($url, $encodage); } else { - $url_string .= $url; + $url_string = Minz_Helper::htmlspecialchars_utf8($url_string . $url); } return $url_string; -- cgit v1.2.3 From 568ab2313d5a922e06d7f0c33b3e39dd152d43db Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 3 Jun 2015 22:07:25 +0200 Subject: PDO options for e.g. SSL https://github.com/FreshRSS/FreshRSS/issues/868 --- data/config.default.php | 3 +++ lib/Minz/ModelPdo.php | 10 ++++------ 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/data/config.default.php b/data/config.default.php index 80d331df7..7c179c8a0 100644 --- a/data/config.default.php +++ b/data/config.default.php @@ -99,6 +99,9 @@ return array( # MySQL table prefix. 'prefix' => '', + 'pdo_options' => array( + ), + ), # List of enabled FreshRSS extensions. diff --git a/lib/Minz/ModelPdo.php b/lib/Minz/ModelPdo.php index 3e8ec1f43..e82c1f30a 100644 --- a/lib/Minz/ModelPdo.php +++ b/lib/Minz/ModelPdo.php @@ -53,21 +53,19 @@ class Minz_ModelPdo { $this->current_user = $currentUser; self::$sharedCurrentUser = $currentUser; + $driver_options = is_array($conf->db['pdo_options']) ? $conf->db['pdo_options'] : array(); + try { $type = $db['type']; if ($type === 'mysql') { $string = 'mysql:host=' . $db['host'] . ';dbname=' . $db['base'] . ';charset=utf8'; - $driver_options = array( - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8', - ); + $driver_options[PDO::MYSQL_ATTR_INIT_COMMAND] = 'SET NAMES utf8'; $this->prefix = $db['prefix'] . $currentUser . '_'; } elseif ($type === 'sqlite') { $string = 'sqlite:' . join_path(DATA_PATH, 'users', $currentUser, 'db.sqlite'); - $driver_options = array( - //PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - ); + //$driver_options[PDO::ATTR_ERRMODE] = PDO::ERRMODE_EXCEPTION; $this->prefix = ''; } else { throw new Minz_PDOConnectionException( -- cgit v1.2.3 From 9b512b82e2e65c6ca18199ed328de2645a7bc84e Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 3 Jun 2015 22:17:40 +0200 Subject: PDO config isset https://github.com/FreshRSS/FreshRSS/issues/868 --- lib/Minz/ModelPdo.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Minz/ModelPdo.php b/lib/Minz/ModelPdo.php index e82c1f30a..25999f02b 100644 --- a/lib/Minz/ModelPdo.php +++ b/lib/Minz/ModelPdo.php @@ -53,7 +53,7 @@ class Minz_ModelPdo { $this->current_user = $currentUser; self::$sharedCurrentUser = $currentUser; - $driver_options = is_array($conf->db['pdo_options']) ? $conf->db['pdo_options'] : array(); + $driver_options = isset($conf->db['pdo_options']) && is_array($conf->db['pdo_options']) ? $conf->db['pdo_options'] : array(); try { $type = $db['type']; -- cgit v1.2.3 From da5033859ba990ef1f5f3dfbc1f5d7a2f4d4d6f8 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 6 Jun 2015 15:10:37 +0200 Subject: Force HTML content-type with charset Force UTF-8 https://github.com/FreshRSS/FreshRSS/issues/870 --- lib/Minz/View.php | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/Minz/View.php b/lib/Minz/View.php index ff5cce4a5..8c5230ab6 100644 --- a/lib/Minz/View.php +++ b/lib/Minz/View.php @@ -91,6 +91,7 @@ class Minz_View { * Construit le layout */ public function buildLayout () { + header('Content-Type: text/html; charset=UTF-8'); $this->includeFile(self::LAYOUT_PATH_NAME . self::LAYOUT_FILENAME); } -- cgit v1.2.3 From c1398afe8c8d220ace5fc4ccac02953adddf18c5 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 4 Jul 2015 14:21:24 +0200 Subject: Full content img data-src https://github.com/FreshRSS/FreshRSS/issues/877 Quick fix while waiting for a new library to retrieve articles --- lib/lib_rss.php | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/lib_rss.php b/lib/lib_rss.php index 191a58f35..c51b4e6de 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -195,17 +195,27 @@ function sanitizeHTML($data, $base = '') { /* permet de récupérer le contenu d'un article pour un flux qui n'est pas complet */ function get_content_by_parsing ($url, $path) { - require_once (LIB_PATH . '/lib_phpQuery.php'); + require_once(LIB_PATH . '/lib_phpQuery.php'); Minz_Log::notice('FreshRSS GET ' . SimplePie_Misc::url_remove_credentials($url)); - $html = file_get_contents ($url); + $html = file_get_contents($url); if ($html) { - $doc = phpQuery::newDocument ($html); - $content = $doc->find ($path); + $doc = phpQuery::newDocument($html); + $content = $doc->find($path); + + foreach (pq('img[data-src]') as $img) { + $imgP = pq($img); + $dataSrc = $imgP->attr('data-src'); + if (strlen($dataSrc) > 4) { + $imgP->attr('src', $dataSrc); + $imgP->removeAttr('data-src'); + } + } + return sanitizeHTML($content->__toString(), $url); } else { - throw new Exception (); + throw new Exception(); } } -- cgit v1.2.3 From cb114a341b904799b73e8e7ff1ebeab39d3c21ab Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 12 Jul 2015 00:35:31 +0200 Subject: Merge some pull requests from upstream silent Warning cast with PHP 5.3.3 (RHEL-6) https://github.com/simplepie/simplepie/pull/258 Use rss:pubDate over atom:updated when determining the posting date https://github.com/simplepie/simplepie/pull/288 Update Item.php: add support for media thumbnails https://github.com/simplepie/simplepie/pull/340 Remove PHP Notices https://github.com/simplepie/simplepie/pull/353 And removed "FreshRSS" annotations for pull requests submitted upstream Fixed a typo in Date.php (wrong number for September) https://github.com/simplepie/simplepie/pull/374 Remove duplicate case https://github.com/simplepie/simplepie/pull/406 Added set_curl_options method to allow custom options https://github.com/simplepie/simplepie/pull/407 Improved error message https://github.com/simplepie/simplepie/pull/412 --- lib/SimplePie/SimplePie.php | 149 ++++++++++++----------- lib/SimplePie/SimplePie/Cache/File.php | 12 +- lib/SimplePie/SimplePie/Decode/HTML/Entities.php | 1 - lib/SimplePie/SimplePie/File.php | 10 +- lib/SimplePie/SimplePie/Item.php | 42 +++++-- lib/SimplePie/SimplePie/Locator.php | 2 +- lib/SimplePie/SimplePie/Misc.php | 8 +- lib/SimplePie/SimplePie/Parse/Date.php | 6 +- lib/SimplePie/SimplePie/Registry.php | 4 +- lib/SimplePie/SimplePie/Sanitize.php | 4 +- 10 files changed, 133 insertions(+), 105 deletions(-) (limited to 'lib') diff --git a/lib/SimplePie/SimplePie.php b/lib/SimplePie/SimplePie.php index 54f4c5770..6c0962a9f 100644 --- a/lib/SimplePie/SimplePie.php +++ b/lib/SimplePie/SimplePie.php @@ -456,7 +456,7 @@ class SimplePie * @see SimplePie::subscribe_url() * @access private */ - public $permanent_url = null; //FreshRSS + public $permanent_url = null; /** * @var object Instance of SimplePie_File to use as a feed @@ -479,6 +479,13 @@ class SimplePie */ public $timeout = 10; + /** + * @var array Custom curl options + * @see SimplePie::set_curl_options() + * @access private + */ + public $curl_options = array(); + /** * @var bool Forces fsockopen() to be used for remote files instead * of cURL, even if a new enough version is installed @@ -754,7 +761,7 @@ class SimplePie else { $this->feed_url = $this->registry->call('Misc', 'fix_protocol', array($url, 1)); - $this->permanent_url = $this->feed_url; //FreshRSS + $this->permanent_url = $this->feed_url; } } @@ -769,7 +776,7 @@ class SimplePie if ($file instanceof SimplePie_File) { $this->feed_url = $file->url; - $this->permanent_url = $this->feed_url; //FreshRSS + $this->permanent_url = $this->feed_url; $this->file =& $file; return true; } @@ -807,6 +814,19 @@ class SimplePie { $this->timeout = (int) $timeout; } + + /** + * Set custom curl options + * + * This allows you to change default curl options + * + * @since 1.0 Beta 3 + * @param array $curl_options Curl options to add to default settings + */ + public function set_curl_options(array $curl_options = array()) + { + $this->curl_options = $curl_options; + } /** * Force SimplePie to use fsockopen() instead of cURL @@ -1251,7 +1271,7 @@ class SimplePie $this->enable_exceptions = $enable; } - function cleanMd5($rss) //FreshRSS + function cleanMd5($rss) { return md5(preg_replace(array('#<(lastBuildDate|pubDate|updated|feedDate|dc:date|slash:comments)>[^<]+#', '##s'), '', $rss)); } @@ -1297,7 +1317,7 @@ class SimplePie // Pass whatever was set with config options over to the sanitizer. // Pass the classes in for legacy support; new classes should use the registry instead $this->sanitize->pass_cache_data($this->cache, $this->cache_location, $this->cache_name_function, $this->registry->get_class('Cache')); - $this->sanitize->pass_file_data($this->registry->get_class('File'), $this->timeout, $this->useragent, $this->force_fsockopen); + $this->sanitize->pass_file_data($this->registry->get_class('File'), $this->timeout, $this->useragent, $this->force_fsockopen, $this->curl_options); if (!empty($this->multifeed_url)) { @@ -1342,7 +1362,7 @@ class SimplePie // Fetch the data via SimplePie_File into $this->raw_data if (($fetched = $this->fetch_data($cache)) === true) { - return $this->data['mtime']; //FreshRSS + return $this->data['mtime']; } elseif ($fetched === false) { return false; @@ -1350,7 +1370,7 @@ class SimplePie list($headers, $sniffed) = $fetched; - if (isset($this->data['md5'])) //FreshRSS + if (isset($this->data['md5'])) { $md5 = $this->data['md5']; } @@ -1435,8 +1455,8 @@ class SimplePie $this->data['headers'] = $headers; } $this->data['build'] = SIMPLEPIE_BUILD; - $this->data['mtime'] = time(); //FreshRSS - $this->data['md5'] = empty($md5) ? $this->cleanMd5($this->raw_data) : $md5; //FreshRSS + $this->data['mtime'] = time(); + $this->data['md5'] = empty($md5) ? $this->cleanMd5($this->raw_data) : $md5; // Cache the file if caching is enabled if ($cache && !$cache->save($this)) @@ -1451,7 +1471,7 @@ class SimplePie if (isset($parser)) { // We have an error, just set SimplePie_Misc::error to it and quit - $this->error = sprintf('This XML document is invalid, likely due to invalid characters. XML error: %s at line %d, column %d', $parser->get_error_string(), $parser->get_current_line(), $parser->get_current_column()); + $this->error = sprintf('This XML document is invalid, likely due to invalid characters. XML error: %s at line %d, column %d, encoding %s, URL: %s', $parser->get_error_string(), $parser->get_current_line(), $parser->get_current_column(), $encoding, $this->feed_url); } else { @@ -1477,7 +1497,7 @@ class SimplePie { // Load the Cache $this->data = $cache->load(); - if ($cache->mtime() + $this->cache_duration > time()) //FreshRSS + if ($cache->mtime() + $this->cache_duration > time()) { $this->raw_data = false; return true; // If the cache is still valid, just return true @@ -1514,71 +1534,58 @@ class SimplePie } } // Check if the cache has been updated - else //if ($cache->mtime() + $this->cache_duration < time()) //FreshRSS removed + else { - // If we have last-modified and/or etag set - //if (isset($this->data['headers']['last-modified']) || isset($this->data['headers']['etag'])) //FreshRSS removed + $headers = array( + 'Accept' => 'application/atom+xml, application/rss+xml, application/rdf+xml;q=0.9, application/xml;q=0.8, text/xml;q=0.8, text/html;q=0.7, unknown/unknown;q=0.1, application/unknown;q=0.1, */*;q=0.1', + ); + if (isset($this->data['headers']['last-modified'])) { - $headers = array( - 'Accept' => 'application/atom+xml, application/rss+xml, application/rdf+xml;q=0.9, application/xml;q=0.8, text/xml;q=0.8, text/html;q=0.7, unknown/unknown;q=0.1, application/unknown;q=0.1, */*;q=0.1', - ); - if (isset($this->data['headers']['last-modified'])) - { - $headers['if-modified-since'] = $this->data['headers']['last-modified']; - } - if (isset($this->data['headers']['etag'])) - { - $headers['if-none-match'] = $this->data['headers']['etag']; - } + $headers['if-modified-since'] = $this->data['headers']['last-modified']; + } + if (isset($this->data['headers']['etag'])) + { + $headers['if-none-match'] = $this->data['headers']['etag']; + } - $file = $this->registry->create('File', array($this->feed_url, $this->timeout, 5, $headers, $this->useragent, $this->force_fsockopen)); //FreshRSS + $file = $this->registry->create('File', array($this->feed_url, $this->timeout, 5, $headers, $this->useragent, $this->force_fsockopen, $this->curl_options)); - if ($file->success) + if ($file->success) + { + if ($file->status_code === 304) { - if ($file->status_code === 304) - { - $cache->touch(); - return true; - } + $cache->touch(); + return true; } - else + } + else + { + $cache->touch(); + $this->error = $file->error; + return !empty($this->data); + } + + $md5 = $this->cleanMd5($file->body); + if ($this->data['md5'] === $md5) { + if ($this->syslog_enabled) { - $cache->touch(); //FreshRSS - $this->error = $file->error; //FreshRSS - return !empty($this->data); //FreshRSS - //unset($file); //FreshRSS removed + syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . SimplePie_Misc::url_remove_credentials($this->feed_url)); } - } - { //FreshRSS - $md5 = $this->cleanMd5($file->body); - if ($this->data['md5'] === $md5) { - if ($this->syslog_enabled) - { - syslog(LOG_DEBUG, 'SimplePie MD5 cache match for ' . SimplePie_Misc::url_remove_credentials($this->feed_url)); - } - $cache->touch(); - return true; //Content unchanged even though server did not send a 304 - } else { - if ($this->syslog_enabled) - { - syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . SimplePie_Misc::url_remove_credentials($this->feed_url)); - } - $this->data['md5'] = $md5; + $cache->touch(); + return true; //Content unchanged even though server did not send a 304 + } else { + if ($this->syslog_enabled) + { + syslog(LOG_DEBUG, 'SimplePie MD5 cache no match for ' . SimplePie_Misc::url_remove_credentials($this->feed_url)); } + $this->data['md5'] = $md5; } } - //// If the cache is still valid, just return true - //else //FreshRSS removed - //{ - // $this->raw_data = false; - // return true; - //} - } - // If the cache is empty, delete it + } + // If the cache is empty else { - //$cache->unlink(); //FreshRSS removed - $cache->touch(); //FreshRSS + $cache->touch(); //To keep the date/time of the last tentative update $this->data = array(); } } @@ -1594,7 +1601,7 @@ class SimplePie $headers = array( 'Accept' => 'application/atom+xml, application/rss+xml, application/rdf+xml;q=0.9, application/xml;q=0.8, text/xml;q=0.8, text/html;q=0.7, unknown/unknown;q=0.1, application/unknown;q=0.1, */*;q=0.1', ); - $file = $this->registry->create('File', array($this->feed_url, $this->timeout, 5, $headers, $this->useragent, $this->force_fsockopen)); + $file = $this->registry->create('File', array($this->feed_url, $this->timeout, 5, $headers, $this->useragent, $this->force_fsockopen, $this->curl_options)); } } // If the file connection has an error, set SimplePie::error to that and quit @@ -1611,15 +1618,15 @@ class SimplePie if (!$locate->is_feed($file)) { - $copyStatusCode = $file->status_code; //FreshRSS - $copyContentType = $file->headers['content-type']; //FreshRSS + $copyStatusCode = $file->status_code; + $copyContentType = $file->headers['content-type']; // We need to unset this so that if SimplePie::set_file() has been called that object is untouched unset($file); try { if (!($file = $locate->find($this->autodiscovery, $this->all_discovered_feeds))) { - $this->error = "A feed could not be found at `$this->feed_url`; the status code is `$copyStatusCode` and content-type is `$copyContentType`"; //FreshRSS + $this->error = "A feed could not be found at `$this->feed_url`; the status code is `$copyStatusCode` and content-type is `$copyContentType`"; $this->registry->call('Misc', 'error', array($this->error, E_USER_NOTICE, __FILE__, __LINE__)); return false; } @@ -1634,8 +1641,8 @@ class SimplePie if ($cache) { $this->data = array('url' => $this->feed_url, 'feed_url' => $file->url, 'build' => SIMPLEPIE_BUILD); - $this->data['mtime'] = time(); //FreshRSS - $this->data['md5'] = empty($md5) ? $this->cleanMd5($file->body) : $md5; //FreshRSS + $this->data['mtime'] = time(); + $this->data['md5'] = empty($md5) ? $this->cleanMd5($file->body) : $md5; if (!$cache->save($this)) { trigger_error("$this->cache_location is not writeable. Make sure you've set the correct relative or absolute path, and that the location is server-writable.", E_USER_WARNING); @@ -1648,7 +1655,7 @@ class SimplePie } $this->raw_data = $file->body; - $this->permanent_url = $file->permanent_url; //FreshRSS + $this->permanent_url = $file->permanent_url; $headers = $file->headers; $sniffer = $this->registry->create('Content_Type_Sniffer', array(&$file)); $sniffed = $sniffer->get_type(); @@ -1852,7 +1859,7 @@ class SimplePie */ public function subscribe_url($permanent = false) { - if ($permanent) //FreshRSS + if ($permanent) { if ($this->permanent_url !== null) { diff --git a/lib/SimplePie/SimplePie/Cache/File.php b/lib/SimplePie/SimplePie/Cache/File.php index cb4b528c4..72e75a4b6 100644 --- a/lib/SimplePie/SimplePie/Cache/File.php +++ b/lib/SimplePie/SimplePie/Cache/File.php @@ -136,11 +136,7 @@ class SimplePie_Cache_File implements SimplePie_Cache_Base */ public function mtime() { - //if (file_exists($this->name)) //FreshRSS removed - { - return @filemtime($this->name); //FreshRSS - } - //return false; //FreshRSS removed + return @filemtime($this->name); } /** @@ -150,11 +146,7 @@ class SimplePie_Cache_File implements SimplePie_Cache_Base */ public function touch() { - //if (file_exists($this->name)) //FreshRSS removed - { - return @touch($this->name); //FreshRSS - } - //return false; //FreshRSS removed + return @touch($this->name); } /** diff --git a/lib/SimplePie/SimplePie/Decode/HTML/Entities.php b/lib/SimplePie/SimplePie/Decode/HTML/Entities.php index cde06c884..46b3a1dff 100644 --- a/lib/SimplePie/SimplePie/Decode/HTML/Entities.php +++ b/lib/SimplePie/SimplePie/Decode/HTML/Entities.php @@ -169,7 +169,6 @@ class SimplePie_Decode_HTML_Entities case "\x09": case "\x0A": case "\x0B": - case "\x0B": case "\x0C": case "\x20": case "\x3C": diff --git a/lib/SimplePie/SimplePie/File.php b/lib/SimplePie/SimplePie/File.php index 1f9e3d502..643648ddb 100644 --- a/lib/SimplePie/SimplePie/File.php +++ b/lib/SimplePie/SimplePie/File.php @@ -66,7 +66,7 @@ class SimplePie_File var $method = SIMPLEPIE_FILE_SOURCE_NONE; var $permanent_url; //FreshRSS - public function __construct($url, $timeout = 10, $redirects = 5, $headers = null, $useragent = null, $force_fsockopen = false, $syslog_enabled = SIMPLEPIE_SYSLOG) + public function __construct($url, $timeout = 10, $redirects = 5, $headers = null, $useragent = null, $force_fsockopen = false, $curl_options = array(), $syslog_enabled = SIMPLEPIE_SYSLOG) { if (class_exists('idna_convert')) { @@ -75,7 +75,7 @@ class SimplePie_File $url = SimplePie_Misc::compress_parse_url($parsed['scheme'], $idn->encode($parsed['authority']), $parsed['path'], $parsed['query'], $parsed['fragment']); } $this->url = $url; - $this->permanent_url = $url; //FreshRSS + $this->permanent_url = $url; $this->useragent = $useragent; if (preg_match('/^http(s)?:\/\//i', $url)) { @@ -119,6 +119,10 @@ class SimplePie_File curl_setopt($fp, CURLOPT_FOLLOWLOCATION, 1); curl_setopt($fp, CURLOPT_MAXREDIRS, $redirects); } + foreach ($curl_options as $curl_param => $curl_value) + { + curl_setopt($fp, $curl_param, $curl_value); + } $this->headers = curl_exec($fp); if (curl_errno($fp) === 23 || curl_errno($fp) === 61) @@ -149,7 +153,7 @@ class SimplePie_File $location = SimplePie_Misc::absolutize_url($this->headers['location'], $url); $previousStatusCode = $this->status_code; $this->__construct($location, $timeout, $redirects, $headers, $useragent, $force_fsockopen); - $this->permanent_url = ($previousStatusCode == 301) ? $location : $url; //FreshRSS + $this->permanent_url = ($previousStatusCode == 301) ? $location : $url; return; } } diff --git a/lib/SimplePie/SimplePie/Item.php b/lib/SimplePie/SimplePie/Item.php index 7bd96c15f..27e93456e 100644 --- a/lib/SimplePie/SimplePie/Item.php +++ b/lib/SimplePie/SimplePie/Item.php @@ -406,6 +406,30 @@ class SimplePie_Item return null; } } + + /** + * Get the media:thumbnail of the item + * + * Uses `` + * + * + * @return array|null + */ + public function get_thumbnail() + { + if (!isset($this->data['thumbnail'])) + { + if ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_MEDIARSS, 'thumbnail')) + { + $this->data['thumbnail'] = $return[0]['attribs']['']; + } + else + { + $this->data['thumbnail'] = null; + } + } + return $this->data['thumbnail']; + } /** * Get a category for the item @@ -738,31 +762,31 @@ class SimplePie_Item { $this->data['date']['raw'] = $return[0]['data']; } - elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_10, 'updated')) + elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_RSS_20, 'pubDate')) { $this->data['date']['raw'] = $return[0]['data']; } - elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_03, 'issued')) + elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_DC_11, 'date')) { $this->data['date']['raw'] = $return[0]['data']; } - elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_03, 'created')) + elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_DC_10, 'date')) { $this->data['date']['raw'] = $return[0]['data']; } - elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_03, 'modified')) + elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_10, 'updated')) { $this->data['date']['raw'] = $return[0]['data']; } - elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_RSS_20, 'pubDate')) + elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_03, 'issued')) { $this->data['date']['raw'] = $return[0]['data']; } - elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_DC_11, 'date')) + elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_03, 'created')) { $this->data['date']['raw'] = $return[0]['data']; } - elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_DC_10, 'date')) + elseif ($return = $this->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_03, 'modified')) { $this->data['date']['raw'] = $return[0]['data']; } @@ -2733,7 +2757,9 @@ class SimplePie_Item { foreach ($content['child'][SIMPLEPIE_NAMESPACE_MEDIARSS]['thumbnail'] as $thumbnail) { - $thumbnails[] = $this->sanitize($thumbnail['attribs']['']['url'], SIMPLEPIE_CONSTRUCT_IRI); + if (isset($thumbnail['attribs']['']['url'])) { + $thumbnails[] = $this->sanitize($thumbnail['attribs']['']['url'], SIMPLEPIE_CONSTRUCT_IRI); + } } if (is_array($thumbnails)) { diff --git a/lib/SimplePie/SimplePie/Locator.php b/lib/SimplePie/SimplePie/Locator.php index 4e5f7c1ca..ba4a843b0 100644 --- a/lib/SimplePie/SimplePie/Locator.php +++ b/lib/SimplePie/SimplePie/Locator.php @@ -148,7 +148,7 @@ class SimplePie_Locator { $sniffer = $this->registry->create('Content_Type_Sniffer', array($file)); $sniffed = $sniffer->get_type(); - if (in_array($sniffed, array('application/rss+xml', 'application/rdf+xml', 'text/rdf', 'application/atom+xml', 'text/xml', 'application/xml', 'application/x-rss+xml'))) //FreshRSS + if (in_array($sniffed, array('application/rss+xml', 'application/rdf+xml', 'text/rdf', 'application/atom+xml', 'text/xml', 'application/xml', 'application/x-rss+xml'))) { return true; } diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index 956a284cb..9e7ac4fa8 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -79,8 +79,8 @@ class SimplePie_Misc public static function absolutize_url($relative, $base) { - if (substr($relative, 0, 2) === '//') //FreshRSS: disable absolutize_url for "//www.example.net" which will pick HTTP or HTTPS automatically - { + if (substr($relative, 0, 2) === '//') + {//Allow protocol-relative URLs "//www.example.net" which will pick HTTP or HTTPS automatically return $relative; } $iri = SimplePie_IRI::absolutize(new SimplePie_IRI($base), $relative); @@ -128,7 +128,7 @@ class SimplePie_Misc { $attribs[$j][2] = $attribs[$j][1]; } - $return[$i]['attribs'][strtolower($attribs[$j][1])]['data'] = SimplePie_Misc::entities_decode(end($attribs[$j]), 'UTF-8'); //FreshRSS + $return[$i]['attribs'][strtolower($attribs[$j][1])]['data'] = SimplePie_Misc::entities_decode(end($attribs[$j]), 'UTF-8'); } } } @@ -142,7 +142,7 @@ class SimplePie_Misc foreach ($element['attribs'] as $key => $value) { $key = strtolower($key); - $full .= " $key=\"" . htmlspecialchars($value['data'], ENT_COMPAT, 'UTF-8') . '"'; //FreshRSS + $full .= " $key=\"" . htmlspecialchars($value['data'], ENT_COMPAT, 'UTF-8') . '"'; } if ($element['self_closing']) { diff --git a/lib/SimplePie/SimplePie/Parse/Date.php b/lib/SimplePie/SimplePie/Parse/Date.php index ba7c0703e..50bb5cffa 100644 --- a/lib/SimplePie/SimplePie/Parse/Date.php +++ b/lib/SimplePie/SimplePie/Parse/Date.php @@ -173,7 +173,7 @@ class SimplePie_Parse_Date 'aug' => 8, 'august' => 8, 'sep' => 9, - 'september' => 8, + 'september' => 9, 'oct' => 10, 'october' => 10, 'nov' => 11, @@ -331,8 +331,8 @@ class SimplePie_Parse_Date 'CCT' => 23400, 'CDT' => -18000, 'CEDT' => 7200, - 'CEST' => 7200, //FreshRSS 'CET' => 3600, + 'CEST' => 7200, 'CGST' => -7200, 'CGT' => -10800, 'CHADT' => 49500, @@ -721,7 +721,7 @@ class SimplePie_Parse_Date { $output .= substr($string, $position, $pos - $position); $position = $pos + 1; - if ($string[$pos - 1] !== '\\') + if ($pos === 0 || $string[$pos - 1] !== '\\') { $depth++; while ($depth && $position < $length) diff --git a/lib/SimplePie/SimplePie/Registry.php b/lib/SimplePie/SimplePie/Registry.php index bd9c1f535..dac55e34e 100755 --- a/lib/SimplePie/SimplePie/Registry.php +++ b/lib/SimplePie/SimplePie/Registry.php @@ -113,7 +113,7 @@ class SimplePie_Registry */ public function register($type, $class, $legacy = false) { - if (!is_subclass_of($class, $this->default[$type])) + if (!@is_subclass_of($class, $this->default[$type])) { return false; } @@ -222,4 +222,4 @@ class SimplePie_Registry $result = call_user_func_array(array($class, $method), $parameters); return $result; } -} \ No newline at end of file +} diff --git a/lib/SimplePie/SimplePie/Sanitize.php b/lib/SimplePie/SimplePie/Sanitize.php index e7c9f925f..a6863ec03 100644 --- a/lib/SimplePie/SimplePie/Sanitize.php +++ b/lib/SimplePie/SimplePie/Sanitize.php @@ -249,7 +249,7 @@ class SimplePie_Sanitize { if ($type & SIMPLEPIE_CONSTRUCT_MAYBE_HTML) { - $data = htmlspecialchars_decode($data, ENT_QUOTES); //FreshRSS + $data = htmlspecialchars_decode($data, ENT_QUOTES); if (preg_match('/(&(#(x[0-9a-fA-F]+|[0-9]+)|[a-zA-Z0-9]+)|<\/[A-Za-z][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E]*' . SIMPLEPIE_PCRE_HTML_ATTRIBUTE . '>)/', $data)) { $type |= SIMPLEPIE_CONSTRUCT_HTML; @@ -280,7 +280,7 @@ class SimplePie_Sanitize $document->loadHTML($data); restore_error_handler(); - $xpath = new DOMXPath($document); //FreshRSS + $xpath = new DOMXPath($document); // Strip comments if ($this->strip_comments) -- cgit v1.2.3 From fa432755cc9d0cc71be54d463bdfb7558d62c5f1 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 12 Jul 2015 12:16:56 +0200 Subject: Implement cURL options https://github.com/FreshRSS/FreshRSS/issues/897 Requires that https://github.com/FreshRSS/FreshRSS/pull/898 is merged first. --- CHANGELOG.md | 2 ++ data/config.default.php | 19 +++++++++++++++++-- lib/SimplePie/SimplePie/File.php | 1 - lib/lib_rss.php | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ebfd50ac..23feda577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,9 @@ * Features * Support for PubSubHubbub for instant notifications from compatible Web sites. [#312](https://github.com/FreshRSS/FreshRSS/issues/312) + * cURL options to use a proxy for retrieving feeds. [#897](https://github.com/FreshRSS/FreshRSS/issues/897) [#675](https://github.com/FreshRSS/FreshRSS/issues/675) * Security + * cURL options to verify or not SSL/TLS certificates (now enabled by default). [#897](https://github.com/FreshRSS/FreshRSS/issues/897) [#502](https://github.com/FreshRSS/FreshRSS/issues/502) * Support for SSL connection to MySQL. [#868](https://github.com/FreshRSS/FreshRSS/issues/868) * Workaround for browsers that have disabled support for `
`. [#880](https://github.com/FreshRSS/FreshRSS/issues/880) * UI diff --git a/data/config.default.php b/data/config.default.php index 97085df29..6a8300872 100644 --- a/data/config.default.php +++ b/data/config.default.php @@ -1,7 +1,7 @@ array( + # Options to disable SSL/TLS certificate check (e.g. for self-signed HTTPS) + //CURLOPT_SSL_VERIFYHOST => 0, + //CURLOPT_SSL_VERIFYPEER => false, + + # Options to use a proxy for retrieving feeds. + //CURLOPT_PROXYTYPE = CURLPROXY_HTTP, + //CURLOPT_PROXYAUTH = CURLAUTH_BASIC, + //CURLOPT_PROXY => '127.0.0.1', + //CURLOPT_PROXYPORT => 8080, + //CURLOPT_PROXYUSERPWD => 'user:password', + ), + 'db' => array( # Type of database: `sqlite` or `mysql`. diff --git a/lib/SimplePie/SimplePie/File.php b/lib/SimplePie/SimplePie/File.php index 1f9e3d502..90dac790a 100644 --- a/lib/SimplePie/SimplePie/File.php +++ b/lib/SimplePie/SimplePie/File.php @@ -113,7 +113,6 @@ class SimplePie_File curl_setopt($fp, CURLOPT_REFERER, $url); curl_setopt($fp, CURLOPT_USERAGENT, $useragent); curl_setopt($fp, CURLOPT_HTTPHEADER, $headers2); - curl_setopt($fp, CURLOPT_SSL_VERIFYPEER, false); //FreshRSS if (!ini_get('open_basedir') && !ini_get('safe_mode') && version_compare(SimplePie_Misc::get_curl_version(), '7.15.2', '>=')) { curl_setopt($fp, CURLOPT_FOLLOWLOCATION, 1); diff --git a/lib/lib_rss.php b/lib/lib_rss.php index c51b4e6de..0118e0f46 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -143,6 +143,7 @@ function customSimplePie() { $simplePie->set_cache_location(CACHE_PATH); $simplePie->set_cache_duration($limits['cache_duration']); $simplePie->set_timeout($limits['timeout']); + $simplePie->set_curl_options($system_conf->curl_options); $simplePie->strip_htmltags(array( 'base', 'blink', 'body', 'doctype', 'embed', 'font', 'form', 'frame', 'frameset', 'html', -- cgit v1.2.3 From 241086fa672226cf799a82daad364fb82272da3b Mon Sep 17 00:00:00 2001 From: Marien Fressinaud Date: Mon, 13 Jul 2015 23:11:05 +0200 Subject: Fix broken links for extension script/style files Fix https://github.com/FreshRSS/FreshRSS/issues/862 --- lib/Minz/Extension.php | 2 +- lib/Minz/Url.php | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/Minz/Extension.php b/lib/Minz/Extension.php index d7ee8fe81..78b8a2725 100644 --- a/lib/Minz/Extension.php +++ b/lib/Minz/Extension.php @@ -168,7 +168,7 @@ class Minz_Extension { $url = '/ext.php?f=' . $file_name_url . '&t=' . $type . '&' . $mtime; - return Minz_Url::display($url); + return Minz_Url::display($url, 'php'); } /** diff --git a/lib/Minz/Url.php b/lib/Minz/Url.php index 879077d0f..a2809257d 100644 --- a/lib/Minz/Url.php +++ b/lib/Minz/Url.php @@ -31,8 +31,10 @@ class Minz_Url { if ($isArray) { $url_string .= self::printUri($url, $encodage); - } else { + } elseif ($encodage === 'html') { $url_string = Minz_Helper::htmlspecialchars_utf8($url_string . $url); + } else { + $url_string .= $url; } return $url_string; -- cgit v1.2.3 From 2d22bf300a8dabcc77237f4bb56bf6532486fca2 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Fri, 17 Jul 2015 23:43:43 +0200 Subject: dirname problem https://github.com/FreshRSS/FreshRSS/issues/906 --- lib/Minz/Request.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Minz/Request.php b/lib/Minz/Request.php index b9eda82a5..67fbae126 100644 --- a/lib/Minz/Request.php +++ b/lib/Minz/Request.php @@ -100,7 +100,10 @@ class Minz_Request { } else { $url .= '://' . $host . ($port == 80 ? '' : ':' . $port); } - $url .= isset($_SERVER['REQUEST_URI']) ? dirname($_SERVER['REQUEST_URI']) : ''; + if (isset($_SERVER['REQUEST_URI'])) { + $path = $_SERVER['REQUEST_URI']; + $url .= substr($path, -1) === '/' ? substr($path, 0, -1) : dirname($path); + } } else { $url = rtrim($url, '/\\') . $baseUrlSuffix; } -- cgit v1.2.3 From 02c3546440f961018adc1e2c8e97c16f2aca18fc Mon Sep 17 00:00:00 2001 From: Marien Fressinaud Date: Wed, 22 Jul 2015 13:52:03 +0200 Subject: Registration action is handled and create a user See https://github.com/FreshRSS/FreshRSS/issues/679 --- app/Controllers/userController.php | 20 +++++++++++++++++--- app/views/auth/register.phtml | 7 +++++++ lib/lib_rss.php | 16 ++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/app/Controllers/userController.php b/app/Controllers/userController.php index c198d1328..46f4f434d 100644 --- a/app/Controllers/userController.php +++ b/app/Controllers/userController.php @@ -12,9 +12,14 @@ class FreshRSS_user_Controller extends Minz_ActionController { * This action is called before every other action in that class. It is * the common boiler plate for every action. It is triggered by the * underlying framework. + * + * @todo clean up the access condition. */ public function firstAction() { - if (!FreshRSS_Auth::hasAccess()) { + if (!FreshRSS_Auth::hasAccess() && !( + Minz_Request::actionName() === 'create' && + !max_registrations_reached() + )) { Minz_Error::error(403); } } @@ -111,11 +116,16 @@ class FreshRSS_user_Controller extends Minz_ActionController { * - new_user_name * - new_user_passwordPlain * - new_user_email + * - r (i.e. a redirection url, optional) * * @todo clean up this method. Idea: write a method to init a user with basic information. + * @todo handle r redirection in Minz_Request::forward directly? */ public function createAction() { - if (Minz_Request::isPost() && FreshRSS_Auth::hasAccess('admin')) { + if (Minz_Request::isPost() && ( + FreshRSS_Auth::hasAccess('admin') || + !max_registrations_reached() + )) { $db = FreshRSS_Context::$system_conf->db; require_once(APP_PATH . '/SQL/install.sql.' . $db['type'] . '.php'); @@ -186,7 +196,11 @@ class FreshRSS_user_Controller extends Minz_ActionController { Minz_Session::_param('notification', $notif); } - Minz_Request::forward(array('c' => 'user', 'a' => 'manage'), true); + $redirect_url = urldecode(Minz_Request::param('r', false, true)); + if (!$redirect_url) { + $redirect_url = array('c' => 'user', 'a' => 'manage'); + } + Minz_Request::forward($redirect_url, true); } /** diff --git a/app/views/auth/register.phtml b/app/views/auth/register.phtml index 31ab89d26..96c91f411 100644 --- a/app/views/auth/register.phtml +++ b/app/views/auth/register.phtml @@ -29,6 +29,13 @@
+ 'index', 'a' => 'index'), + 'php', true + )); + ?> +
diff --git a/lib/lib_rss.php b/lib/lib_rss.php index 0118e0f46..c99e2c7e8 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -266,6 +266,22 @@ function listUsers() { } +/** + * Return if the maximum number of registrations has been reached. + * + * Note a max_regstrations of 0 means there is no limit. + * + * @return true if number of users >= max registrations, false else. + */ +function max_registrations_reached() { + $system_conf = Minz_Configuration::get('system'); + $limit_registrations = $system_conf->limits['max_registrations']; + $number_accounts = count(listUsers()); + + return $limit_registrations > 0 && $number_accounts >= $limit_registrations; +} + + /** * Register and return the configuration for a given user. * -- cgit v1.2.3 From 6db09411968ff0eac722efde79628b501b8dbe5e Mon Sep 17 00:00:00 2001 From: Marien Fressinaud Date: Thu, 23 Jul 2015 10:05:32 +0200 Subject: Fix unexpected behaviour in getBaseUrl - getBaseUrl() returns info from configuration only and always append the suffix - add a guessBaseUrl() to extract base_url from $_SERVER info - fix Url::display() to take this change in consideration Fix https://github.com/FreshRSS/FreshRSS/issues/906 Use https://github.com/FreshRSS/FreshRSS/pull/910 --- lib/Minz/Request.php | 48 ++++++++++++++++++++++++++++-------------------- lib/Minz/Url.php | 3 +++ 2 files changed, 31 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/Minz/Request.php b/lib/Minz/Request.php index 67fbae126..059b4359c 100644 --- a/lib/Minz/Request.php +++ b/lib/Minz/Request.php @@ -85,29 +85,37 @@ class Minz_Request { } /** - * Détermine la base de l'url - * @return la base de l'url + * Try to guess the base URL from $_SERVER information + * + * @return the base url (e.g. http://example.com/) */ - public static function getBaseUrl($baseUrlSuffix = '') { - $conf = Minz_Configuration::get('system'); - $url = $conf->base_url; - if ($url == '' || !preg_match('%^https?://%i', $url)) { - $url = 'http'; - $host = empty($_SERVER['HTTP_HOST']) ? $_SERVER['SERVER_NAME'] : $_SERVER['HTTP_HOST']; - $port = empty($_SERVER['SERVER_PORT']) ? 80 : $_SERVER['SERVER_PORT']; - if (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on') { - $url .= 's://' . $host . ($port == 443 ? '' : ':' . $port); - } else { - $url .= '://' . $host . ($port == 80 ? '' : ':' . $port); - } - if (isset($_SERVER['REQUEST_URI'])) { - $path = $_SERVER['REQUEST_URI']; - $url .= substr($path, -1) === '/' ? substr($path, 0, -1) : dirname($path); - } + public static function guessBaseUrl() { + $url = 'http'; + $host = empty($_SERVER['HTTP_HOST']) ? $_SERVER['SERVER_NAME'] : $_SERVER['HTTP_HOST']; + $port = empty($_SERVER['SERVER_PORT']) ? 80 : $_SERVER['SERVER_PORT']; + if (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on') { + $url .= 's://' . $host . ($port == 443 ? '' : ':' . $port); } else { - $url = rtrim($url, '/\\') . $baseUrlSuffix; + $url .= '://' . $host . ($port == 80 ? '' : ':' . $port); + } + if (isset($_SERVER['REQUEST_URI'])) { + $path = $_SERVER['REQUEST_URI']; + $url .= substr($path, -1) === '/' ? substr($path, 0, -1) : dirname($path); } - return filter_var($url . '/', FILTER_SANITIZE_URL); + + return $url; + } + + /** + * Return the base_url from configuration and add a suffix if given. + * + * @param $base_url_suffix a string to add at base_url (default: empty string) + * @return the base_url with a suffix. + */ + public static function getBaseUrl($base_url_suffix = '') { + $conf = Minz_Configuration::get('system'); + $url = rtrim($conf->base_url, '/\\') . $base_url_suffix; + return filter_var($url, FILTER_SANITIZE_URL); } /** diff --git a/lib/Minz/Url.php b/lib/Minz/Url.php index a2809257d..4279b045b 100644 --- a/lib/Minz/Url.php +++ b/lib/Minz/Url.php @@ -25,6 +25,9 @@ class Minz_Url { if ($absolute) { $url_string = Minz_Request::getBaseUrl(PUBLIC_TO_INDEX_PATH); + if ($url_string === PUBLIC_TO_INDEX_PATH) { + $url_string = Minz_Request::guessBaseUrl(); + } } else { $url_string = $isArray ? '.' : PUBLIC_RELATIVE; } -- cgit v1.2.3 From 1e65fd687e030a24773e88a2e7fb173e8439a99a Mon Sep 17 00:00:00 2001 From: Marien Fressinaud Date: Thu, 23 Jul 2015 11:38:56 +0200 Subject: Use filter_var in guessBaseUrl See https://github.com/FreshRSS/FreshRSS/issues/906 See https://github.com/FreshRSS/FreshRSS/pull/915/files#r35304704 --- lib/Minz/Request.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Minz/Request.php b/lib/Minz/Request.php index 059b4359c..bf01bc26f 100644 --- a/lib/Minz/Request.php +++ b/lib/Minz/Request.php @@ -103,7 +103,7 @@ class Minz_Request { $url .= substr($path, -1) === '/' ? substr($path, 0, -1) : dirname($path); } - return $url; + return filter_var($url, FILTER_SANITIZE_URL); } /** -- cgit v1.2.3 From 339e32424fa60fc0c99a4c10890abef139444f6d Mon Sep 17 00:00:00 2001 From: Marien Fressinaud Date: Thu, 23 Jul 2015 12:38:22 +0200 Subject: Add a simple test to detect if server is public If the server is not accessible by an external server, pubsubhubbub should be disable. See https://github.com/FreshRSS/FreshRSS/issues/865 --- app/install.php | 1 + lib/lib_rss.php | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) (limited to 'lib') diff --git a/app/install.php b/app/install.php index 3930a3d41..65138a683 100644 --- a/app/install.php +++ b/app/install.php @@ -229,6 +229,7 @@ function saveStep3() { 'prefix' => $_SESSION['bd_prefix'], 'pdo_options' => array(), ), + 'enable_pubsubhubbub' => server_is_public($base_url), ); @unlink(join_path(DATA_PATH, 'config.php')); //To avoid access-rights problems diff --git a/lib/lib_rss.php b/lib/lib_rss.php index c99e2c7e8..2a23fca45 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -83,6 +83,33 @@ function checkUrl($url) { } } + +/** + * Test if a given server address is publicly accessible. + * + * Note: for the moment it tests only if address is corresponding to a + * localhost address. + * + * @param $address the address to test, can be an IP or a URL. + * @return true if server is accessible, false else. + * @todo improve test with a more valid technique (e.g. test with an external server?) + */ +function server_is_public($address) { + $host = parse_url($address, PHP_URL_HOST); + + $is_public = !in_array($host, array( + '127.0.0.1', + 'localhost', + 'localhost.localdomain', + '[::1]', + 'localhost6', + 'localhost6.localdomain6', + )); + + return $is_public; +} + + function format_number($n, $precision = 0) { // number_format does not seem to be Unicode-compatible return str_replace(' ', ' ', //Espace fine insécable -- cgit v1.2.3 From c1a44a8761778da81bde608f6d20f5decdd8b9b6 Mon Sep 17 00:00:00 2001 From: Marien Fressinaud Date: Mon, 27 Jul 2015 14:46:41 +0200 Subject: Load configuration by recursion - Remove Minz_Configuration::$data_default - Default values are loaded first in $data - $data values are replaced by values from config file Fix https://github.com/FreshRSS/FreshRSS/issues/923 --- lib/Minz/Configuration.php | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/Minz/Configuration.php b/lib/Minz/Configuration.php index ab5bb4fc2..1db8e2daf 100644 --- a/lib/Minz/Configuration.php +++ b/lib/Minz/Configuration.php @@ -84,11 +84,6 @@ class Minz_Configuration { */ private $data = array(); - /** - * The default values, an empty array by default. - */ - private $data_default = array(); - /** * An object which help to set good values in configuration. */ @@ -119,21 +114,22 @@ class Minz_Configuration { $configuration_setter = null) { $this->namespace = $namespace; $this->config_filename = $config_filename; + $this->default_filename = $default_filename; + $this->_configurationSetter($configuration_setter); + + if (!is_null($this->default_filename)) { + $this->data = self::load($this->default_filename); + } try { - $this->data = self::load($this->config_filename); + $this->data = array_replace_recursive( + $this->data, self::load($this->config_filename) + ); } catch (Minz_FileNotExistException $e) { - if (is_null($default_filename)) { + if (is_null($this->default_filename)) { throw $e; } } - - $this->default_filename = $default_filename; - if (!is_null($this->default_filename)) { - $this->data_default = self::load($this->default_filename); - } - - $this->_configurationSetter($configuration_setter); } /** @@ -160,8 +156,6 @@ class Minz_Configuration { return $this->data[$key]; } elseif (!is_null($default)) { return $default; - } elseif (isset($this->data_default[$key])) { - return $this->data_default[$key]; } else { Minz_Log::warning($key . ' does not exist in configuration'); return null; -- cgit v1.2.3 From 6dbe33c51e83bbfc1e18c5bdf764411799502a99 Mon Sep 17 00:00:00 2001 From: Marien Fressinaud Date: Wed, 29 Jul 2015 07:59:05 +0200 Subject: Don't hide errors in configuration Fix https://github.com/FreshRSS/FreshRSS/issues/920 --- lib/Minz/Configuration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Minz/Configuration.php b/lib/Minz/Configuration.php index 1db8e2daf..d695d4a53 100644 --- a/lib/Minz/Configuration.php +++ b/lib/Minz/Configuration.php @@ -39,7 +39,7 @@ class Minz_Configuration { throw new Minz_FileNotExistException($filename); } - $data = @include($filename); + $data = include($filename); if (is_array($data)) { return $data; } else { -- cgit v1.2.3 From f7190c34e1a1ea36bbc81a7dea8dcb7a39cea7cf Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Thu, 30 Jul 2015 23:42:28 +0200 Subject: Minz session cookie path bug https://github.com/FreshRSS/FreshRSS/issues/924#issuecomment-126499403 --- lib/Minz/Session.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/Minz/Session.php b/lib/Minz/Session.php index 058685ada..14a093bf7 100644 --- a/lib/Minz/Session.php +++ b/lib/Minz/Session.php @@ -65,10 +65,9 @@ class Minz_Session { * @param $l la durée de vie */ public static function keepCookie($l) { - // Get the script_name (e.g. /p/i/index.php) and keep only the path. - $cookie_dir = empty($_SERVER['SCRIPT_NAME']) ? '' : $_SERVER['SCRIPT_NAME']; - $cookie_dir = dirname($cookie_dir); - session_set_cookie_params($l, $cookie_dir, '', false, true); + // Get the script_name (e.g. /p/i/index.php) and keep only the path. + $cookie_dir = dirname(empty($_SERVER['REQUEST_URI']) ? '/' : dirname($_SERVER['REQUEST_URI'])); + session_set_cookie_params($l, $cookie_dir, '', false, false); } -- cgit v1.2.3 From 59daed3d4eca6bf6260a8dc422c54f470895ac63 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Fri, 31 Jul 2015 00:12:55 +0200 Subject: Minz slight change in session cookie path https://github.com/FreshRSS/FreshRSS/issues/924#issuecomment-126499403 --- lib/Minz/Session.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Minz/Session.php b/lib/Minz/Session.php index 14a093bf7..705aae2ec 100644 --- a/lib/Minz/Session.php +++ b/lib/Minz/Session.php @@ -66,7 +66,10 @@ class Minz_Session { */ public static function keepCookie($l) { // Get the script_name (e.g. /p/i/index.php) and keep only the path. - $cookie_dir = dirname(empty($_SERVER['REQUEST_URI']) ? '/' : dirname($_SERVER['REQUEST_URI'])); + $cookie_dir = empty($_SERVER['REQUEST_URI']) ? '/' : $_SERVER['REQUEST_URI']; + if (substr($cookie_dir, -1) !== '/') { + $cookie_dir = dirname($cookie_dir) . '/'; + } session_set_cookie_params($l, $cookie_dir, '', false, false); } -- cgit v1.2.3 From 760ec5f223c4a18c8e8c8f3ecdf6b7140aa70611 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Fri, 31 Jul 2015 00:17:32 +0200 Subject: Whitespace --- lib/Minz/Session.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/Minz/Session.php b/lib/Minz/Session.php index 705aae2ec..de671f173 100644 --- a/lib/Minz/Session.php +++ b/lib/Minz/Session.php @@ -65,12 +65,12 @@ class Minz_Session { * @param $l la durée de vie */ public static function keepCookie($l) { - // Get the script_name (e.g. /p/i/index.php) and keep only the path. - $cookie_dir = empty($_SERVER['REQUEST_URI']) ? '/' : $_SERVER['REQUEST_URI']; - if (substr($cookie_dir, -1) !== '/') { - $cookie_dir = dirname($cookie_dir) . '/'; - } - session_set_cookie_params($l, $cookie_dir, '', false, false); + // Get the script_name (e.g. /p/i/index.php) and keep only the path. + $cookie_dir = empty($_SERVER['REQUEST_URI']) ? '/' : $_SERVER['REQUEST_URI']; + if (substr($cookie_dir, -1) !== '/') { + $cookie_dir = dirname($cookie_dir) . '/'; + } + session_set_cookie_params($l, $cookie_dir, '', false, false); } -- cgit v1.2.3 From 189e790f32d4e389cf1dc6da669a579717fff436 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Fri, 31 Jul 2015 11:26:57 +0200 Subject: Minz cookie session httpOnly https://github.com/FreshRSS/FreshRSS/issues/924 https://github.com/FreshRSS/FreshRSS/pull/936/files#r35948311 --- lib/Minz/Session.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/Minz/Session.php b/lib/Minz/Session.php index de671f173..057e7746a 100644 --- a/lib/Minz/Session.php +++ b/lib/Minz/Session.php @@ -70,7 +70,7 @@ class Minz_Session { if (substr($cookie_dir, -1) !== '/') { $cookie_dir = dirname($cookie_dir) . '/'; } - session_set_cookie_params($l, $cookie_dir, '', false, false); + session_set_cookie_params($l, $cookie_dir, '', false, true); } -- cgit v1.2.3 From bfae186e3668af8155b77abb5c4b3dc5f151e14c Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 25 Oct 2015 01:27:15 +0200 Subject: Use HTTP_X_FORWARDED_ https://github.com/FreshRSS/FreshRSS/issues/975 --- lib/Minz/Request.php | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/Minz/Request.php b/lib/Minz/Request.php index bf01bc26f..effb9943c 100644 --- a/lib/Minz/Request.php +++ b/lib/Minz/Request.php @@ -91,9 +91,30 @@ class Minz_Request { */ public static function guessBaseUrl() { $url = 'http'; - $host = empty($_SERVER['HTTP_HOST']) ? $_SERVER['SERVER_NAME'] : $_SERVER['HTTP_HOST']; - $port = empty($_SERVER['SERVER_PORT']) ? 80 : $_SERVER['SERVER_PORT']; - if (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on') { + + if (isset($_SERVER['HTTP_X_FORWARDED_PROTO'])) { + $https = strtolower($_SERVER['HTTP_X_FORWARDED_PROTO']) === 'https'; + } else { + $https = isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on'; + } + + if (!empty($_SERVER['HTTP_HOST'])) { + $host = $_SERVER['HTTP_HOST']; + } elseif (!empty($_SERVER['SERVER_NAME'])) { + $host = $_SERVER['SERVER_NAME']; + } else { + $host = 'localhost'; + } + + if (!empty($_SERVER['HTTP_X_FORWARDED_PORT'])) { + $port = intval($_SERVER['HTTP_X_FORWARDED_PORT']); + } elseif (!empty($_SERVER['SERVER_PORT'])) { + $port = intval($_SERVER['SERVER_PORT']); + } else { + $port = $https ? 443 : 80; + } + + if ($https) { $url .= 's://' . $host . ($port == 443 ? '' : ':' . $port); } else { $url .= '://' . $host . ($port == 80 ? '' : ':' . $port); -- cgit v1.2.3