From 22b41f3bfcbd5a54d59789c2cebfda6dc23b7dde Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 26 Mar 2017 00:01:11 +0100 Subject: Candidate implementation of defered insertion https://github.com/FreshRSS/FreshRSS/issues/530 --- p/scripts/main.js | 1 + 1 file changed, 1 insertion(+) (limited to 'p') diff --git a/p/scripts/main.js b/p/scripts/main.js index 5dbb95c91..c2f60bf7f 100644 --- a/p/scripts/main.js +++ b/p/scripts/main.js @@ -809,6 +809,7 @@ function updateFeed(feeds, feeds_count) { url: feed.url, data : { _csrf: context.csrf, + isLastFeed: feeds.length <= 0 ? 1 : 0, }, }).always(function (data) { feed_processed++; -- cgit v1.2.3 From 0d4c26c673d548a1367a0f96f49546ca27619d90 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 29 Mar 2017 21:42:40 +0200 Subject: Add manual commit & refresh cache to deferred insertion --- app/Controllers/feedController.php | 15 +++++++++++++-- p/scripts/main.js | 15 +++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) (limited to 'p') diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index 07a763613..d1f0d3535 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -460,6 +460,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { * - id (default: false): Feed ID * - url (default: false): Feed URL * - force (default: false) + * - noCommit (default: 0): Set to 1 to prevent committing the new articles to the main database * If id and url are not specified, all the feeds are actualized. But if force is * false, process stops at 10 feeds to avoid time execution problem. */ @@ -468,9 +469,19 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $id = Minz_Request::param('id'); $url = Minz_Request::param('url'); $force = Minz_Request::param('force'); - $noCommit = Minz_Request::fetchPOST('isLastFeed', 1) != 1; + $noCommit = Minz_Request::fetchPOST('noCommit', 0) == 1; - list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force, null, false, $noCommit); + if ($id == -1 && !$noCommit) { //Special request only to commit & refresh DB cache + $updated_feeds = 0; + $entryDAO = FreshRSS_Factory::createEntryDao(); + $feedDAO = FreshRSS_Factory::createFeedDao(); + $entryDAO->beginTransaction(); + $entryDAO->commitNewEntries(); + $feedDAO->updateCachedValues(); + $entryDAO->commit(); + } else { + list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force, null, false, $noCommit); + } if (Minz_Request::param('ajax')) { // Most of the time, ajax request is for only one feed. But since diff --git a/p/scripts/main.js b/p/scripts/main.js index c2f60bf7f..9aca75b93 100644 --- a/p/scripts/main.js +++ b/p/scripts/main.js @@ -803,13 +803,12 @@ function updateFeed(feeds, feeds_count) { if (!feed) { return; } - $.ajax({ type: 'POST', url: feed.url, data : { _csrf: context.csrf, - isLastFeed: feeds.length <= 0 ? 1 : 0, + noCommit: feeds.length > 0 ? 1 : 0, }, }).always(function (data) { feed_processed++; @@ -831,7 +830,6 @@ function init_actualize() { if (ajax_loading) { return false; } - ajax_loading = true; $.getJSON('./?c=javascript&a=actualize').done(function (data) { @@ -842,7 +840,16 @@ function init_actualize() { } if (data.feeds.length === 0) { openNotification(data.feedback_no_refresh, "good"); - ajax_loading = false; + $.ajax({ //Empty request to force refresh server database cache + type: 'POST', + url: './?c=feed&a=actualize&id=-1', + data : { + _csrf: context.csrf, + noCommit: 0, + }, + }).always(function (data) { + ajax_loading = false; + }); return; } //Progress bar -- cgit v1.2.3 From dbead5879a786111253fa75692f6df8fa09767c0 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 8 Apr 2017 11:20:15 +0200 Subject: More padding bottom menu https://github.com/FreshRSS/FreshRSS/issues/1479 --- p/themes/base-theme/template.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'p') diff --git a/p/themes/base-theme/template.css b/p/themes/base-theme/template.css index cc36c3ffa..d7ae7be51 100644 --- a/p/themes/base-theme/template.css +++ b/p/themes/base-theme/template.css @@ -345,7 +345,7 @@ a.btn { /*=== Tree */ .tree { margin: 0; - padding: 0; + padding: 0 0 15em 0; list-style: none; text-align: left; } -- cgit v1.2.3 From 535aa35ba70d8cc584f37388692022272ab883b2 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 10 Apr 2017 19:09:21 +0200 Subject: PSHB better unsubscribe Cases when a user is deleted, or when a feed is deleted. Removed random key do reduce the risk of subscribing several times to the same PSHB feed. --- app/Controllers/userController.php | 1 + app/Models/Feed.php | 15 ++++++++------- p/api/pshb.php | 39 +++++++++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 16 deletions(-) (limited to 'p') diff --git a/app/Controllers/userController.php b/app/Controllers/userController.php index f910cecd9..ddc0c6fe4 100644 --- a/app/Controllers/userController.php +++ b/app/Controllers/userController.php @@ -213,6 +213,7 @@ class FreshRSS_user_Controller extends Minz_ActionController { $userDAO = new FreshRSS_UserDAO(); $ok &= $userDAO->deleteUser($username); $ok &= recursive_unlink($user_data); + array_map('unlink', glob(PSHB_PATH . '/feeds/*/' . $username . '.txt')); } return $ok; } diff --git a/app/Models/Feed.php b/app/Models/Feed.php index 7a9cf8612..1bc6e48e8 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -429,7 +429,7 @@ class FreshRSS_Feed extends Minz_Model { } } else { @mkdir($path, 0777, true); - $key = sha1($path . FreshRSS_Context::$system_conf->salt . uniqid(mt_rand(), true)); + $key = sha1($path . FreshRSS_Context::$system_conf->salt); $hubJson = array( 'hub' => $this->hubUrl, 'key' => $key, @@ -451,15 +451,16 @@ class FreshRSS_Feed extends Minz_Model { //Parameter true to subscribe, false to unsubscribe. function pubSubHubbubSubscribe($state) { - if (FreshRSS_Context::$system_conf->base_url && $this->hubUrl && $this->selfUrl) { - $hubFilename = PSHB_PATH . '/feeds/' . base64url_encode($this->selfUrl) . '/!hub.json'; + $url = $this->selfUrl ? $this->selfUrl : $this->url; + if (FreshRSS_Context::$system_conf->base_url && $url) { + $hubFilename = PSHB_PATH . '/feeds/' . base64url_encode($url) . '/!hub.json'; $hubFile = @file_get_contents($hubFilename); if ($hubFile === false) { Minz_Log::warning('JSON not found for PubSubHubbub: ' . $this->url); return false; } $hubJson = json_decode($hubFile, true); - if (!$hubJson || empty($hubJson['key']) || !ctype_xdigit($hubJson['key'])) { + if (!$hubJson || empty($hubJson['key']) || !ctype_xdigit($hubJson['key']) || empty($hubJson['hub'])) { Minz_Log::warning('Invalid JSON for PubSubHubbub: ' . $this->url); return false; } @@ -474,13 +475,13 @@ class FreshRSS_Feed extends Minz_Model { } $ch = curl_init(); curl_setopt_array($ch, array( - CURLOPT_URL => $this->hubUrl, + CURLOPT_URL => $hubJson['hub'], CURLOPT_FOLLOWLOCATION => true, CURLOPT_RETURNTRANSFER => true, CURLOPT_USERAGENT => 'FreshRSS/' . FRESHRSS_VERSION . ' (' . PHP_OS . '; ' . FRESHRSS_WEBSITE . ')', CURLOPT_POSTFIELDS => 'hub.verify=sync' . '&hub.mode=' . ($state ? 'subscribe' : 'unsubscribe') - . '&hub.topic=' . urlencode($this->selfUrl) + . '&hub.topic=' . urlencode($url) . '&hub.callback=' . urlencode($callbackUrl) ) ); @@ -488,7 +489,7 @@ class FreshRSS_Feed extends Minz_Model { $info = curl_getinfo($ch); file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . - 'PubSubHubbub ' . ($state ? 'subscribe' : 'unsubscribe') . ' to ' . $this->selfUrl . + 'PubSubHubbub ' . ($state ? 'subscribe' : 'unsubscribe') . ' to ' . $url . ' with callback ' . $callbackUrl . ': ' . $info['http_code'] . ' ' . $response . "\n", FILE_APPEND); if (substr($info['http_code'], 0, 1) == '2') { diff --git a/p/api/pshb.php b/p/api/pshb.php index e9b66b167..378f43516 100644 --- a/p/api/pshb.php +++ b/p/api/pshb.php @@ -23,8 +23,13 @@ if (!ctype_xdigit($key)) { chdir(PSHB_PATH); $canonical64 = @file_get_contents('keys/' . $key . '.txt'); if ($canonical64 === false) { + if (!empty($_REQUEST['hub_mode']) && $_REQUEST['hub_mode'] === 'unsubscribe') { + logMe('Warning: Accept unknown unsubscribe'); + header('Connection: close'); + exit(isset($_REQUEST['hub_challenge']) ? $_REQUEST['hub_challenge'] : ''); + } header('HTTP/1.1 404 Not Found'); - logMe('Error: Feed key not found!: ' . $key); + logMe('Warning: Feed key not found!: ' . $key); die('Feed key not found!'); } $canonical64 = trim($canonical64); @@ -36,7 +41,7 @@ if (!preg_match('/^[A-Za-z0-9_-]+$/D', $canonical64)) { $hubFile = @file_get_contents('feeds/' . $canonical64 . '/!hub.json'); if ($hubFile === false) { header('HTTP/1.1 404 Not Found'); - //@unlink('keys/' . $key . '.txt'); + unlink('keys/' . $key . '.txt'); logMe('Error: Feed info not found!: ' . $canonical64); die('Feed info not found!'); } @@ -50,8 +55,19 @@ chdir('feeds/' . $canonical64); $users = glob('*.txt', GLOB_NOSORT); if (empty($users)) { header('HTTP/1.1 410 Gone'); - logMe('Error: Nobody is subscribed to this feed anymore!: ' . $canonical64); - die('Nobody is subscribed to this feed anymore!'); + $url = base64url_decode($canonical64); + logMe('Warning: Nobody subscribes to this feed anymore!: ' . $url); + unlink('../../keys/' . $key . '.txt'); + Minz_Configuration::register('system', + DATA_PATH . '/config.php', + DATA_PATH . '/config.default.php'); + FreshRSS_Context::$system_conf = Minz_Configuration::get('system'); + $feed = new FreshRSS_Feed($url); + $feed->pubSubHubbubSubscribe(false); + unlink('!hub.json'); + chdir('..'); + recursive_unlink($canonical64); + die('Nobody subscribes to this feed anymore!'); } if (!empty($_REQUEST['hub_mode']) && $_REQUEST['hub_mode'] === 'subscribe') { @@ -108,7 +124,9 @@ $nb = 0; foreach ($users as $userFilename) { $username = basename($userFilename, '.txt'); if (!file_exists(USERS_PATH . '/' . $username . '/config.php')) { - break; + logMe('Warning: Removing broken user link: ' . $username . ' for ' . $self); + unlink($userFilename); + continue; } try { @@ -119,11 +137,14 @@ foreach ($users as $userFilename) { new Minz_ModelPdo($username); //TODO: FIXME: Quick-fix while waiting for a better FreshRSS() constructor/init FreshRSS_Context::init(); list($updated_feeds, $feed) = FreshRSS_feed_Controller::actualizeFeed(0, $self, false, $simplePie); - if ($updated_feeds > 0) { + if ($updated_feeds > 0 || $feed != false) { $nb++; + } else { + logMe('Warning: User ' . $username . ' does not subscribe anymore to ' . $self); + unlink($userFilename); } } catch (Exception $e) { - logMe('Error: ' . $e->getMessage()); + logMe('Error: ' . $e->getMessage() . ' for user ' . $username . ' and feed ' . $self); } } @@ -132,8 +153,8 @@ unset($simplePie); if ($nb === 0) { header('HTTP/1.1 410 Gone'); - logMe('Error: Nobody is subscribed to this feed anymore after all!: ' . $self); - die('Nobody is subscribed to this feed anymore after all!'); + logMe('Warning: Nobody subscribes to this feed anymore after all!: ' . $self); + die('Nobody subscribes to this feed anymore after all!'); } elseif (!empty($hubJson['error'])) { $hubJson['error'] = false; file_put_contents('./!hub.json', json_encode($hubJson)); -- cgit v1.2.3 From 44c9ae51c44478e56ee70ce692ade6a275981320 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 23 Apr 2017 14:06:37 +0200 Subject: Rewriten Favicon library using cURL Reduce the number of requests, more robust, many more cases working, reduced code --- CHANGELOG.md | 2 +- lib/Favicon/DataAccess.php | 43 ----- lib/Favicon/Favicon.php | 396 ------------------------------------------ lib/Favicon/FaviconDLType.php | 23 --- lib/favicons.php | 110 ++++++++++-- p/f.php | 3 +- 6 files changed, 98 insertions(+), 479 deletions(-) delete mode 100644 lib/Favicon/DataAccess.php delete mode 100644 lib/Favicon/Favicon.php delete mode 100644 lib/Favicon/FaviconDLType.php (limited to 'p') diff --git a/CHANGELOG.md b/CHANGELOG.md index e680736db..08025bb3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ * Improve English [#1465](https://github.com/FreshRSS/FreshRSS/pull/1465) * Misc. * Fall back to article URL when the article GUID is empty [#1482](https://github.com/FreshRSS/FreshRSS/issues/1482) - * Update to version 1.2 of Favicon library [#1501](https://github.com/FreshRSS/FreshRSS/issues/1501) + * Rewriten Favicon library using cURL [#1503](https://github.com/FreshRSS/FreshRSS/pull/1503) ## 2017-03-11 FreshRSS 1.6.3 diff --git a/lib/Favicon/DataAccess.php b/lib/Favicon/DataAccess.php deleted file mode 100644 index 1445e9343..000000000 --- a/lib/Favicon/DataAccess.php +++ /dev/null @@ -1,43 +0,0 @@ -set_context(); - return @file_get_contents($url); - } - - public function retrieveHeader($url) { - $this->set_context(); - $headers = @get_headers($url, 1); - return is_array($headers) ? array_change_key_case($headers) : array(); - } - - public function saveCache($file, $data) { - file_put_contents($file, $data); - } - - public function readCache($file) { - return file_get_contents($file); - } - - private function set_context() { - stream_context_set_default( - array( - 'http' => array( - 'method' => 'GET', - 'follow_location' => 0, - 'max_redirects' => 1, - 'timeout' => 10, - 'header' => "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:20.0; Favicon; +https://github.com/ArthurHoaro/favicon) Gecko/20100101 Firefox/32.0\r\n", - ) - ) - ); - } -} \ No newline at end of file diff --git a/lib/Favicon/Favicon.php b/lib/Favicon/Favicon.php deleted file mode 100644 index c026d8a95..000000000 --- a/lib/Favicon/Favicon.php +++ /dev/null @@ -1,396 +0,0 @@ -url = $args['url']; - } - - $this->cacheDir = __DIR__ . '/../../resources/cache'; - $this->cacheTimeout = 604800; - $this->dataAccess = new DataAccess(); - } - - /** - * Set cache settings: - * - dir: cache directory - * - timeout: in seconds - * - * @param array $args - */ - public function cache($args = array()) { - if (isset($args['dir'])) { - $this->cacheDir = $args['dir']; - } - - if (!empty($args['timeout'])) { - $this->cacheTimeout = $args['timeout']; - } - } - - public static function baseUrl($url, $path = false) - { - $return = ''; - - if (!$url = parse_url($url)) { - return FALSE; - } - - // Scheme - $scheme = isset($url['scheme']) ? strtolower($url['scheme']) : null; - if ($scheme != 'http' && $scheme != 'https') { - - return FALSE; - } - $return .= "{$scheme}://"; - - // Username and password - if (isset($url['user'])) { - $return .= $url['user']; - if (isset($url['pass'])) { - $return .= ":{$url['pass']}"; - } - $return .= '@'; - } - - // Hostname - if( !isset($url['host']) ) { - return FALSE; - } - - $return .= $url['host']; - - // Port - if (isset($url['port'])) { - $return .= ":{$url['port']}"; - } - - // Path - if( $path && isset($url['path']) ) { - $return .= $url['path']; - } - $return .= '/'; - - return $return; - } - - public function info($url) - { - if(empty($url) || $url === false) { - return false; - } - - $max_loop = 5; - - // Discover real status by following redirects. - $loop = TRUE; - while ($loop && $max_loop-- > 0) { - $headers = $this->dataAccess->retrieveHeader($url); - if (empty($headers)) { - return false; - } - $exploded = explode(' ', $headers[0]); - - if( !isset($exploded[1]) ) { - return false; - } - list(,$status) = $exploded; - - switch ($status) { - case '301': - case '302': - $url = isset($headers['location']) ? $headers['location'] : ''; - if (is_array($url)) { - $url = end($url); - } - break; - default: - $loop = FALSE; - break; - } - } - - return array('status' => $status, 'url' => $url); - } - - public function endRedirect($url) { - $out = $this->info($url); - return !empty($out['url']) ? $out['url'] : false; - } - - /** - * Find remote (or cached) favicon - * - * @param string $url to look for a favicon - * @param int $type type of retrieval (FaviconDLType): - * - HOTLINK_URL: returns remote URL - * - DL_FILE_PATH: returns file path of the favicon downloaded locally - * - RAW_IMAGE: returns the favicon image binary string - * - * @return string|bool favicon URL, false if nothing was found - */ - public function get($url = '', $type = FaviconDLType::HOTLINK_URL) - { - // URLs passed to this method take precedence. - if (!empty($url)) { - $this->url = $url; - } - - // Get the base URL without the path for clearer concatenations. - $url = rtrim($this->baseUrl($this->url, true), '/'); - $original = $url; - if (($favicon = $this->checkCache($original, self::$TYPE_CACHE_URL)) === false - && ! $favicon = $this->getFavicon($original, false) - ) { - $url = rtrim($this->endRedirect($this->baseUrl($this->url, false)), '/'); - if (($favicon = $this->checkCache($url, self::$TYPE_CACHE_URL)) === false - && ! $favicon = $this->getFavicon($url) - ) { - $url = $original; - } - } - - $this->saveCache($url, $favicon, self::$TYPE_CACHE_URL); - - switch ($type) { - case FaviconDLType::DL_FILE_PATH: - return $this->getImage($url, $favicon, false); - case FaviconDLType::RAW_IMAGE: - return $this->getImage($url, $favicon, true); - case FaviconDLType::HOTLINK_URL: - default: - return empty($favicon) ? false : $favicon; - } - } - - private function getFavicon($url, $checkDefault = true) { - $favicon = false; - - if(empty($url)) { - return false; - } - - // Try /favicon.ico first. - if( $checkDefault ) { - $info = $this->info("{$url}/favicon.ico"); - if ($info['status'] == '200') { - $favicon = $info['url']; - } - } - - // See if it's specified in a link tag in domain url. - if (!$favicon) { - $favicon = trim($this->getInPage($url)); - } - if (substr($favicon, 0, 2) === '//') { - $favicon = 'https:' . $favicon; - } - - // Make sure the favicon is an absolute URL. - if( $favicon && filter_var($favicon, FILTER_VALIDATE_URL) === false ) { - $favicon = $url . '/' . $favicon; - } - - // Sometimes people lie, so check the status. - // And sometimes, it's not even an image. Sneaky bastards! - // If cacheDir isn't writable, that's not our problem - if ($favicon && is_writable($this->cacheDir) && extension_loaded('fileinfo') && !$this->checkImageMType($favicon)) { - $favicon = false; - } - - return $favicon; - } - - /** - * Find remote favicon and return it as an image - */ - private function getImage($url, $faviconUrl = '', $image = false) - { - if (empty($faviconUrl)) { - return false; - } - - $favicon = $this->checkCache($url, self::$TYPE_CACHE_IMG); - // Favicon not found in the cache - if( $favicon === false ) { - $favicon = $this->dataAccess->retrieveUrl($faviconUrl); - // Definitely not found - if (!$this->checkImageMTypeContent($favicon)) { - return false; - } else { - $this->saveCache($url, $favicon, self::$TYPE_CACHE_IMG); - } - } - - if( $image ) { - return $favicon; - } - else - return self::$TYPE_CACHE_IMG . md5($url); - } - - /** - * Display data as a PNG Favicon, then exit - * @param $data - */ - private function displayFavicon($data) { - header('Content-Type: image/png'); - header('Cache-Control: private, max-age=10800, pre-check=10800'); - header('Pragma: private'); - header('Expires: ' . date(DATE_RFC822,strtotime('7 day'))); - echo $data; - exit; - } - - private function getInPage($url) { - $html = $this->dataAccess->retrieveUrl("{$url}/"); - preg_match('!.*!ims', $html, $match); - - if(empty($match) || count($match) == 0) { - return false; - } - - $head = $match[0]; - - $dom = new \DOMDocument(); - // Use error suppression, because the HTML might be too malformed. - if (@$dom->loadHTML($head)) { - $links = $dom->getElementsByTagName('link'); - foreach ($links as $link) { - if ($link->hasAttribute('rel') && strtolower($link->getAttribute('rel')) == 'shortcut icon') { - return $link->getAttribute('href'); - } - } - foreach ($links as $link) { - if ($link->hasAttribute('rel') && strtolower($link->getAttribute('rel')) == 'icon') { - return $link->getAttribute('href'); - } - } - foreach ($links as $link) { - if ($link->hasAttribute('href') && strpos($link->getAttribute('href'), 'favicon') !== FALSE) { - return $link->getAttribute('href'); - } - } - } - return false; - } - - private function checkCache($url, $type) { - if ($this->cacheTimeout) { - $cache = $this->cacheDir . '/'. $type . md5($url); - if (file_exists($cache) && is_readable($cache) - && ($this->cacheTimeout === -1 || time() - filemtime($cache) < $this->cacheTimeout) - ) { - return $this->dataAccess->readCache($cache); - } - } - return false; - } - - /** - * Will save data in cacheDir if the directory writable and any previous cache is expired (cacheTimeout) - * @param $url - * @param $data - * @param $type - * @return string cache file path - */ - private function saveCache($url, $data, $type) { - // Save cache if necessary - $cache = $this->cacheDir . '/'. $type . md5($url); - if ($this->cacheTimeout && !file_exists($cache) - || (is_writable($cache) && $this->cacheTimeout !== -1 && time() - filemtime($cache) > $this->cacheTimeout) - ) { - $this->dataAccess->saveCache($cache, $data); - } - return $cache; - } - - private function checkImageMType($url) { - - $fileContent = $this->dataAccess->retrieveUrl($url); - - return $this->checkImageMTypeContent($fileContent); - } - - private function checkImageMTypeContent($content) { - if(empty($content)) return false; - - $isImage = true; - try { - $fInfo = finfo_open(FILEINFO_MIME_TYPE); - $isImage = strpos(finfo_buffer($fInfo, $content), 'image') !== false; - finfo_close($fInfo); - } catch (\Exception $e) { - error_log('Favicon checkImageMTypeContent error: ' . $e->getMessage()); - } - - return $isImage; - } - - /** - * @return mixed - */ - public function getCacheDir() - { - return $this->cacheDir; - } - - /** - * @param mixed $cacheDir - */ - public function setCacheDir($cacheDir) - { - $this->cacheDir = $cacheDir; - } - - /** - * @return mixed - */ - public function getCacheTimeout() - { - return $this->cacheTimeout; - } - - /** - * @param mixed $cacheTimeout - */ - public function setCacheTimeout($cacheTimeout) - { - $this->cacheTimeout = $cacheTimeout; - } - - /** - * @return string - */ - public function getUrl() - { - return $this->url; - } - - /** - * @param string $url - */ - public function setUrl($url) - { - $this->url = $url; - } - - /** - * @param DataAccess|\PHPUnit_Framework_MockObject_MockObject $dataAccess - */ - public function setDataAccess($dataAccess) - { - $this->dataAccess = $dataAccess; - } -} diff --git a/lib/Favicon/FaviconDLType.php b/lib/Favicon/FaviconDLType.php deleted file mode 100644 index 6da525a7f..000000000 --- a/lib/Favicon/FaviconDLType.php +++ /dev/null @@ -1,23 +0,0 @@ -setCacheDir($tmpPath); - $favicon_getter->setCacheTimeout(-1); - $favicon_path = $favicon_getter->get($website, \Favicon\FaviconDLType::DL_FILE_PATH); +function downloadHttp(&$url, $curlOptions = array()) { + syslog(LOG_INFO, 'FreshRSS Favicon GET ' . $url); + if (substr($url, 0, 2) === '//') { + $url = 'https:' . $favicon; + } + if ($url == '' || filter_var($url, FILTER_VALIDATE_URL) === false) { + return ''; + } + $ch = curl_init($url); + curl_setopt_array($ch, array( + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_MAXREDIRS => 10, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_TIMEOUT => 15, + CURLOPT_USERAGENT => 'FreshRSS/' . FRESHRSS_VERSION . ' (' . PHP_OS . '; ' . FRESHRSS_WEBSITE . ')', + )); + if (defined('CURLOPT_ENCODING')) { + curl_setopt($ch, CURLOPT_ENCODING, ''); //Enable all encodings + } + curl_setopt_array($ch, $curlOptions); + $response = curl_exec($ch); + $info = curl_getinfo($ch); + curl_close($ch); + if (!empty($info['url']) && (filter_var($info['url'], FILTER_VALIDATE_URL) !== false)) { + $url = $info['url']; + } + return $info['http_code'] == 200 ? $response : ''; +} + +function searchFavicon(&$url) { + $dom = new DOMDocument(); + $html = downloadHttp($url); + if ($html != '' && @$dom->loadHTML($html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING)) { + $rels = array('shortcut icon', 'icon'); + $links = $dom->getElementsByTagName('link'); + foreach ($rels as $rel) { + foreach ($links as $link) { + if ($link->hasAttribute('rel') && $link->hasAttribute('href') && + strtolower(trim($link->getAttribute('rel'))) === $rel) { + $href = trim($link->getAttribute('href')); + if (substr($href, 0, 2) === '//') { + $href = 'https:' . $href; + } + if (filter_var($href, FILTER_VALIDATE_URL) === false) { + $href = SimplePie_IRI::absolutize($url, $href); + } + $favicon = downloadHttp($href, array( + CURLOPT_REFERER => $url, + )); + if (isImgMime($favicon)) { + return $favicon; + } + } + } + } + } + return ''; +} - return ($favicon_path != false && @rename($tmpPath . '/' . $favicon_path, $dest)) || +function download_favicon($url, $dest) { + global $default_favicon; + $url = trim($url); + $favicon = searchFavicon($url); + if ($favicon == '') { + $rootUrl = preg_replace('%^(https?://[^/]+).*$%i', '$1/', $url); + if ($rootUrl != $url) { + $url = $rootUrl; + $favicon = searchFavicon($url); + } + if ($favicon == '') { + $link = $rootUrl . 'favicon.ico'; + $favicon = downloadHttp($link, array( + CURLOPT_REFERER => $url, + )); + if (!isImgMime($favicon)) { + $favicon = ''; + } + } + } + return ($favicon != '' && file_put_contents($dest, $favicon)) || @copy($default_favicon, $dest); } diff --git a/p/f.php b/p/f.php index e4c82bb16..c47fa747a 100644 --- a/p/f.php +++ b/p/f.php @@ -1,6 +1,6 @@