From 9d4a4457587d04030118447847a5c99d9095b937 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 24 Feb 2016 21:08:24 +0100 Subject: Work in progress HTTPS --- data/force-https.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 data/force-https.txt (limited to 'data') diff --git a/data/force-https.txt b/data/force-https.txt new file mode 100644 index 000000000..3da802d01 --- /dev/null +++ b/data/force-https.txt @@ -0,0 +1,3 @@ +dailymotion.com +tumblr.com +youtube.com -- cgit v1.2.3 From 9711f02db75006d622d75142d46c3c8d714c957f Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 28 Feb 2016 21:34:54 +0100 Subject: SimplePie Force HTTPS custom list domains Load from data/force-https.default.txt and data/force-https.txt Efficient tree structure to search the domains https://github.com/FreshRSS/FreshRSS/pull/1087 --- data/.gitignore | 1 + data/force-https.default.txt | 4 ++ data/force-https.txt | 3 -- lib/SimplePie/SimplePie.php | 14 ++++++ lib/SimplePie/SimplePie/Misc.php | 15 ------- lib/SimplePie/SimplePie/Sanitize.php | 85 +++++++++++++++++++++++++++++++++++- lib/lib_rss.php | 10 +++++ 7 files changed, 113 insertions(+), 19 deletions(-) create mode 100644 data/force-https.default.txt delete mode 100644 data/force-https.txt (limited to 'data') diff --git a/data/.gitignore b/data/.gitignore index 20364e266..c2ed350a6 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -7,3 +7,4 @@ no-cache.txt *.lock.txt last_update.txt update.php +force-https.txt diff --git a/data/force-https.default.txt b/data/force-https.default.txt new file mode 100644 index 000000000..a1bddd549 --- /dev/null +++ b/data/force-https.default.txt @@ -0,0 +1,4 @@ +dailymotion.com +feedburner.com +tumblr.com +youtube.com diff --git a/data/force-https.txt b/data/force-https.txt deleted file mode 100644 index 3da802d01..000000000 --- a/data/force-https.txt +++ /dev/null @@ -1,3 +0,0 @@ -dailymotion.com -tumblr.com -youtube.com diff --git a/lib/SimplePie/SimplePie.php b/lib/SimplePie/SimplePie.php index 6c0962a9f..61bad4e9d 100644 --- a/lib/SimplePie/SimplePie.php +++ b/lib/SimplePie/SimplePie.php @@ -1123,6 +1123,7 @@ class SimplePie $this->strip_attributes(false); $this->add_attributes(false); $this->set_image_handler(false); + $this->set_https_domains(array()); } } @@ -1233,6 +1234,19 @@ class SimplePie $this->sanitize->set_url_replacements($element_attribute); } + /** + * Set the list of domains for which force HTTPS. + * @see SimplePie_Misc::https_url() + * FreshRSS + */ + public function set_https_domains($domains = array()) + { + if (is_array($domains)) + { + $this->sanitize->set_https_domains($domains); + } + } + /** * Set the handler to enable the display of cached images. * diff --git a/lib/SimplePie/SimplePie/Misc.php b/lib/SimplePie/SimplePie/Misc.php index b9d74f894..2d154cbcb 100644 --- a/lib/SimplePie/SimplePie/Misc.php +++ b/lib/SimplePie/SimplePie/Misc.php @@ -77,21 +77,6 @@ class SimplePie_Misc return $time; } - /** - * Force HTTPS for selected Web sites - * FreshRSS - */ - public static function https_url($url) - { - if (strtolower(substr($url, 0, 7)) === 'http://') - { - $domain = parse_url($url, PHP_URL_HOST); - return preg_replace('%^http://((?:[^/]*?\.)?(?:youtube|dailymotion|tumblr)\.com/)%i', 'https://$1', $url); - return substr_replace($url, 's', 4, 0); //Add the 's' to HTTPS - } - return $url; - } - public static function absolutize_url($relative, $base) { if (substr($relative, 0, 2) === '//') diff --git a/lib/SimplePie/SimplePie/Sanitize.php b/lib/SimplePie/SimplePie/Sanitize.php index fc916a259..b37aeec3d 100644 --- a/lib/SimplePie/SimplePie/Sanitize.php +++ b/lib/SimplePie/SimplePie/Sanitize.php @@ -73,6 +73,14 @@ class SimplePie_Sanitize var $force_fsockopen = false; var $replace_url_attributes = null; + /** + * List of domains for which force HTTPS. + * @see SimplePie_Misc::https_url() + * Array is tree split at DNS levels. Example array('biz' => true, 'com' => array('example' => true), 'example' => array('test') => array('www' => true)); + * FreshRSS + */ + var $https_domains = array('com' => array('youtube' => true)); + public function __construct() { // Set defaults @@ -242,6 +250,81 @@ class SimplePie_Sanitize $this->replace_url_attributes = (array) $element_attribute; } + /** + * Set the list of domains for which force HTTPS. + * @see SimplePie_Misc::https_url() + * Example array('biz', 'example.com', 'example.org', 'www.example.net'); + * FreshRSS + */ + public function set_https_domains($domains) + { + $this->https_domains = array(); + foreach ($domains as $domain) + { + $domain = trim($domain, ". \t\n\r\0\x0B"); + $segments = array_reverse(explode('.', $domain)); + if (count($segments) > 0) + { + $node =& $this->https_domains; + foreach ($segments as $segment) + {//Build a tree + if ($node === true) + { + break; + } + if (!isset($node[$segment])) + { + $node[$segment] = array(); + } + $node =& $node[$segment]; + } + $node = true; + } + } + } + + /** + * Check if the domain is in the list of forced HTTPS + * FreshRSS + */ + protected function is_https_domain($domain) + { + $domain = trim($domain, '. '); + $segments = array_reverse(explode('.', $domain)); + if (count($segments) > 0) + { + $node =& $this->https_domains; + foreach ($segments as $segment) + {//Explore the tree + if ($node === true) + { + return true; + } + if (isset($node[$segment])) + { + $node =& $node[$segment]; + } + else + { + break; + } + } + } + return false; + } + + /** + * Force HTTPS for selected Web sites + * FreshRSS + */ + protected function https_url($url) + { + return (strtolower(substr($url, 0, 7)) === 'http://') && + $this->is_https_domain(parse_url($url, PHP_URL_HOST)) ? + substr_replace($url, 's', 4, 0) : //Add the 's' to HTTPS + $url; + } + public function sanitize($data, $type, $base = '') { $data = trim($data); @@ -451,7 +534,7 @@ class SimplePie_Sanitize if ($element->hasAttribute($attribute)) { $value = $this->registry->call('Misc', 'absolutize_url', array($element->getAttribute($attribute), $this->base)); - $value = SimplePie_Misc::https_url($value); //FreshRSS + $value = $this->https_url($value); //FreshRSS if ($value) { $element->setAttribute($attribute, $value); diff --git a/lib/lib_rss.php b/lib/lib_rss.php index b0189c162..5092982aa 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -238,6 +238,16 @@ function customSimplePie() { 'src', ), )); + $https_domains = array(); + $force = @file(DATA_PATH . '/force-https.default.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + if (is_array($force)) { + $https_domains = array_merge($https_domains, $force); + } + $force = @file(DATA_PATH . '/force-https.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + if (is_array($force)) { + $https_domains = array_merge($https_domains, $force); + } + $simplePie->set_https_domains($https_domains); return $simplePie; } -- cgit v1.2.3 From d344da59fdd2f818f53c6b4aab4ded4a9336851c Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 29 Feb 2016 17:09:44 +0100 Subject: HTTPS search tree: simplify code https://github.com/FreshRSS/FreshRSS/issues/1083 --- data/force-https.default.txt | 2 ++ lib/SimplePie/SimplePie/Sanitize.php | 58 ++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 32 deletions(-) (limited to 'data') diff --git a/data/force-https.default.txt b/data/force-https.default.txt index a1bddd549..668a7c0e8 100644 --- a/data/force-https.default.txt +++ b/data/force-https.default.txt @@ -1,4 +1,6 @@ dailymotion.com feedburner.com +gstatic.com tumblr.com +wordpress.com youtube.com diff --git a/lib/SimplePie/SimplePie/Sanitize.php b/lib/SimplePie/SimplePie/Sanitize.php index b37aeec3d..f656909a0 100644 --- a/lib/SimplePie/SimplePie/Sanitize.php +++ b/lib/SimplePie/SimplePie/Sanitize.php @@ -263,23 +263,20 @@ class SimplePie_Sanitize { $domain = trim($domain, ". \t\n\r\0\x0B"); $segments = array_reverse(explode('.', $domain)); - if (count($segments) > 0) - { - $node =& $this->https_domains; - foreach ($segments as $segment) - {//Build a tree - if ($node === true) - { - break; - } - if (!isset($node[$segment])) - { - $node[$segment] = array(); - } - $node =& $node[$segment]; + $node =& $this->https_domains; + foreach ($segments as $segment) + {//Build a tree + if ($node === true) + { + break; + } + if (!isset($node[$segment])) + { + $node[$segment] = array(); } - $node = true; + $node =& $node[$segment]; } + $node = true; } } @@ -291,23 +288,20 @@ class SimplePie_Sanitize { $domain = trim($domain, '. '); $segments = array_reverse(explode('.', $domain)); - if (count($segments) > 0) - { - $node =& $this->https_domains; - foreach ($segments as $segment) - {//Explore the tree - if ($node === true) - { - return true; - } - if (isset($node[$segment])) - { - $node =& $node[$segment]; - } - else - { - break; - } + $node =& $this->https_domains; + foreach ($segments as $segment) + {//Explore the tree + if ($node === true) + { + return true; + } + if (isset($node[$segment])) + { + $node =& $node[$segment]; + } + else + { + break; } } return false; -- cgit v1.2.3