Merge pull request #831 from Alkarex/PubSubHubbub

PubSubHubbub

8 files changed, 227 insertions, 23 deletions

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index 8db273aca..957a809cd 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -168,6 +168,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 			// Ok, feed has been added in database. Now we have to refresh entries.
 			$feed->_id($id);
 			$feed->faviconPrepare();
+			//$feed->pubSubHubbubPrepare();	//TODO: prepare PubSubHubbub already when adding the feed
 
 			$is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0;
 
@@ -261,12 +262,13 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 	 * This action actualizes entries from one or several feeds.
 	 *
 	 * Parameters are:
-	 *   - id (default: false)
+	 *   - id (default: false): Feed ID
+	 *   - url (default: false): Feed URL
 	 *   - force (default: false)
-	 * If id is not specified, all the feeds are actualized. But if force is
+	 * If id and url are not specified, all the feeds are actualized. But if force is
 	 * false, process stops at 10 feeds to avoid time execution problem.
 	 */
-	public function actualizeAction() {
+	public function actualizeAction($simplePiePush = null) {
 		@set_time_limit(300);
 
 		$feedDAO = FreshRSS_Factory::createFeedDao();
@@ -274,14 +276,15 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 
 		Minz_Session::_param('actualize_feeds', false);
 		$id = Minz_Request::param('id');
+		$url = Minz_Request::param('url');
 		$force = Minz_Request::param('force');
 
 		// Create a list of feeds to actualize.
 		// If id is set and valid, corresponding feed is added to the list but
 		// alone in order to automatize further process.
 		$feeds = array();
-		if ($id) {
-			$feed = $feedDAO->searchById($id);
+		if ($id || $url) {
+			$feed = $id ? $feedDAO->searchById($id) : $feedDAO->searchByUrl($url);
 			if ($feed) {
 				$feeds[] = $feed;
 			}
@@ -292,19 +295,32 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 		// Calculate date of oldest entries we accept in DB.
 		$nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1);
 		$date_min = time() - (3600 * 24 * 30 * $nb_month_old);
+		$pshbMinAge = time() - (3600 * 24);	//TODO: Make a configuration.
 
 		$updated_feeds = 0;
 		$is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0;
 		foreach ($feeds as $feed) {
+			$url = $feed->url();	//For detection of HTTP 301
+
+			$pubSubHubbubEnabled = $feed->pubSubHubbubEnabled();
+			if ((!$simplePiePush) && (!$id) && $pubSubHubbubEnabled && ($feed->lastUpdate() > $pshbMinAge)) {
+				$text = 'Skip pull of feed using PubSubHubbub: ' . $url;
+				//Minz_Log::debug($text);
+				file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND);
+				continue;	//When PubSubHubbub is used, do not pull refresh so often
+			}
+
 			if (!$feed->lock()) {
 				Minz_Log::notice('Feed already being actualized: ' . $feed->url());
 				continue;
 			}
 
-			$url = $feed->url();	//For detection of HTTP 301
 			try {
-				// Load entries
-				$feed->load(false);
+				if ($simplePiePush) {
+					$feed->loadEntries($simplePiePush);	//Used by PubSubHubbub
+				} else {
+					$feed->load(false);
+				}
 			} catch (FreshRSS_Feed_Exception $e) {
 				Minz_Log::notice($e->getMessage());
 				$feedDAO->updateLastUpdate($feed->id(), true);
@@ -368,6 +384,14 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 							continue;
 						}
 
+						if ($pubSubHubbubEnabled && !$simplePiePush) {	//We use push, but have discovered an article by pull!
+							$text = 'An article was discovered by pull although we use PubSubHubbub!: Feed ' . $url . ' GUID ' . $entry->guid();
+							file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND);
+							Minz_Log::warning($text);
+							$pubSubHubbubEnabled = false;
+							$feed->pubSubHubbubError(true);
+						}
+
 						if (!$entryDAO->hasTransaction()) {
 							$entryDAO->beginTransaction();
 						}
@@ -398,13 +422,32 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				$entryDAO->commit();
 			}
 
-			if ($feed->url() !== $url) {
-				// HTTP 301 Moved Permanently
+			if ($feed->hubUrl() && $feed->selfUrl()) {	//selfUrl has priority for PubSubHubbub
+				if ($feed->selfUrl() !== $url) {	//https://code.google.com/p/pubsubhubbub/wiki/MovingFeedsOrChangingHubs
+					$selfUrl = checkUrl($feed->selfUrl());
+					if ($selfUrl) {
+						Minz_Log::debug('PubSubHubbub unsubscribe ' . $feed->url());
+						if (!$feed->pubSubHubbubSubscribe(false)) {	//Unsubscribe
+							Minz_Log::warning('Error while PubSubHubbub unsubscribing from ' . $feed->url());
+						}
+						$feed->_url($selfUrl, false);
+						Minz_Log::notice('Feed ' . $url . ' canonical address moved to ' . $feed->url());
+						$feedDAO->updateFeed($feed->id(), array('url' => $feed->url()));
+					}
+				}
+			}
+			elseif ($feed->url() !== $url) {	// HTTP 301 Moved Permanently
 				Minz_Log::notice('Feed ' . $url . ' moved permanently to ' . $feed->url());
 				$feedDAO->updateFeed($feed->id(), array('url' => $feed->url()));
 			}
 
 			$feed->faviconPrepare();
+			if ($feed->pubSubHubbubPrepare()) {
+				Minz_Log::notice('PubSubHubbub subscribe ' . $feed->url());
+				if (!$feed->pubSubHubbubSubscribe(true)) {	//Subscribe
+					Minz_Log::warning('Error while PubSubHubbub subscribing to ' . $feed->url());
+				}
+			}
 			$feed->unlock();
 			$updated_feeds++;
 			unset($feed);
@@ -427,20 +470,20 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 			Minz_Session::_param('notification', $notif);
 			// No layout in ajax request.
 			$this->view->_useLayout(false);
-			return;
-		}
-
-		// Redirect to the main page with correct notification.
-		if ($updated_feeds === 1) {
-			$feed = reset($feeds);
-			Minz_Request::good(_t('feedback.sub.feed.actualized', $feed->name()), array(
-				'params' => array('get' => 'f_' . $feed->id())
-			));
-		} elseif ($updated_feeds > 1) {
-			Minz_Request::good(_t('feedback.sub.feed.n_actualized', $updated_feeds), array());
 		} else {
-			Minz_Request::good(_t('feedback.sub.feed.no_refresh'), array());
+			// Redirect to the main page with correct notification.
+			if ($updated_feeds === 1) {
+				$feed = reset($feeds);
+				Minz_Request::good(_t('feedback.sub.feed.actualized', $feed->name()), array(
+					'params' => array('get' => 'f_' . $feed->id())
+				));
+			} elseif ($updated_feeds > 1) {
+				Minz_Request::good(_t('feedback.sub.feed.n_actualized', $updated_feeds), array());
+			} else {
+				Minz_Request::good(_t('feedback.sub.feed.no_refresh'), array());
+			}
 		}
+		return $updated_feeds;
 	}
 
 	/**
diff --git a/app/Models/Feed.php b/app/Models/Feed.php
index 85fb173ec..bf7ed3967 100644
--- a/app/Models/Feed.php
+++ b/app/Models/Feed.php
@@ -19,6 +19,8 @@ class FreshRSS_Feed extends Minz_Model {
 	private $ttl = -2;
 	private $hash = null;
 	private $lockPath = '';
+	private $hubUrl = '';
+	private $selfUrl = '';
 
 	public function __construct($url, $validate=true) {
 		if ($validate) {
@@ -49,6 +51,12 @@ class FreshRSS_Feed extends Minz_Model {
 	public function url() {
 		return $this->url;
 	}
+	public function selfUrl() {
+		return $this->selfUrl;
+	}
+	public function hubUrl() {
+		return $this->hubUrl;
+	}
 	public function category() {
 		return $this->category;
 	}
@@ -96,6 +104,16 @@ class FreshRSS_Feed extends Minz_Model {
 	public function ttl() {
 		return $this->ttl;
 	}
+	// public function ttlExpire() {
+		// $ttl = $this->ttl;
+		// if ($ttl == -2) {	//Default
+			// $ttl = FreshRSS_Context::$user_conf->ttl_default;
+		// }
+		// if ($ttl == -1) {	//Never
+			// $ttl = 64000000;	//~2 years. Good enough for PubSubHubbub logic
+		// }
+		// return $this->lastUpdate + $ttl;
+	// }
 	public function nbEntries() {
 		if ($this->nbEntries < 0) {
 			$feedDAO = FreshRSS_Factory::createFeedDao();
@@ -226,6 +244,11 @@ class FreshRSS_Feed extends Minz_Model {
 					throw new FreshRSS_Feed_Exception(($errorMessage == '' ? 'Feed error' : $errorMessage) . ' [' . $url . ']');
 				}
 
+				$links = $feed->get_links('self');
+				$this->selfUrl = isset($links[0]) ? $links[0] : null;
+				$links = $feed->get_links('hub');
+				$this->hubUrl = isset($links[0]) ? $links[0] : null;
+
 				if ($loadDetails) {
 					// si on a utilisé l'auto-discover, notre url va avoir changé
 					$subscribe_url = $feed->subscribe_url(false);
@@ -259,7 +282,7 @@ class FreshRSS_Feed extends Minz_Model {
 		}
 	}
 
-	private function loadEntries($feed) {
+	public function loadEntries($feed) {
 		$entries = array();
 
 		foreach ($feed->get_items() as $item) {
@@ -333,4 +356,129 @@ class FreshRSS_Feed extends Minz_Model {
 	function unlock() {
 		@unlink($this->lockPath);
 	}
+
+	//<PubSubHubbub>
+
+	function pubSubHubbubEnabled() {
+		$url = $this->selfUrl ? $this->selfUrl : $this->url;
+		$hubFilename = PSHB_PATH . '/feeds/' . base64url_encode($url) . '/!hub.json';
+		if ($hubFile = @file_get_contents($hubFilename)) {
+			$hubJson = json_decode($hubFile, true);
+			if ($hubJson && empty($hubJson['error']) &&
+				(empty($hubJson['lease_end']) || $hubJson['lease_end'] > time())) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	function pubSubHubbubError($error = true) {
+		$url = $this->selfUrl ? $this->selfUrl : $this->url;
+		$hubFilename = PSHB_PATH . '/feeds/' . base64url_encode($url) . '/!hub.json';
+		$hubFile = @file_get_contents($hubFilename);
+		$hubJson = $hubFile ? json_decode($hubFile, true) : array();
+		if (!isset($hubJson['error']) || $hubJson['error'] !== (bool)$error) {
+			$hubJson['error'] = (bool)$error;
+			file_put_contents($hubFilename, json_encode($hubJson));
+			file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t"
+				. 'Set error to ' . ($error ? 1 : 0) . ' for ' . $url . "\n", FILE_APPEND);
+		}
+		return false;
+	}
+
+	function pubSubHubbubPrepare() {
+		$key = '';
+		if (FreshRSS_Context::$system_conf->base_url && $this->hubUrl && $this->selfUrl && @is_dir(PSHB_PATH)) {
+			$path = PSHB_PATH . '/feeds/' . base64url_encode($this->selfUrl);
+			$hubFilename = $path . '/!hub.json';
+			if ($hubFile = @file_get_contents($hubFilename)) {
+				$hubJson = json_decode($hubFile, true);
+				if (!$hubJson || empty($hubJson['key']) || !ctype_xdigit($hubJson['key'])) {
+					$text = 'Invalid JSON for PubSubHubbub: ' . $this->url;
+					Minz_Log::warning($text);
+					file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND);
+					return false;
+				}
+				if ((!empty($hubJson['lease_end'])) && ($hubJson['lease_end'] < (time() + (3600 * 23)))) {	//TODO: Make a better policy
+					$text = 'PubSubHubbub lease ends at '
+						. date('c', empty($hubJson['lease_end']) ? time() : $hubJson['lease_end'])
+						. ' and needs renewal: ' . $this->url;
+					Minz_Log::warning($text);
+					file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND);
+					$key = $hubJson['key'];	//To renew our lease
+				} elseif (((!empty($hubJson['error'])) || empty($hubJson['lease_end'])) &&
+					(empty($hubJson['lease_start']) || $hubJson['lease_start'] < time() - (3600 * 23))) {	//Do not renew too often
+					$key = $hubJson['key'];	//To renew our lease
+				}
+			} else {
+				@mkdir($path, 0777, true);
+				$key = sha1($path . FreshRSS_Context::$system_conf->salt . uniqid(mt_rand(), true));
+				$hubJson = array(
+					'hub' => $this->hubUrl,
+					'key' => $key,
+				);
+				file_put_contents($hubFilename, json_encode($hubJson));
+				@mkdir(PSHB_PATH . '/keys/');
+				file_put_contents(PSHB_PATH . '/keys/' . $key . '.txt', base64url_encode($this->selfUrl));
+				$text = 'PubSubHubbub prepared for ' . $this->url;
+				Minz_Log::debug($text);
+				file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND);
+			}
+			$currentUser = Minz_Session::param('currentUser');
+			if (ctype_alnum($currentUser) && !file_exists($path . '/' . $currentUser . '.txt')) {
+				touch($path . '/' . $currentUser . '.txt');
+			}
+		}
+		return $key;
+	}
+
+	//Parameter true to subscribe, false to unsubscribe.
+	function pubSubHubbubSubscribe($state) {
+		if (FreshRSS_Context::$system_conf->base_url && $this->hubUrl && $this->selfUrl) {
+			$hubFilename = PSHB_PATH . '/feeds/' . base64url_encode($this->selfUrl) . '/!hub.json';
+			$hubFile = @file_get_contents($hubFilename);
+			if ($hubFile === false) {
+				Minz_Log::warning('JSON not found for PubSubHubbub: ' . $this->url);
+				return false;
+			}
+			$hubJson = json_decode($hubFile, true);
+			if (!$hubJson || empty($hubJson['key']) || !ctype_xdigit($hubJson['key'])) {
+				Minz_Log::warning('Invalid JSON for PubSubHubbub: ' . $this->url);
+				return false;
+			}
+			$callbackUrl = checkUrl(FreshRSS_Context::$system_conf->base_url . 'api/pshb.php?k=' . $hubJson['key']);
+			if ($callbackUrl == '') {
+				Minz_Log::warning('Invalid callback for PubSubHubbub: ' . $this->url);
+				return false;
+			}
+			$ch = curl_init();
+			curl_setopt_array($ch, array(
+				CURLOPT_URL => $this->hubUrl,
+				CURLOPT_FOLLOWLOCATION => true,
+				CURLOPT_RETURNTRANSFER => true,
+				CURLOPT_USERAGENT => _t('gen.freshrss') . '/' . FRESHRSS_VERSION . ' (' . PHP_OS . '; ' . FRESHRSS_WEBSITE . ')',
+				CURLOPT_POSTFIELDS => 'hub.verify=sync'
+					. '&hub.mode=' . ($state ? 'subscribe' : 'unsubscribe')
+					. '&hub.topic=' . urlencode($this->selfUrl)
+					. '&hub.callback=' . urlencode($callbackUrl)
+				)
+			);
+			$response = curl_exec($ch);
+			$info = curl_getinfo($ch);
+
+			file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" .
+				'PubSubHubbub ' . ($state ? 'subscribe' : 'unsubscribe') . ' to ' . $this->selfUrl .
+				' with callback ' . $callbackUrl . ': ' . $info['http_code'] . ' ' . $response . "\n", FILE_APPEND);
+
+			if (!$state) {	//unsubscribe
+				$hubJson['lease_end'] = time() - 60;
+				file_put_contents($hubFilename, json_encode($hubJson));
+			}
+
+			return substr($info['http_code'], 0, 1) == '2';
+		}
+		return false;
+	}
+
+	//</PubSubHubbub>
 }
diff --git a/app/i18n/cz/conf.php b/app/i18n/cz/conf.php
index 29fb1e4d4..9518df66d 100644
--- a/app/i18n/cz/conf.php
+++ b/app/i18n/cz/conf.php
@@ -84,6 +84,7 @@ return array(
 		'articles_per_page' => 'Počet článků na stranu',
 		'auto_load_more' => 'Načítat další články dole na stránce',
 		'auto_remove_article' => 'Po přečtení články schovat',
+		'mark_updated_article_unread' => 'Označte aktualizované položky jako nepřečtené',
 		'confirm_enabled' => 'Vyžadovat potvrzení pro akci “označit vše jako přečtené”',
 		'display_articles_unfolded' => 'Ve výchozím stavu zobrazovat články otevřené',
 		'display_categories_unfolded' => 'Ve výchozím stavu zobrazovat kategorie zavřené',
diff --git a/app/i18n/cz/sub.php b/app/i18n/cz/sub.php
index 78712506c..cea0541e3 100644
--- a/app/i18n/cz/sub.php
+++ b/app/i18n/cz/sub.php
@@ -37,6 +37,7 @@ return array(
 		'url' => 'URL kanálu',
 		'validator' => 'Zkontrolovat platnost kanálu',
 		'website' => 'URL webové stránky',
+		'pubsubhubbub' => 'Okamžité oznámení s PubSubHubbub',
 	),
 	'import_export' => array(
 		'export' => 'Export',
diff --git a/app/i18n/de/sub.php b/app/i18n/de/sub.php
index 0479b8f46..0f05a5635 100644
--- a/app/i18n/de/sub.php
+++ b/app/i18n/de/sub.php
@@ -37,6 +37,7 @@ return array(
 		'url' => 'Feed-URL',
 		'validator' => 'Überprüfen Sie die Gültigkeit des Feeds',
 		'website' => 'Webseiten-URL',
+		'pubsubhubbub' => 'Sofortbenachrichtigung mit PubSubHubbub',
 	),
 	'import_export' => array(
 		'export' => 'Exportieren',
diff --git a/app/i18n/en/sub.php b/app/i18n/en/sub.php
index 8fe150bb2..aaaa02827 100644
--- a/app/i18n/en/sub.php
+++ b/app/i18n/en/sub.php
@@ -37,6 +37,7 @@ return array(
 		'url' => 'Feed URL',
 		'validator' => 'Check the validity of the feed',
 		'website' => 'Website URL',
+		'pubsubhubbub' => 'Instant notification with PubSubHubbub',
 	),
 	'import_export' => array(
 		'export' => 'Export',
diff --git a/app/i18n/fr/sub.php b/app/i18n/fr/sub.php
index a3f7c4d6d..0a1a03e41 100644
--- a/app/i18n/fr/sub.php
+++ b/app/i18n/fr/sub.php
@@ -37,6 +37,7 @@ return array(
 		'url' => 'URL du flux',
 		'validator' => 'Vérifier la valididé du flux',
 		'website' => 'URL du site',
+		'pubsubhubbub' => 'Notification instantanée par PubSubHubbub',
 	),
 	'import_export' => array(
 		'export' => 'Exporter',
diff --git a/app/views/helpers/feed/update.phtml b/app/views/helpers/feed/update.phtml
index 0b08d036c..b2cf9f93c 100644
--- a/app/views/helpers/feed/update.phtml
+++ b/app/views/helpers/feed/update.phtml
@@ -126,6 +126,14 @@
 				?></select>
 			</div>
 		</div>
+		<div class="form-group">
+			<label class="group-name" for="pubsubhubbub"><?php echo _t('sub.feed.pubsubhubbub'); ?></label>
+			<div class="group-controls">
+				<label class="checkbox" for="pubsubhubbub">
+					<input type="checkbox" name="pubsubhubbub" id="pubsubhubbub" disabled="disabled" value="1"<?php echo $this->feed->pubSubHubbubEnabled() ? ' checked="checked"' : ''; ?> />
+				</label>
+			</div>
+		</div>
 		<div class="form-group form-actions">
 			<div class="group-controls">
 				<button class="btn btn-important"><?php echo _t('gen.action.submit'); ?></button>