From bc8eb560afd50290745ea6a500c0f930df2559eb Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sun, 6 Jul 2014 00:09:31 +0200 Subject: Add TTL to control feed freshness https://github.com/marienfressinaud/FreshRSS/issues/250 --- app/Controllers/configureController.php | 3 + app/Controllers/entryController.php | 2 +- app/Controllers/feedController.php | 2 +- app/Controllers/javascriptController.php | 2 +- app/Models/Configuration.php | 5 ++ app/Models/Feed.php | 150 ++++++++++++++++--------------- app/Models/FeedDAO.php | 19 ++-- app/i18n/en.php | 3 +- app/i18n/fr.php | 1 + app/views/configure/archiving.phtml | 21 +++++ app/views/configure/feed.phtml | 21 +++++ lib/lib_rss.php | 2 +- 12 files changed, 150 insertions(+), 81 deletions(-) diff --git a/app/Controllers/configureController.php b/app/Controllers/configureController.php index a608df162..5fd09a263 100755 --- a/app/Controllers/configureController.php +++ b/app/Controllers/configureController.php @@ -109,6 +109,7 @@ class FreshRSS_configure_Controller extends Minz_ActionController { 'priority' => intval(Minz_Request::param ('priority', 0)), 'httpAuth' => $httpAuth, 'keep_history' => intval(Minz_Request::param ('keep_history', -2)), + 'ttl' => intval(Minz_Request::param('ttl', -2)), ); if ($feedDAO->updateFeed ($id, $values)) { @@ -274,9 +275,11 @@ class FreshRSS_configure_Controller extends Minz_ActionController { if (Minz_Request::isPost()) { $old = Minz_Request::param('old_entries', 3); $keepHistoryDefault = Minz_Request::param('keep_history_default', 0); + $ttlDefault = Minz_Request::param('ttl_default', -2); $this->view->conf->_old_entries($old); $this->view->conf->_keep_history_default($keepHistoryDefault); + $this->view->conf->_ttl_default($ttlDefault); $this->view->conf->save(); invalidateHttpCache(); diff --git a/app/Controllers/entryController.php b/app/Controllers/entryController.php index 2d7fa718a..ac43587ea 100755 --- a/app/Controllers/entryController.php +++ b/app/Controllers/entryController.php @@ -125,7 +125,7 @@ class FreshRSS_entry_Controller extends Minz_ActionController { $date_min = time() - (3600 * 24 * 30 * $nb_month_old); $feedDAO = FreshRSS_Factory::createFeedDao(); - $feeds = $feedDAO->listFeedsOrderUpdate(); + $feeds = $feedDAO->listFeeds(); $nbTotal = 0; invalidateHttpCache(); diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index d30b60877..36425ca9b 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -233,7 +233,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $feeds = array ($feed); } } else { - $feeds = $feedDAO->listFeedsOrderUpdate (); + $feeds = $feedDAO->listFeedsOrderUpdate($this->view->conf->ttl_default); } // on calcule la date des articles les plus anciens qu'on accepte diff --git a/app/Controllers/javascriptController.php b/app/Controllers/javascriptController.php index 737908c91..67148350f 100755 --- a/app/Controllers/javascriptController.php +++ b/app/Controllers/javascriptController.php @@ -8,7 +8,7 @@ class FreshRSS_javascript_Controller extends Minz_ActionController { public function actualizeAction () { header('Content-Type: text/javascript; charset=UTF-8'); $feedDAO = FreshRSS_Factory::createFeedDao(); - $this->view->feeds = $feedDAO->listFeedsOrderUpdate(); + $this->view->feeds = $feedDAO->listFeedsOrderUpdate($this->view->conf->ttl_default); } public function nbUnreadsPerFeedAction() { diff --git a/app/Models/Configuration.php b/app/Models/Configuration.php index ffd20deca..f2084b833 100644 --- a/app/Models/Configuration.php +++ b/app/Models/Configuration.php @@ -7,6 +7,7 @@ class FreshRSS_Configuration { 'language' => 'en', 'old_entries' => 3, 'keep_history_default' => 0, + 'ttl_default' => 3600, 'mail_login' => '', 'token' => '', 'passwordHash' => '', //CRYPT_BLOWFISH @@ -159,6 +160,10 @@ class FreshRSS_Configuration { $value = intval($value); $this->data['keep_history_default'] = $value >= -1 ? $value : 0; } + public function _ttl_default($value) { + $value = intval($value); + $this->data['ttl_default'] = $value >= -1 ? $value : 3600; + } public function _shortcuts ($values) { foreach ($values as $key => $value) { if (isset($this->data['shortcuts'][$key])) { diff --git a/app/Models/Feed.php b/app/Models/Feed.php index c9f1230ad..576f37760 100644 --- a/app/Models/Feed.php +++ b/app/Models/Feed.php @@ -16,18 +16,19 @@ class FreshRSS_Feed extends Minz_Model { private $httpAuth = ''; private $error = false; private $keep_history = -2; + private $ttl = -2; private $hash = null; private $lockPath = ''; - public function __construct ($url, $validate=true) { + public function __construct($url, $validate=true) { if ($validate) { - $this->_url ($url); + $this->_url($url); } else { $this->url = $url; } } - public function id () { + public function id() { return $this->id; } @@ -38,72 +39,75 @@ class FreshRSS_Feed extends Minz_Model { return $this->hash; } - public function url () { + public function url() { return $this->url; } - public function category () { + public function category() { return $this->category; } - public function entries () { + public function entries() { return $this->entries === null ? array() : $this->entries; } - public function name () { + public function name() { return $this->name; } - public function website () { + public function website() { return $this->website; } - public function description () { + public function description() { return $this->description; } - public function lastUpdate () { + public function lastUpdate() { return $this->lastUpdate; } - public function priority () { + public function priority() { return $this->priority; } - public function pathEntries () { + public function pathEntries() { return $this->pathEntries; } - public function httpAuth ($raw = true) { + public function httpAuth($raw = true) { if ($raw) { return $this->httpAuth; } else { - $pos_colon = strpos ($this->httpAuth, ':'); - $user = substr ($this->httpAuth, 0, $pos_colon); - $pass = substr ($this->httpAuth, $pos_colon + 1); + $pos_colon = strpos($this->httpAuth, ':'); + $user = substr($this->httpAuth, 0, $pos_colon); + $pass = substr($this->httpAuth, $pos_colon + 1); - return array ( + return array( 'username' => $user, 'password' => $pass ); } } - public function inError () { + public function inError() { return $this->error; } - public function keepHistory () { + public function keepHistory() { return $this->keep_history; } - public function nbEntries () { + public function ttl() { + return $this->ttl; + } + public function nbEntries() { if ($this->nbEntries < 0) { $feedDAO = FreshRSS_Factory::createFeedDao(); - $this->nbEntries = $feedDAO->countEntries ($this->id ()); + $this->nbEntries = $feedDAO->countEntries($this->id()); } return $this->nbEntries; } - public function nbNotRead () { + public function nbNotRead() { if ($this->nbNotRead < 0) { $feedDAO = FreshRSS_Factory::createFeedDao(); - $this->nbNotRead = $feedDAO->countNotRead ($this->id ()); + $this->nbNotRead = $feedDAO->countNotRead($this->id()); } return $this->nbNotRead; } public function faviconPrepare() { $file = DATA_PATH . '/favicons/' . $this->hash() . '.txt'; - if (!file_exists ($file)) { + if (!file_exists($file)) { $t = $this->website; if ($t == '') { $t = $this->url; @@ -116,92 +120,98 @@ class FreshRSS_Feed extends Minz_Model { @unlink($path . '.ico'); @unlink($path . '.txt'); } - public function favicon () { - return Minz_Url::display ('/f.php?' . $this->hash()); + public function favicon() { + return Minz_Url::display('/f.php?' . $this->hash()); } - public function _id ($value) { + public function _id($value) { $this->id = $value; } - public function _url ($value, $validate=true) { + public function _url($value, $validate=true) { $this->hash = null; if ($validate) { $value = checkUrl($value); } - if (empty ($value)) { - throw new FreshRSS_BadUrl_Exception ($value); + if (empty($value)) { + throw new FreshRSS_BadUrl_Exception($value); } $this->url = $value; } - public function _category ($value) { + public function _category($value) { $value = intval($value); $this->category = $value >= 0 ? $value : 0; } - public function _name ($value) { + public function _name($value) { $this->name = $value === null ? '' : $value; } - public function _website ($value, $validate=true) { + public function _website($value, $validate=true) { if ($validate) { $value = checkUrl($value); } - if (empty ($value)) { + if (empty($value)) { $value = ''; } $this->website = $value; } - public function _description ($value) { + public function _description($value) { $this->description = $value === null ? '' : $value; } - public function _lastUpdate ($value) { + public function _lastUpdate($value) { $this->lastUpdate = $value; } - public function _priority ($value) { + public function _priority($value) { $value = intval($value); $this->priority = $value >= 0 ? $value : 10; } - public function _pathEntries ($value) { + public function _pathEntries($value) { $this->pathEntries = $value; } - public function _httpAuth ($value) { + public function _httpAuth($value) { $this->httpAuth = $value; } - public function _error ($value) { + public function _error($value) { $this->error = (bool)$value; } - public function _keepHistory ($value) { + public function _keepHistory($value) { $value = intval($value); $value = min($value, 1000000); $value = max($value, -2); $this->keep_history = $value; } - public function _nbNotRead ($value) { + public function _ttl($value) { + $value = intval($value); + $value = min($value, 100000000); + $value = max($value, -2); + $this->ttl = $value; + } + public function _nbNotRead($value) { $this->nbNotRead = intval($value); } - public function _nbEntries ($value) { + public function _nbEntries($value) { $this->nbEntries = intval($value); } - public function load ($loadDetails = false) { + public function load($loadDetails = false) { if ($this->url !== null) { if (CACHE_PATH === false) { - throw new Minz_FileNotExistException ( + throw new Minz_FileNotExistException( 'CACHE_PATH', Minz_Exception::ERROR ); } else { - $url = htmlspecialchars_decode ($this->url, ENT_QUOTES); + $url = htmlspecialchars_decode($this->url, ENT_QUOTES); if ($this->httpAuth != '') { - $url = preg_replace ('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $url); + $url = preg_replace('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $url); } $feed = customSimplePie(); - $feed->set_feed_url ($url); + $feed->set_feed_url($url); if (!$loadDetails) { //Only activates auto-discovery when adding a new feed $feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE); } $mtime = $feed->init(); if ((!$mtime) || $feed->error()) { - throw new FreshRSS_Feed_Exception ($feed->error() . ' [' . $url . ']'); + throw new FreshRSS_Feed_Exception($feed->error() . ' [' . $url . ']'); } if ($loadDetails) { @@ -209,7 +219,7 @@ class FreshRSS_Feed extends Minz_Model { $subscribe_url = $feed->subscribe_url(false); $title = strtr(html_only_entity_decode($feed->get_title()), array('<' => '<', '>' => '>', '"' => '"')); //HTML to HTML-PRE //ENT_COMPAT except & - $this->_name ($title == '' ? $this->url : $title); + $this->_name($title == '' ? $this->url : $title); $this->_website(html_only_entity_decode($feed->get_link())); $this->_description(html_only_entity_decode($feed->get_description())); @@ -221,12 +231,12 @@ class FreshRSS_Feed extends Minz_Model { if ($subscribe_url !== null && $subscribe_url !== $this->url) { if ($this->httpAuth != '') { // on enlève les id si authentification HTTP - $subscribe_url = preg_replace ('#((.+)://)((.+)@)(.+)#', '${1}${5}', $subscribe_url); + $subscribe_url = preg_replace('#((.+)://)((.+)@)(.+)#', '${1}${5}', $subscribe_url); } - $this->_url ($subscribe_url); + $this->_url($subscribe_url); } - if (($mtime === true) || ($mtime > $this->lastUpdate)) { + if (($mtime === true) ||($mtime > $this->lastUpdate)) { syslog(LOG_DEBUG, 'FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $subscribe_url); $this->loadEntries($feed); // et on charge les articles du flux } else { @@ -240,25 +250,25 @@ class FreshRSS_Feed extends Minz_Model { } } - private function loadEntries ($feed) { - $entries = array (); + private function loadEntries($feed) { + $entries = array(); - foreach ($feed->get_items () as $item) { - $title = html_only_entity_decode (strip_tags ($item->get_title ())); - $author = $item->get_author (); - $link = $item->get_permalink (); - $date = @strtotime ($item->get_date ()); + foreach ($feed->get_items() as $item) { + $title = html_only_entity_decode(strip_tags($item->get_title())); + $author = $item->get_author(); + $link = $item->get_permalink(); + $date = @strtotime($item->get_date()); // gestion des tags (catégorie == tag) - $tags_tmp = $item->get_categories (); - $tags = array (); + $tags_tmp = $item->get_categories(); + $tags = array(); if ($tags_tmp !== null) { foreach ($tags_tmp as $tag) { - $tags[] = html_only_entity_decode ($tag->get_label ()); + $tags[] = html_only_entity_decode($tag->get_label()); } } - $content = html_only_entity_decode ($item->get_content ()); + $content = html_only_entity_decode($item->get_content()); $elinks = array(); foreach ($item->get_enclosures() as $enclosure) { @@ -276,16 +286,16 @@ class FreshRSS_Feed extends Minz_Model { } } - $entry = new FreshRSS_Entry ( - $this->id (), - $item->get_id (), + $entry = new FreshRSS_Entry( + $this->id(), + $item->get_id(), $title === null ? '' : $title, - $author === null ? '' : html_only_entity_decode ($author->name), + $author === null ? '' : html_only_entity_decode($author->name), $content === null ? '' : $content, $link === null ? '' : $link, - $date ? $date : time () + $date ? $date : time() ); - $entry->_tags ($tags); + $entry->_tags($tags); // permet de récupérer le contenu des flux tronqués $entry->loadCompleteContent($this->pathEntries()); diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index 5281b371d..83f3a6231 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -2,7 +2,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo { public function addFeed($valuesTmp) { - $sql = 'INSERT INTO `' . $this->prefix . 'feed` (url, category, name, website, description, lastUpdate, priority, httpAuth, error, keep_history) VALUES(?, ?, ?, ?, ?, ?, 10, ?, 0, -2)'; + $sql = 'INSERT INTO `' . $this->prefix . 'feed` (url, category, name, website, description, lastUpdate, priority, httpAuth, error, keep_history, ttl) VALUES(?, ?, ?, ?, ?, ?, 10, ?, 0, -2, -2)'; $stm = $this->bd->prepare($sql); $values = array( @@ -222,13 +222,19 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo { return $feedCategoryNames; } - public function listFeedsOrderUpdate($cacheDuration = 1500) { - $sql = 'SELECT id, url, name, website, lastUpdate, pathEntries, httpAuth, keep_history ' + public function listFeedsOrderUpdate($defaultCacheDuration = 3600) { + $sql = 'SELECT id, url, name, website, lastUpdate, pathEntries, httpAuth, keep_history, ttl ' . 'FROM `' . $this->prefix . 'feed` ' - . 'WHERE lastUpdate < ' . (time() - intval($cacheDuration)) - . ' ORDER BY lastUpdate'; + . 'WHERE ttl <> -1 AND lastUpdate < (' . (time() + 60) . '-(CASE WHEN ttl=-2 THEN ' . intval($defaultCacheDuration) . ' ELSE ttl END)) ' + . 'ORDER BY lastUpdate'; $stm = $this->bd->prepare($sql); - $stm->execute(); + if (!($stm && $stm->execute())) { + $sql2 = 'ALTER TABLE `' . $this->prefix . 'feed` ADD COLUMN ttl INT NOT NULL DEFAULT -2'; //v0.7.3 + $stm = $this->bd->prepare($sql2); + $stm->execute(); + $stm = $this->bd->prepare($sql); + $stm->execute(); + } return self::daoToFeed($stm->fetchAll(PDO::FETCH_ASSOC)); } @@ -365,6 +371,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo { $myFeed->_httpAuth(isset($dao['httpAuth']) ? base64_decode($dao['httpAuth']) : ''); $myFeed->_error(isset($dao['error']) ? $dao['error'] : 0); $myFeed->_keepHistory(isset($dao['keep_history']) ? $dao['keep_history'] : -2); + $myFeed->_ttl(isset($dao['ttl']) ? $dao['ttl'] : -2); $myFeed->_nbNotRead(isset($dao['cache_nbUnreads']) ? $dao['cache_nbUnreads'] : 0); $myFeed->_nbEntries(isset($dao['cache_nbEntries']) ? $dao['cache_nbEntries'] : 0); if (isset($dao['id'])) { diff --git a/app/i18n/en.php b/app/i18n/en.php index 19cf4a06d..569903782 100644 --- a/app/i18n/en.php +++ b/app/i18n/en.php @@ -41,7 +41,7 @@ return array ( 'query_state_15' => 'Display all articles', 'query_number' => 'Query n°%d', 'add_query' => 'Add a query', - 'no_query' => 'You haven’t created user queries yet.', + 'no_query' => 'You haven’t created any user query yet.', 'query_filter' => 'Filter applied:', 'no_query_filter' => 'No filter', 'about' => 'About', @@ -197,6 +197,7 @@ return array ( 'by_feed' => 'by feed', 'by_default' => 'By default', 'keep_history' => 'Minimum number of articles to keep', + 'ttl' => 'Do not automatically refresh more often than', 'categorize' => 'Store in a category', 'truncate' => 'Delete all articles', 'advanced' => 'Advanced', diff --git a/app/i18n/fr.php b/app/i18n/fr.php index 54fe55ea0..5fc60bd15 100644 --- a/app/i18n/fr.php +++ b/app/i18n/fr.php @@ -197,6 +197,7 @@ return array ( 'by_feed' => 'par flux', 'by_default' => 'Par défaut', 'keep_history' => 'Nombre minimum d’articles à conserver', + 'ttl' => 'Ne pas automatiquement rafraîchir plus souvent que', 'categorize' => 'Ranger dans une catégorie', 'truncate' => 'Supprimer tous les articles', 'advanced' => 'Avancé', diff --git a/app/views/configure/archiving.phtml b/app/views/configure/archiving.phtml index e144d0f45..04fa19b0d 100644 --- a/app/views/configure/archiving.phtml +++ b/app/views/configure/archiving.phtml @@ -24,6 +24,27 @@ ?> () +
+ +
+ () +
+
diff --git a/app/views/configure/feed.phtml b/app/views/configure/feed.phtml index 27b0990ff..a8dd9a8cb 100644 --- a/app/views/configure/feed.phtml +++ b/app/views/configure/feed.phtml @@ -103,6 +103,27 @@ ?>
+
+ +
+ +
+
diff --git a/lib/lib_rss.php b/lib/lib_rss.php index 1b1e4b021..7ca611b04 100644 --- a/lib/lib_rss.php +++ b/lib/lib_rss.php @@ -109,7 +109,7 @@ function customSimplePie() { $simplePie = new SimplePie(); $simplePie->set_useragent(Minz_Translate::t('freshrss') . '/' . FRESHRSS_VERSION . ' (' . PHP_OS . '; ' . FRESHRSS_WEBSITE . ') ' . SIMPLEPIE_NAME . '/' . SIMPLEPIE_VERSION); $simplePie->set_cache_location(CACHE_PATH); - $simplePie->set_cache_duration(1500); + $simplePie->set_cache_duration(800); $simplePie->strip_htmltags(array( 'base', 'blink', 'body', 'doctype', 'embed', 'font', 'form', 'frame', 'frameset', 'html', -- cgit v1.2.3