From cc0db9af4f980829faa4bf0960617807b32fb4fa Mon Sep 17 00:00:00 2001 From: Alexis Degrugillier Date: Wed, 23 Oct 2019 00:52:15 +0200 Subject: Feature/new archiving (#2335) * Change archiving config page layout I've changed some wording and moved actions into a maintenance section. * Update purge action Now we have more control on the purge action. The configuration allows us to choose what to keep and what to discard in a more precise way. At the moment, the configuration applies for all feeds. * Add purge configuration on feed level Now the extend purge configuration is available on feed level. It is stored as attributes and will be used in the purge action. * Update purge action Now the purge action uses the feed configuration if it exists and defaults on user configuration if not. * Add empty option in period list * Fix configuration warnings * Add archiving configuration on categories See #2369 * Add user info back * Add explanations in UI * Fixes for SQLite + error + misc. * Fix invalid feed reference * Short array syntax Only for new code, so far * Fix prefix error * Query performance, default values Work in progress * Fix default values and confirm before leaving Form cancel and confirm changes before leaving were broken. And start taking advantage of the short echo syntax `` as we have moved to PHP 5.4+ * More work * Tuning SQL * Fix MariaDB + performance issue * SQL performance * Fix SQLite bug * Fix some attributes JSON encoding bugs Especially for SQLite export/import * More uniform, fix bugs More uniform between global, category, feed settings * Drop special cases for old articles during refresh Instead will use lastSeen date with the new archiving logic. This was generating problems anyway https://github.com/FreshRSS/FreshRSS/issues/2154 * Draft drop index keep_history Not needed anymore * MySQL typo Now properly tested with MySQL, PostgreSQL, SQLite * More work for legacy values Important to avoid overriding user's preference and risking deleting data erroneously * Fix PHP 7.3 / 7.4 warnings @aledeg "Trying to use values of type null, bool, int, float or resource as an array (such as $null["key"]) will now generate a notice. " https://php.net/migration74.incompatible * Reintroduce min articles and take care of legacy parameters * A few changes forgotten * Draft of migration + DROP of feed.keep_history * Fix several errors And give up using const for SQL to allow multiple database types (and we cannot redefine a const) * Add keep_min to categories + factorise archiving logic * Legacy fix * Fix bug yield from * Minor: Use JSON_UNESCAPED_SLASHE for attributes And make more uniform * Fix sign and missing variable * Fine tune the logic --- app/Controllers/configureController.php | 43 ++++++++++++++++++++-- app/Controllers/entryController.php | 20 +++-------- app/Controllers/feedController.php | 24 ++----------- app/Controllers/subscriptionController.php | 58 ++++++++++++++++++++++++++++-- 4 files changed, 101 insertions(+), 44 deletions(-) (limited to 'app/Controllers') diff --git a/app/Controllers/configureController.php b/app/Controllers/configureController.php index 85ca9da39..b38d3289a 100755 --- a/app/Controllers/configureController.php +++ b/app/Controllers/configureController.php @@ -196,9 +196,31 @@ class FreshRSS_configure_Controller extends Minz_ActionController { */ public function archivingAction() { if (Minz_Request::isPost()) { - FreshRSS_Context::$user_conf->old_entries = Minz_Request::param('old_entries', 3); - FreshRSS_Context::$user_conf->keep_history_default = Minz_Request::param('keep_history_default', 0); + if (!Minz_Request::paramBoolean('enable_keep_max')) { + $keepMax = false; + } elseif (!$keepMax = Minz_Request::param('keep_max')) { + $keepMax = FreshRSS_Feed::ARCHIVING_RETENTION_COUNT_LIMIT; + } + if ($enableRetentionPeriod = Minz_Request::paramBoolean('enable_keep_period')) { + $keepPeriod = FreshRSS_Feed::ARCHIVING_RETENTION_PERIOD; + if (is_numeric(Minz_Request::param('keep_period_count')) && preg_match('/^PT?1[YMWDH]$/', Minz_Request::param('keep_period_unit'))) { + $keepPeriod = str_replace('1', Minz_Request::param('keep_period_count'), Minz_Request::param('keep_period_unit')); + } + } else { + $keepPeriod = false; + } + FreshRSS_Context::$user_conf->ttl_default = Minz_Request::param('ttl_default', FreshRSS_Feed::TTL_DEFAULT); + FreshRSS_Context::$user_conf->archiving = [ + 'keep_period' => $keepPeriod, + 'keep_max' => $keepMax, + 'keep_min' => Minz_Request::param('keep_min_default', 0), + 'keep_favourites' => Minz_Request::paramBoolean('keep_favourites'), + 'keep_labels' => Minz_Request::paramBoolean('keep_labels'), + 'keep_unreads' => Minz_Request::paramBoolean('keep_unreads'), + ]; + FreshRSS_Context::$user_conf->keep_history_default = null; //Legacy < FreshRSS 1.15 + FreshRSS_Context::$user_conf->old_entries = null; //Legacy < FreshRSS 1.15 FreshRSS_Context::$user_conf->save(); invalidateHttpCache(); @@ -206,7 +228,20 @@ class FreshRSS_configure_Controller extends Minz_ActionController { array('c' => 'configure', 'a' => 'archiving')); } - Minz_View::prependTitle(_t('conf.archiving.title') . ' · '); + $volatile = [ + 'enable_keep_period' => false, + 'keep_period_count' => '3', + 'keep_period_unit' => 'P1M', + ]; + $keepPeriod = FreshRSS_Context::$user_conf->archiving['keep_period']; + if (preg_match('/^PT?(?P\d+)[YMWDH]$/', $keepPeriod, $matches)) { + $volatile = [ + 'enable_keep_period' => true, + 'keep_period_count' => $matches['count'], + 'keep_period_unit' => str_replace($matches['count'], 1, $keepPeriod), + ]; + } + FreshRSS_Context::$user_conf->volatile = $volatile; $entryDAO = FreshRSS_Factory::createEntryDao(); $this->view->nb_total = $entryDAO->count(); @@ -217,6 +252,8 @@ class FreshRSS_configure_Controller extends Minz_ActionController { if (FreshRSS_Auth::hasAccess('admin')) { $this->view->size_total = $databaseDAO->size(true); } + + Minz_View::prependTitle(_t('conf.archiving.title') . ' · '); } /** diff --git a/app/Controllers/entryController.php b/app/Controllers/entryController.php index 0215128f4..7881cb3ec 100755 --- a/app/Controllers/entryController.php +++ b/app/Controllers/entryController.php @@ -181,32 +181,20 @@ class FreshRSS_entry_Controller extends Minz_ActionController { public function purgeAction() { @set_time_limit(300); - $nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1); - $date_min = time() - (3600 * 24 * 30 * $nb_month_old); - - $entryDAO = FreshRSS_Factory::createEntryDao(); $feedDAO = FreshRSS_Factory::createFeedDao(); $feeds = $feedDAO->listFeeds(); $nb_total = 0; invalidateHttpCache(); - foreach ($feeds as $feed) { - $feed_history = $feed->keepHistory(); - if (FreshRSS_Feed::KEEP_HISTORY_DEFAULT === $feed_history) { - $feed_history = FreshRSS_Context::$user_conf->keep_history_default; - } + $feedDAO->beginTransaction(); - if ($feed_history >= 0) { - $nb = $entryDAO->cleanOldEntries($feed->id(), $date_min, $feed_history); - if ($nb > 0) { - $nb_total += $nb; - Minz_Log::debug($nb . ' old entries cleaned in feed [' . $feed->url(false) . ']'); - } - } + foreach ($feeds as $feed) { + $nb_total += $feed->cleanOldEntries(); } $feedDAO->updateCachedValues(); + $feedDAO->commit(); $databaseDAO = FreshRSS_Factory::createDatabaseDAO(); $databaseDAO->minorDbMaintenance(); diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php index ea07d96e4..aabeb80ff 100755 --- a/app/Controllers/feedController.php +++ b/app/Controllers/feedController.php @@ -267,10 +267,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $maxFeeds = 10; } - // Calculate date of oldest entries we accept in DB. - $nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1); - $date_min = time() - (3600 * 24 * 30 * $nb_month_old); - // WebSub (PubSubHubbub) support $pubsubhubbubEnabledGeneral = FreshRSS_Context::$system_conf->pubsubhubbub_enabled; $pshbMinAge = time() - (3600 * 24); //TODO: Make a configuration. @@ -323,12 +319,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController { continue; } - $feed_history = $feed->keepHistory(); - if ($isNewFeed) { - $feed_history = FreshRSS_Feed::KEEP_HISTORY_INFINITE; - } elseif (FreshRSS_Feed::KEEP_HISTORY_DEFAULT === $feed_history) { - $feed_history = FreshRSS_Context::$user_conf->keep_history_default; - } $needFeedCacheRefresh = false; // We want chronological order and SimplePie uses reverse order. @@ -376,15 +366,9 @@ class FreshRSS_feed_Controller extends Minz_ActionController { } $entryDAO->updateEntry($entry->toArray()); } - } elseif ($feed_history == 0 && $entry_date < $date_min) { - // This entry should not be added considering configuration and date. - $oldGuids[] = $entry->guid(); } else { $id = uTimeString(); $entry->_id($id); - if ($entry_date < $date_min) { - $entry->_isRead(true); //Old article that was not in database. Probably an error, so mark as read - } $entry->applyFilterActions(); @@ -413,17 +397,13 @@ class FreshRSS_feed_Controller extends Minz_ActionController { $entryDAO->updateLastSeen($feed->id(), $oldGuids, $mtime); } - if ($feed_history >= 0 && mt_rand(0, 30) === 1) { - // TODO: move this function in web cron when available (see entry::purge) - // Remove old entries once in 30. + if (mt_rand(0, 30) === 1) { // Remove old entries once in 30. if (!$entryDAO->inTransaction()) { $entryDAO->beginTransaction(); } - - $nb = $entryDAO->cleanOldEntries($feed->id(), $date_min, max($feed_history, count($entries) + 10)); + $nb = $feed->cleanOldEntries(); if ($nb > 0) { $needFeedCacheRefresh = true; - Minz_Log::debug($nb . ' old entries cleaned in feed [' . $feed->url(false) . ']'); } } diff --git a/app/Controllers/subscriptionController.php b/app/Controllers/subscriptionController.php index f6d5e9457..f9497f0be 100644 --- a/app/Controllers/subscriptionController.php +++ b/app/Controllers/subscriptionController.php @@ -121,6 +121,32 @@ class FreshRSS_subscription_Controller extends Minz_ActionController { $feed->_attributes('timeout', null); } + if (Minz_Request::paramBoolean('use_default_purge_options')) { + $feed->_attributes('archiving', null); + } else { + if (!Minz_Request::paramBoolean('enable_keep_max')) { + $keepMax = false; + } elseif (!$keepMax = Minz_Request::param('keep_max')) { + $keepMax = FreshRSS_Feed::ARCHIVING_RETENTION_COUNT_LIMIT; + } + if ($enableRetentionPeriod = Minz_Request::paramBoolean('enable_keep_period')) { + $keepPeriod = FreshRSS_Feed::ARCHIVING_RETENTION_PERIOD; + if (is_numeric(Minz_Request::param('keep_period_count')) && preg_match('/^PT?1[YMWDH]$/', Minz_Request::param('keep_period_unit'))) { + $keepPeriod = str_replace(1, Minz_Request::param('keep_period_count'), Minz_Request::param('keep_period_unit')); + } + } else { + $keepPeriod = false; + } + $feed->_attributes('archiving', [ + 'keep_period' => $keepPeriod, + 'keep_max' => $keepMax, + 'keep_min' => intval(Minz_Request::param('keep_min', 0)), + 'keep_favourites' => Minz_Request::paramBoolean('keep_favourites'), + 'keep_labels' => Minz_Request::paramBoolean('keep_labels'), + 'keep_unreads' => Minz_Request::paramBoolean('keep_unreads'), + ]); + } + $feed->_filtersAction('read', preg_split('/[\n\r]+/', Minz_Request::param('filteractions_read', ''))); $values = array( @@ -132,7 +158,6 @@ class FreshRSS_subscription_Controller extends Minz_ActionController { 'pathEntries' => Minz_Request::param('path_entries', ''), 'priority' => intval(Minz_Request::param('priority', FreshRSS_Feed::PRIORITY_MAIN_STREAM)), 'httpAuth' => $httpAuth, - 'keep_history' => intval(Minz_Request::param('keep_history', FreshRSS_Feed::KEEP_HISTORY_DEFAULT)), 'ttl' => $ttl * ($mute ? -1 : 1), 'attributes' => $feed->attributes(), ); @@ -165,9 +190,36 @@ class FreshRSS_subscription_Controller extends Minz_ActionController { $this->view->category = $category; if (Minz_Request::isPost()) { - $values = array( + if (Minz_Request::paramBoolean('use_default_purge_options')) { + $category->_attributes('archiving', null); + } else { + if (!Minz_Request::paramBoolean('enable_keep_max')) { + $keepMax = false; + } elseif (!$keepMax = Minz_Request::param('keep_max')) { + $keepMax = FreshRSS_Feed::ARCHIVING_RETENTION_COUNT_LIMIT; + } + if ($enableRetentionPeriod = Minz_Request::paramBoolean('enable_keep_period')) { + $keepPeriod = FreshRSS_Feed::ARCHIVING_RETENTION_PERIOD; + if (is_numeric(Minz_Request::param('keep_period_count')) && preg_match('/^PT?1[YMWDH]$/', Minz_Request::param('keep_period_unit'))) { + $keepPeriod = str_replace(1, Minz_Request::param('keep_period_count'), Minz_Request::param('keep_period_unit')); + } + } else { + $keepPeriod = false; + } + $category->_attributes('archiving', [ + 'keep_period' => $keepPeriod, + 'keep_max' => $keepMax, + 'keep_min' => intval(Minz_Request::param('keep_min', 0)), + 'keep_favourites' => Minz_Request::paramBoolean('keep_favourites'), + 'keep_labels' => Minz_Request::paramBoolean('keep_labels'), + 'keep_unreads' => Minz_Request::paramBoolean('keep_unreads'), + ]); + } + + $values = [ 'name' => Minz_Request::param('name', ''), - ); + 'attributes' => $category->attributes(), + ]; invalidateHttpCache(); -- cgit v1.2.3