Feature/new archiving (#2335)

* Change archiving config page layout

I've changed some wording and moved actions into a
maintenance section.

* Update purge action

Now we have more control on the purge action. The configuration allows
us to choose what to keep and what to discard in a more precise way.
At the moment, the configuration applies for all feeds.

* Add purge configuration on feed level

Now the extend purge configuration is available on feed level.
It is stored as attributes and will be used in the purge action.

* Update purge action

Now the purge action uses the feed configuration if it exists and
defaults on user configuration if not.

* Add empty option in period list

* Fix configuration warnings

* Add archiving configuration on categories

See #2369

* Add user info back

* Add explanations in UI

* Fixes for SQLite + error + misc.

* Fix invalid feed reference

* Short array syntax

Only for new code, so far

* Fix prefix error

* Query performance, default values

Work in progress

* Fix default values and confirm before leaving

Form cancel and confirm changes before leaving were broken.
And start taking advantage of the short echo syntax `<?= ?>` as we have
moved to PHP 5.4+

* More work

* Tuning SQL

* Fix MariaDB + performance issue

* SQL performance

* Fix SQLite bug

* Fix some attributes JSON encoding bugs

Especially for SQLite export/import

* More uniform, fix bugs

More uniform between global, category, feed settings

* Drop special cases for old articles during refresh

Instead will use lastSeen date with the new archiving logic.
This was generating problems anyway
https://github.com/FreshRSS/FreshRSS/issues/2154

* Draft drop index keep_history

Not needed anymore

* MySQL typo

Now properly tested with MySQL, PostgreSQL, SQLite

* More work for legacy values

Important to avoid overriding user's preference and risking deleting
data erroneously

* Fix PHP 7.3 / 7.4 warnings

@aledeg "Trying to use values of type null, bool, int, float or resource
as an array (such as $null["key"]) will now generate a notice. "
https://php.net/migration74.incompatible

* Reintroduce min articles and take care of legacy parameters

* A few changes forgotten

* Draft of migration + DROP of feed.keep_history

* Fix several errors

And give up using const for SQL to allow multiple database types (and we
cannot redefine a const)

* Add keep_min to categories + factorise archiving logic

* Legacy fix

* Fix bug yield from

* Minor: Use JSON_UNESCAPED_SLASHE for attributes

And make more uniform

* Fix sign and missing variable

* Fine tune the logic

4 files changed, 101 insertions, 44 deletions

diff --git a/app/Controllers/configureController.php b/app/Controllers/configureController.php
index 85ca9da39..b38d3289a 100755
--- a/app/Controllers/configureController.php
+++ b/app/Controllers/configureController.php
@@ -196,9 +196,31 @@ class FreshRSS_configure_Controller extends Minz_ActionController {
 	 */
 	public function archivingAction() {
 		if (Minz_Request::isPost()) {
-			FreshRSS_Context::$user_conf->old_entries = Minz_Request::param('old_entries', 3);
-			FreshRSS_Context::$user_conf->keep_history_default = Minz_Request::param('keep_history_default', 0);
+			if (!Minz_Request::paramBoolean('enable_keep_max')) {
+				$keepMax = false;
+			} elseif (!$keepMax = Minz_Request::param('keep_max')) {
+				$keepMax = FreshRSS_Feed::ARCHIVING_RETENTION_COUNT_LIMIT;
+			}
+			if ($enableRetentionPeriod = Minz_Request::paramBoolean('enable_keep_period')) {
+				$keepPeriod = FreshRSS_Feed::ARCHIVING_RETENTION_PERIOD;
+				if (is_numeric(Minz_Request::param('keep_period_count')) && preg_match('/^PT?1[YMWDH]$/', Minz_Request::param('keep_period_unit'))) {
+					$keepPeriod = str_replace('1', Minz_Request::param('keep_period_count'), Minz_Request::param('keep_period_unit'));
+				}
+			} else {
+				$keepPeriod = false;
+			}
+
 			FreshRSS_Context::$user_conf->ttl_default = Minz_Request::param('ttl_default', FreshRSS_Feed::TTL_DEFAULT);
+			FreshRSS_Context::$user_conf->archiving = [
+				'keep_period' => $keepPeriod,
+				'keep_max' => $keepMax,
+				'keep_min' => Minz_Request::param('keep_min_default', 0),
+				'keep_favourites' => Minz_Request::paramBoolean('keep_favourites'),
+				'keep_labels' => Minz_Request::paramBoolean('keep_labels'),
+				'keep_unreads' => Minz_Request::paramBoolean('keep_unreads'),
+			];
+			FreshRSS_Context::$user_conf->keep_history_default = null;	//Legacy < FreshRSS 1.15
+			FreshRSS_Context::$user_conf->old_entries = null;	//Legacy < FreshRSS 1.15
 			FreshRSS_Context::$user_conf->save();
 			invalidateHttpCache();
 
@@ -206,7 +228,20 @@ class FreshRSS_configure_Controller extends Minz_ActionController {
 			                   array('c' => 'configure', 'a' => 'archiving'));
 		}
 
-		Minz_View::prependTitle(_t('conf.archiving.title') . ' · ');
+		$volatile = [
+				'enable_keep_period' => false,
+				'keep_period_count' => '3',
+				'keep_period_unit' => 'P1M',
+			];
+		$keepPeriod = FreshRSS_Context::$user_conf->archiving['keep_period'];
+		if (preg_match('/^PT?(?P<count>\d+)[YMWDH]$/', $keepPeriod, $matches)) {
+			$volatile = [
+				'enable_keep_period' => true,
+				'keep_period_count' => $matches['count'],
+				'keep_period_unit' => str_replace($matches['count'], 1, $keepPeriod),
+			];
+		}
+		FreshRSS_Context::$user_conf->volatile = $volatile;
 
 		$entryDAO = FreshRSS_Factory::createEntryDao();
 		$this->view->nb_total = $entryDAO->count();
@@ -217,6 +252,8 @@ class FreshRSS_configure_Controller extends Minz_ActionController {
 		if (FreshRSS_Auth::hasAccess('admin')) {
 			$this->view->size_total = $databaseDAO->size(true);
 		}
+
+		Minz_View::prependTitle(_t('conf.archiving.title') . ' · ');
 	}
 
 	/**
diff --git a/app/Controllers/entryController.php b/app/Controllers/entryController.php
index 0215128f4..7881cb3ec 100755
--- a/app/Controllers/entryController.php
+++ b/app/Controllers/entryController.php
@@ -181,32 +181,20 @@ class FreshRSS_entry_Controller extends Minz_ActionController {
 	public function purgeAction() {
 		@set_time_limit(300);
 
-		$nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1);
-		$date_min = time() - (3600 * 24 * 30 * $nb_month_old);
-
-		$entryDAO = FreshRSS_Factory::createEntryDao();
 		$feedDAO = FreshRSS_Factory::createFeedDao();
 		$feeds = $feedDAO->listFeeds();
 		$nb_total = 0;
 
 		invalidateHttpCache();
 
-		foreach ($feeds as $feed) {
-			$feed_history = $feed->keepHistory();
-			if (FreshRSS_Feed::KEEP_HISTORY_DEFAULT === $feed_history) {
-				$feed_history = FreshRSS_Context::$user_conf->keep_history_default;
-			}
+		$feedDAO->beginTransaction();
 
-			if ($feed_history >= 0) {
-				$nb = $entryDAO->cleanOldEntries($feed->id(), $date_min, $feed_history);
-				if ($nb > 0) {
-					$nb_total += $nb;
-					Minz_Log::debug($nb . ' old entries cleaned in feed [' . $feed->url(false) . ']');
-				}
-			}
+		foreach ($feeds as $feed) {
+			$nb_total += $feed->cleanOldEntries();
 		}
 
 		$feedDAO->updateCachedValues();
+		$feedDAO->commit();
 
 		$databaseDAO = FreshRSS_Factory::createDatabaseDAO();
 		$databaseDAO->minorDbMaintenance();
diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index ea07d96e4..aabeb80ff 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -267,10 +267,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 			$maxFeeds = 10;
 		}
 
-		// Calculate date of oldest entries we accept in DB.
-		$nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1);
-		$date_min = time() - (3600 * 24 * 30 * $nb_month_old);
-
 		// WebSub (PubSubHubbub) support
 		$pubsubhubbubEnabledGeneral = FreshRSS_Context::$system_conf->pubsubhubbub_enabled;
 		$pshbMinAge = time() - (3600 * 24);  //TODO: Make a configuration.
@@ -323,12 +319,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				continue;
 			}
 
-			$feed_history = $feed->keepHistory();
-			if ($isNewFeed) {
-				$feed_history = FreshRSS_Feed::KEEP_HISTORY_INFINITE;
-			} elseif (FreshRSS_Feed::KEEP_HISTORY_DEFAULT === $feed_history) {
-				$feed_history = FreshRSS_Context::$user_conf->keep_history_default;
-			}
 			$needFeedCacheRefresh = false;
 
 			// We want chronological order and SimplePie uses reverse order.
@@ -376,15 +366,9 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 							}
 							$entryDAO->updateEntry($entry->toArray());
 						}
-					} elseif ($feed_history == 0 && $entry_date < $date_min) {
-						// This entry should not be added considering configuration and date.
-						$oldGuids[] = $entry->guid();
 					} else {
 						$id = uTimeString();
 						$entry->_id($id);
-						if ($entry_date < $date_min) {
-							$entry->_isRead(true);	//Old article that was not in database. Probably an error, so mark as read
-						}
 
 						$entry->applyFilterActions();
 
@@ -413,17 +397,13 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				$entryDAO->updateLastSeen($feed->id(), $oldGuids, $mtime);
 			}
 
-			if ($feed_history >= 0 && mt_rand(0, 30) === 1) {
-				// TODO: move this function in web cron when available (see entry::purge)
-				// Remove old entries once in 30.
+			if (mt_rand(0, 30) === 1) {	// Remove old entries once in 30.
 				if (!$entryDAO->inTransaction()) {
 					$entryDAO->beginTransaction();
 				}
-
-				$nb = $entryDAO->cleanOldEntries($feed->id(), $date_min, max($feed_history, count($entries) + 10));
+				$nb = $feed->cleanOldEntries();
 				if ($nb > 0) {
 					$needFeedCacheRefresh = true;
-					Minz_Log::debug($nb . ' old entries cleaned in feed [' . $feed->url(false) . ']');
 				}
 			}
 
diff --git a/app/Controllers/subscriptionController.php b/app/Controllers/subscriptionController.php
index f6d5e9457..f9497f0be 100644
--- a/app/Controllers/subscriptionController.php
+++ b/app/Controllers/subscriptionController.php
@@ -121,6 +121,32 @@ class FreshRSS_subscription_Controller extends Minz_ActionController {
 				$feed->_attributes('timeout', null);
 			}
 
+			if (Minz_Request::paramBoolean('use_default_purge_options')) {
+				$feed->_attributes('archiving', null);
+			} else {
+				if (!Minz_Request::paramBoolean('enable_keep_max')) {
+					$keepMax = false;
+				} elseif (!$keepMax = Minz_Request::param('keep_max')) {
+					$keepMax = FreshRSS_Feed::ARCHIVING_RETENTION_COUNT_LIMIT;
+				}
+				if ($enableRetentionPeriod = Minz_Request::paramBoolean('enable_keep_period')) {
+					$keepPeriod = FreshRSS_Feed::ARCHIVING_RETENTION_PERIOD;
+					if (is_numeric(Minz_Request::param('keep_period_count')) && preg_match('/^PT?1[YMWDH]$/', Minz_Request::param('keep_period_unit'))) {
+						$keepPeriod = str_replace(1, Minz_Request::param('keep_period_count'), Minz_Request::param('keep_period_unit'));
+					}
+				} else {
+					$keepPeriod = false;
+				}
+				$feed->_attributes('archiving', [
+					'keep_period' => $keepPeriod,
+					'keep_max' => $keepMax,
+					'keep_min' => intval(Minz_Request::param('keep_min', 0)),
+					'keep_favourites' => Minz_Request::paramBoolean('keep_favourites'),
+					'keep_labels' => Minz_Request::paramBoolean('keep_labels'),
+					'keep_unreads' => Minz_Request::paramBoolean('keep_unreads'),
+				]);
+			}
+
 			$feed->_filtersAction('read', preg_split('/[\n\r]+/', Minz_Request::param('filteractions_read', '')));
 
 			$values = array(
@@ -132,7 +158,6 @@ class FreshRSS_subscription_Controller extends Minz_ActionController {
 				'pathEntries' => Minz_Request::param('path_entries', ''),
 				'priority' => intval(Minz_Request::param('priority', FreshRSS_Feed::PRIORITY_MAIN_STREAM)),
 				'httpAuth' => $httpAuth,
-				'keep_history' => intval(Minz_Request::param('keep_history', FreshRSS_Feed::KEEP_HISTORY_DEFAULT)),
 				'ttl' => $ttl * ($mute ? -1 : 1),
 				'attributes' => $feed->attributes(),
 			);
@@ -165,9 +190,36 @@ class FreshRSS_subscription_Controller extends Minz_ActionController {
 		$this->view->category = $category;
 
 		if (Minz_Request::isPost()) {
-			$values = array(
+			if (Minz_Request::paramBoolean('use_default_purge_options')) {
+				$category->_attributes('archiving', null);
+			} else {
+				if (!Minz_Request::paramBoolean('enable_keep_max')) {
+					$keepMax = false;
+				} elseif (!$keepMax = Minz_Request::param('keep_max')) {
+					$keepMax = FreshRSS_Feed::ARCHIVING_RETENTION_COUNT_LIMIT;
+				}
+				if ($enableRetentionPeriod = Minz_Request::paramBoolean('enable_keep_period')) {
+					$keepPeriod = FreshRSS_Feed::ARCHIVING_RETENTION_PERIOD;
+					if (is_numeric(Minz_Request::param('keep_period_count')) && preg_match('/^PT?1[YMWDH]$/', Minz_Request::param('keep_period_unit'))) {
+						$keepPeriod = str_replace(1, Minz_Request::param('keep_period_count'), Minz_Request::param('keep_period_unit'));
+					}
+				} else {
+					$keepPeriod = false;
+				}
+				$category->_attributes('archiving', [
+					'keep_period' => $keepPeriod,
+					'keep_max' => $keepMax,
+					'keep_min' => intval(Minz_Request::param('keep_min', 0)),
+					'keep_favourites' => Minz_Request::paramBoolean('keep_favourites'),
+					'keep_labels' => Minz_Request::paramBoolean('keep_labels'),
+					'keep_unreads' => Minz_Request::paramBoolean('keep_unreads'),
+				]);
+			}
+
+			$values = [
 				'name' => Minz_Request::param('name', ''),
-			);
+				'attributes' => $category->attributes(),
+			];
 
 			invalidateHttpCache();