Dynamic OPML (#4407)

* Dynamic OPML draft
#fix https://github.com/FreshRSS/FreshRSS/issues/4191

* Export dynamic OPML
http://opml.org/spec2.opml#1629043127000

* Restart with simpler approach

* Minor revert

* Export dynamic OPML also for single feeds

* Special category type for importing dynamic OPML

* Parameter for excludeMutedFeeds

* Details

* More draft

* i18n

* Fix update

* Draft manual import working

* Working manual refresh

* Draft automatic update

* Working Web refresh + fixes

* Import/export dynamic OPML settings

* Annoying numerous lines in SQL logs

* Fix minor JavaScript error

* Fix auto adding new columns

* Add require

* Add missing 🗲

* Missing space

* Disable adding new feeds to dynamic categories

* Link from import

* i18n typo

* Improve theme icon function

* Fix pink-dark

8 files changed, 141 insertions, 39 deletions

diff --git a/p/api/greader.php b/p/api/greader.php
index 43e3647d1..9a96823d7 100644
--- a/p/api/greader.php
+++ b/p/api/greader.php
@@ -307,8 +307,8 @@ function subscriptionExport() {
 function subscriptionImport($opml) {
 	$user = Minz_Session::param('currentUser', '_');
 	$importService = new FreshRSS_Import_Service($user);
-	$ok = $importService->importOpml($opml);
-	if ($ok) {
+	$importService->importOpml($opml);
+	if ($importService->lastStatus()) {
 		list($nbUpdatedFeeds, $feed, $nbNewArticles) = FreshRSS_feed_Controller::actualizeFeed(0, '', true);
 		invalidateHttpCache($user);
 		exit('OK');
diff --git a/p/scripts/extra.js b/p/scripts/extra.js
index 39f9d049a..7be235aa4 100644
--- a/p/scripts/extra.js
+++ b/p/scripts/extra.js
@@ -202,8 +202,8 @@ function updateHref(ev) {
 }
 
 // set event listener on "show url" buttons
-function init_url_observers() {
-	document.querySelectorAll('.open-url').forEach(function (btn) {
+function init_url_observers(parent) {
+	parent.querySelectorAll('.open-url').forEach(function (btn) {
 		btn.addEventListener('mouseover', updateHref);
 		btn.addEventListener('click', updateHref);
 	});
@@ -276,7 +276,6 @@ function init_extra_afterDOM() {
 	if (!['normal', 'global', 'reader'].includes(context.current_view)) {
 		init_crypto_form();
 		init_password_observers(document.body);
-		init_url_observers();
 		init_select_observers();
 		init_configuration_alert();
 
@@ -284,8 +283,10 @@ function init_extra_afterDOM() {
 		if (slider) {
 			init_slider(slider);
 			init_archiving(slider);
+			init_url_observers(slider);
 		} else {
 			init_archiving(document.body);
+			init_url_observers(document.body);
 		}
 	}
 
diff --git a/p/scripts/feed.js b/p/scripts/feed.js
index 2a213b422..a5e43c614 100644
--- a/p/scripts/feed.js
+++ b/p/scripts/feed.js
@@ -1,6 +1,6 @@
 // @license magnet:?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt AGPL-3.0
 'use strict';
-/* globals init_archiving, init_configuration_alert, init_password_observers, init_slider */
+/* globals init_archiving, init_configuration_alert, init_password_observers, init_slider, init_url_observers */
 
 // <popup>
 let popup = null;
@@ -67,6 +67,22 @@ function init_popup_preview_selector() {
 /**
  * Allow a <select class="select-show"> to hide/show elements defined by <option data-show="elem-id"></option>
  */
+function init_disable_elements_on_update(parent) {
+	const inputs = parent.querySelectorAll('input[data-disable-update]');
+	for (const input of inputs) {
+		input.addEventListener('input', (e) => {
+			const elem = document.getElementById(e.target.dataset.disableUpdate);
+			if (elem) {
+				elem.disabled = true;
+				elem.remove();
+			}
+		});
+	}
+}
+
+/**
+ * Allow a <select class="select-show"> to hide/show elements defined by <option data-show="elem-id"></option>
+ */
 function init_select_show(parent) {
 	const listener = (select) => {
 		const options = select.querySelectorAll('option[data-show]');
@@ -120,7 +136,9 @@ function init_feed_afterDOM() {
 			init_popup();
 			init_popup_preview_selector();
 			init_select_show(slider);
+			init_disable_elements_on_update(slider);
 			init_password_observers(slider);
+			init_url_observers(slider);
 			init_valid_xpath(slider);
 		});
 		init_slider(slider);
@@ -130,6 +148,7 @@ function init_feed_afterDOM() {
 		init_popup();
 		init_popup_preview_selector();
 		init_select_show(document.body);
+		init_disable_elements_on_update(document.body);
 		init_password_observers(document.body);
 		init_valid_xpath(document.body);
 	}
diff --git a/p/scripts/main.js b/p/scripts/main.js
index 1eb8a1ff8..461dc1b10 100644
--- a/p/scripts/main.js
+++ b/p/scripts/main.js
@@ -115,9 +115,10 @@ function incUnreadsFeed(article, feed_id, nb) {
 	}
 
 	// Update unread: category
-	elem = document.getElementById(feed_id).closest('.category');
-	feed_unreads = elem ? str2int(elem.getAttribute('data-unread')) : 0;
+	elem = document.getElementById(feed_id);
+	elem = elem ? elem.closest('.category') : null;
 	if (elem) {
+		feed_unreads = str2int(elem.getAttribute('data-unread'));
 		elem.setAttribute('data-unread', feed_unreads + nb);
 		elem = elem.querySelector('.title');
 		if (elem) {
@@ -147,7 +148,7 @@ function incUnreadsFeed(article, feed_id, nb) {
 	// Update unread: title
 	document.title = document.title.replace(/^((?:\([\s0-9]+\) )?)/, function (m, p1) {
 		const feed = document.getElementById(feed_id);
-		if (article || feed.closest('.active')) {
+		if (article || (feed && feed.closest('.active'))) {
 			isCurrentView = true;
 			return incLabel(p1, nb, true);
 		} else if (document.querySelector('.all.active')) {
@@ -1287,9 +1288,11 @@ function loadDynamicTags(div) {
 }
 
 // <actualize>
-let feed_processed = 0;
+let feeds_processed = 0;
+let categories_processed = 0;
+let to_process = 0;
 
-function updateFeed(feeds, feeds_count) {
+function refreshFeed(feeds, feeds_count) {
 	const feed = feeds.pop();
 	if (!feed) {
 		return;
@@ -1297,14 +1300,15 @@ function updateFeed(feeds, feeds_count) {
 	const req = new XMLHttpRequest();
 	req.open('POST', feed.url, true);
 	req.onloadend = function (e) {
+		feeds_processed++;
 		if (this.status != 200) {
-			return badAjax(false);
+			badAjax(false);
+		} else {
+			const div = document.getElementById('actualizeProgress');
+			div.querySelector('.progress').innerHTML = (categories_processed + feeds_processed) + ' / ' + to_process;
+			div.querySelector('.title').innerHTML = feed.title;
 		}
-		feed_processed++;
-		const div = document.getElementById('actualizeProgress');
-		div.querySelector('.progress').innerHTML = feed_processed + ' / ' + feeds_count;
-		div.querySelector('.title').innerHTML = feed.title;
-		if (feed_processed === feeds_count) {
+		if (feeds_processed === feeds_count) {
 			// Empty request to commit new articles
 			const req2 = new XMLHttpRequest();
 			req2.open('POST', './?c=feed&a=actualize&id=-1&ajax=1', true);
@@ -1317,7 +1321,7 @@ function updateFeed(feeds, feeds_count) {
 				noCommit: 0,
 			}));
 		} else {
-			updateFeed(feeds, feeds_count);
+			refreshFeed(feeds, feeds_count);
 		}
 	};
 	req.setRequestHeader('Content-Type', 'application/json');
@@ -1327,8 +1331,73 @@ function updateFeed(feeds, feeds_count) {
 	}));
 }
 
+function refreshFeeds(json) {
+	feeds_processed = 0;
+	if (!json.feeds || json.feeds.length === 0) {
+		// Empty request to commit new articles
+		const req2 = new XMLHttpRequest();
+		req2.open('POST', './?c=feed&a=actualize&id=-1&ajax=1', true);
+		req2.onloadend = function (e) {
+			context.ajax_loading = false;
+		};
+		req2.setRequestHeader('Content-Type', 'application/json');
+		req2.send(JSON.stringify({
+			_csrf: context.csrf,
+			noCommit: 0,
+		}));
+	} else {
+		const feeds_count = json.feeds.length;
+		for (let i = 10; i > 0; i--) {
+			refreshFeed(json.feeds, feeds_count);
+		}
+	}
+}
+
+function refreshDynamicOpml(categories, categories_count, next) {
+	const category = categories.pop();
+	if (!category) {
+		return;
+	}
+	const req = new XMLHttpRequest();
+	req.open('POST', category.url, true);
+	req.onloadend = function (e) {
+		categories_processed++;
+		if (this.status != 200) {
+			badAjax(false);
+		} else {
+			const div = document.getElementById('actualizeProgress');
+			div.querySelector('.progress').innerHTML = (categories_processed + feeds_processed) + ' / ' + to_process;
+			div.querySelector('.title').innerHTML = category.title;
+		}
+		if (categories_processed === categories_count) {
+			if (next) { next(); }
+		} else {
+			refreshDynamicOpml(categories, categories_count, next);
+		}
+	};
+	req.setRequestHeader('Content-Type', 'application/json');
+	req.send(JSON.stringify({
+		_csrf: context.csrf,
+		noCommit: 1,
+	}));
+}
+
+function refreshDynamicOpmls(json, next) {
+	categories_processed = 0;
+	if (json.categories && json.categories.length > 0) {
+		const categories_count = json.categories.length;
+		for (let i = 10; i > 0; i--) {
+			refreshDynamicOpml(json.categories, categories_count, next);
+		}
+	} else {
+		if (next) { next(); }
+	}
+}
+
 function init_actualize() {
 	let auto = false;
+	let nbCategoriesFirstRound = 0;
+	let skipCategories = false;
 
 	const actualize = document.getElementById('actualize');
 	if (!actualize) {
@@ -1352,33 +1421,29 @@ function init_actualize() {
 			if (!json) {
 				return badAjax(false);
 			}
-			if (auto && json.feeds.length < 1) {
+			if (auto && json.categories.length < 1 && json.feeds.length < 1) {
 				auto = false;
 				context.ajax_loading = false;
 				return false;
 			}
-			if (json.feeds.length === 0) {
+			to_process = json.categories.length + json.feeds.length + nbCategoriesFirstRound;
+			if (json.categories.length + json.feeds.length > 0 && !document.getElementById('actualizeProgress')) {
+				document.body.insertAdjacentHTML('beforeend', '<div id="actualizeProgress" class="notification good">' +
+					json.feedback_actualize + '<br /><span class="title">/</span><br /><span class="progress">0 / ' +
+					to_process + '</span></div>');
+			} else {
 				openNotification(json.feedback_no_refresh, 'good');
-				// Empty request to commit new articles
-				const req2 = new XMLHttpRequest();
-				req2.open('POST', './?c=feed&a=actualize&id=-1&ajax=1', true);
-				req2.onloadend = function (e) {
-					context.ajax_loading = false;
-				};
-				req2.setRequestHeader('Content-Type', 'application/json');
-				req2.send(JSON.stringify({
-					_csrf: context.csrf,
-					noCommit: 0,
-				}));
-				return;
 			}
-			// Progress bar
-			const feeds_count = json.feeds.length;
-			document.body.insertAdjacentHTML('beforeend', '<div id="actualizeProgress" class="notification good">' +
-					json.feedback_actualize + '<br /><span class="title">/</span><br /><span class="progress">0 / ' +
-					feeds_count + '</span></div>');
-			for (let i = 10; i > 0; i--) {
-				updateFeed(json.feeds, feeds_count);
+			if (json.categories.length > 0 && !skipCategories) {
+				skipCategories = true;	// To avoid risk of infinite loop
+				nbCategoriesFirstRound = json.categories.length;
+				// If some dynamic OPML categories are refreshed, need to reload the list of feeds before updating them
+				refreshDynamicOpmls(json, () => {
+					context.ajax_loading = false;
+					actualize.click();
+				});
+			} else {
+				refreshFeeds(json);
 			}
 		};
 		req.setRequestHeader('Content-Type', 'application/json');
diff --git a/p/themes/Dark-pink/pinkdark.css b/p/themes/Dark-pink/pinkdark.css
index b3b10ce5e..e47a997f5 100644
--- a/p/themes/Dark-pink/pinkdark.css
+++ b/p/themes/Dark-pink/pinkdark.css
@@ -117,6 +117,7 @@ input:focus {
 .icon[src*="/sort-up"],
 .icon[src*="/sort-down"],
 .icon[src*="/key"],
+.icon[src*="/opml-dyn"],
 .icon[src*="/configure"],
 .icon[src*="/category"] {
 	/* Color light grey icons */
diff --git a/p/themes/Dark-pink/pinkdark.rtl.css b/p/themes/Dark-pink/pinkdark.rtl.css
index 705c88b36..3ff3cb44d 100644
--- a/p/themes/Dark-pink/pinkdark.rtl.css
+++ b/p/themes/Dark-pink/pinkdark.rtl.css
@@ -117,6 +117,7 @@ input:focus {
 .icon[src*="/sort-up"],
 .icon[src*="/sort-down"],
 .icon[src*="/key"],
+.icon[src*="/opml-dyn"],
 .icon[src*="/configure"],
 .icon[src*="/category"] {
 	/* Color light grey icons */
diff --git a/p/themes/icons/opml-dyn.svg b/p/themes/icons/opml-dyn.svg
new file mode 100644
index 000000000..4c0711367
--- /dev/null
+++ b/p/themes/icons/opml-dyn.svg
@@ -0,0 +1,8 @@
+<svg width="93.619" height="93.619" viewBox="0 0 24.77 24.77" xml:space="preserve" xmlns="http://www.w3.org/2000/svg">
+	<g transform="translate(33.09 -98.68)">
+		<path style="fill:#000;stroke-width:.264583" d="M98.217 120.656V96.711H132.084v47.89H98.217Zm19.925 5.03c5.583-1.455 9.438-6.373 9.438-12.041 0-7.464-6.765-13.365-14.031-12.24-9.125 1.412-13.645 11.61-8.549 19.289 2.769 4.17 8.27 6.26 13.142 4.992zm-5.108-3.822c-1.295-.395-2.793-1.303-3.807-2.306-3.247-3.214-3.255-8.545-.018-11.837 1.622-1.65 3.547-2.412 6.074-2.402 2.59.009 4.158.678 5.999 2.558 1.733 1.77 2.423 3.685 2.272 6.297-.2 3.452-2.21 6.207-5.398 7.4-1.464.547-3.839.682-5.122.29zm3.713-5.181c.92-.476 1.578-1.634 1.578-2.774 0-2.794-3.33-4.117-5.331-2.117-.626.625-.754.977-.754 2.061 0 2.57 2.263 3.99 4.507 2.83z"/>
+		<path style="opacity:1;fill:#666;stroke:none;stroke-width:1.12498;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:2;stroke-opacity:1;paint-order:markers fill stroke" d="M-20.705 98.68a12.385 12.385 0 0 0-12.384 12.385 12.385 12.385 0 0 0 12.384 12.385 12.385 12.385 0 0 0 12.386-12.385A12.385 12.385 0 0 0-20.705 98.68zm0 3.616a8.77 8.77 0 0 1 8.77 8.77 8.77 8.77 0 0 1-8.77 8.77 8.77 8.77 0 0 1-8.77-8.77 8.77 8.77 0 0 1 8.77-8.77z"/>
+		<circle style="opacity:1;fill:#666;stroke:none;stroke-width:.276252;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:2;stroke-opacity:1;paint-order:markers fill stroke" cx="-20.704" cy="111.065" r="3.041"/>
+		<path style="fill:#666;stroke-width:.00362641;fill-opacity:1" d="M-17.936 117.039c-4.746 1.911-6.857-.321-8.826-3.398.058-.03.967-.532 1.018-.558 1.935 2.8 3.752 4.62 7.194 2.712l-.482-.983c.787.2 1.619.415 2.465.64-.322.801-.639 1.604-.966 2.402l-.403-.815zM-23.767 105.435c4.745-1.911 6.856.321 8.825 3.398-.058.03-.966.532-1.017.558-1.935-2.8-3.753-4.62-7.195-2.712l.482.983c-.787-.2-1.618-.415-2.465-.64.322-.801.64-1.604.967-2.402.134.272.267.543.403.815z"/>
+	</g>
+</svg>
diff --git a/p/themes/icons/opml.svg b/p/themes/icons/opml.svg
new file mode 100644
index 000000000..b3e8a830c
--- /dev/null
+++ b/p/themes/icons/opml.svg
@@ -0,0 +1,7 @@
+<svg width="93.619" height="93.619" viewBox="0 0 24.77 24.77" xml:space="preserve" xmlns="http://www.w3.org/2000/svg">
+	<g transform="translate(33.09 -98.68)">
+		<path style="fill:#000;stroke-width:.264583" d="M98.217 120.656V96.711H132.084v47.89H98.217Zm19.925 5.03c5.583-1.455 9.438-6.373 9.438-12.041 0-7.464-6.765-13.365-14.031-12.24-9.125 1.412-13.645 11.61-8.549 19.289 2.769 4.17 8.27 6.26 13.142 4.992zm-5.108-3.822c-1.295-.395-2.793-1.303-3.807-2.306-3.247-3.214-3.255-8.545-.018-11.837 1.622-1.65 3.547-2.412 6.074-2.402 2.59.009 4.158.678 5.999 2.558 1.733 1.77 2.423 3.685 2.272 6.297-.2 3.452-2.21 6.207-5.398 7.4-1.464.547-3.839.682-5.122.29zm3.713-5.181c.92-.476 1.578-1.634 1.578-2.774 0-2.794-3.33-4.117-5.331-2.117-.626.625-.754.977-.754 2.061 0 2.57 2.263 3.99 4.507 2.83z"/>
+		<path style="opacity:1;fill:#666;stroke:none;stroke-width:1.12498;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:2;stroke-opacity:1;paint-order:markers fill stroke" d="M-20.705 98.68a12.385 12.385 0 0 0-12.384 12.385 12.385 12.385 0 0 0 12.384 12.385 12.385 12.385 0 0 0 12.386-12.385A12.385 12.385 0 0 0-20.705 98.68zm0 3.616a8.77 8.77 0 0 1 8.77 8.77 8.77 8.77 0 0 1-8.77 8.77 8.77 8.77 0 0 1-8.77-8.77 8.77 8.77 0 0 1 8.77-8.77z"/>
+		<circle style="opacity:1;fill:#666;stroke:none;stroke-width:.276252;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:2;stroke-opacity:1;paint-order:markers fill stroke" cx="-20.704" cy="111.065" r="3.041"/>
+	</g>
+</svg>