JSONFeeds, JSON scraping, and POST requests for feeds (#5662)

* allow POST requests for feeds

* added json dotpath and jsonfeed subscriptions. No translation strings yet

* debug and fix jsonfeed parser

* bugfix params saved when editing feed

* added translations for JSON features

* Update docs for web scraping

* make fix-all
and revert unrelated changes, plus a few manual fixes, but there are still several type errors

* Fix some i18n

* refactor json parsing for both feed types

* cleanup unnecessary comment

* refactored generation of SimplePie for XPath and JSON feeds

* Fix merge error

* Update to newer FreshRSS code

* A bit of refactoring

* doc, whitespace

* JSON Feed is in two words

* Add support for array syntax

* Whitespace

* Add OPML export/import

* Work on i18n

* Accept application/feed+json

* Rework POST

* Fix update

* OPML for cURL options

* Fix types

* Fix Typos

---------

Co-authored-by: Erion Elmasllari <elmasllari@factorsixty.com>
Co-authored-by: Alexandre Alapetite <alexandre@alapetite.fr>

2 files changed, 162 insertions, 11 deletions

diff --git a/app/views/helpers/export/opml.phtml b/app/views/helpers/export/opml.phtml
index ce53bfc02..736854f46 100644
--- a/app/views/helpers/export/opml.phtml
+++ b/app/views/helpers/export/opml.phtml
@@ -3,7 +3,7 @@ declare(strict_types=1);
 
 /**
  * @param array<FreshRSS_Feed> $feeds
- * @return array<array<string,string|null>>
+ * @return array<array<string,string|bool|int>>
  */
 function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array {
 	$outlines = [];
@@ -20,15 +20,22 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array {
 			'description' => htmlspecialchars_decode($feed->description(), ENT_QUOTES),
 		];
 
+		switch ($feed->kind()) {
+			case FreshRSS_Feed::KIND_HTML_XPATH:
+				$outline['type'] = FreshRSS_Export_Service::TYPE_HTML_XPATH;
+				break;
+			case FreshRSS_Feed::KIND_XML_XPATH:
+				$outline['type'] = FreshRSS_Export_Service::TYPE_XML_XPATH;
+				break;
+			case FreshRSS_Feed::KIND_JSON_DOTPATH:
+				$outline['type'] = FreshRSS_Export_Service::TYPE_JSON_DOTPATH;
+				break;
+			case FreshRSS_Feed::KIND_JSONFEED:
+				$outline['type'] = FreshRSS_Export_Service::TYPE_JSONFEED;
+				break;
+		}
+
 		if ($feed->kind() === FreshRSS_Feed::KIND_HTML_XPATH || $feed->kind() === FreshRSS_Feed::KIND_XML_XPATH) {
-			switch ($feed->kind()) {
-				case FreshRSS_Feed::KIND_HTML_XPATH:
-					$outline['type'] = FreshRSS_Export_Service::TYPE_HTML_XPATH;
-					break;
-				case FreshRSS_Feed::KIND_XML_XPATH:
-					$outline['type'] = FreshRSS_Export_Service::TYPE_XML_XPATH;
-					break;
-			}
 			/** @var array<string,string> */
 			$xPathSettings = $feed->attributeArray('xpath') ?? [];
 			$outline['frss:xPathItem'] = $xPathSettings['item'] ?? null;
@@ -41,6 +48,19 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array {
 			$outline['frss:xPathItemThumbnail'] = $xPathSettings['itemThumbnail'] ?? null;
 			$outline['frss:xPathItemCategories'] = $xPathSettings['itemCategories'] ?? null;
 			$outline['frss:xPathItemUid'] = $xPathSettings['itemUid'] ?? null;
+		} elseif ($feed->kind() === FreshRSS_Feed::KIND_JSON_DOTPATH) {
+			/** @var array<string,string> */
+			$jsonSettings = $feed->attributeArray('json_dotpath') ?? [];
+			$outline['frss:jsonItem'] = $jsonSettings['item'] ?? null;
+			$outline['frss:jsonItemTitle'] = $jsonSettings['itemTitle'] ?? null;
+			$outline['frss:jsonItemContent'] = $jsonSettings['itemContent'] ?? null;
+			$outline['frss:jsonItemUri'] = $jsonSettings['itemUri'] ?? null;
+			$outline['frss:jsonItemAuthor'] = $jsonSettings['itemAuthor'] ?? null;
+			$outline['frss:jsonItemTimestamp'] = $jsonSettings['itemTimestamp'] ?? null;
+			$outline['frss:jsonItemTimeformat'] = $jsonSettings['itemTimeformat'] ?? null;
+			$outline['frss:jsonItemThumbnail'] = $jsonSettings['itemThumbnail'] ?? null;
+			$outline['frss:jsonItemCategories'] = $jsonSettings['itemCategories'] ?? null;
+			$outline['frss:jsonItemUid'] = $jsonSettings['itemUid'] ?? null;
 		}
 
 		if (!empty($feed->filtersAction('read'))) {
@@ -60,8 +80,30 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array {
 			$outline['frss:cssFullContentFilter'] = $feed->attributeString('path_entries_filter');
 		}
 
-		// Remove null attributes
-		$outline = array_filter($outline, static function (?string $value) { return $value !== null; });
+		$curl_params = $feed->attributeArray('curl_params');
+		if (!empty($curl_params)) {
+			$outline['frss:CURLOPT_COOKIE'] = $curl_params[CURLOPT_COOKIE] ?? null;
+			$outline['frss:CURLOPT_COOKIEFILE'] = $curl_params[CURLOPT_COOKIEFILE] ?? null;
+			$outline['frss:CURLOPT_FOLLOWLOCATION'] = $curl_params[CURLOPT_FOLLOWLOCATION] ?? null;
+			$outline['frss:CURLOPT_MAXREDIRS'] = $curl_params[CURLOPT_MAXREDIRS] ?? null;
+			$outline['frss:CURLOPT_POST'] = $curl_params[CURLOPT_POST] ?? null;
+			$outline['frss:CURLOPT_POSTFIELDS'] = $curl_params[CURLOPT_POSTFIELDS] ?? null;
+			$outline['frss:CURLOPT_PROXY'] = $curl_params[CURLOPT_PROXY] ?? null;
+			$outline['frss:CURLOPT_PROXYTYPE'] = $curl_params[CURLOPT_PROXYTYPE] ?? null;
+			$outline['frss:CURLOPT_USERAGENT'] = $curl_params[CURLOPT_USERAGENT] ?? null;
+
+			if (!empty($curl_params[CURLOPT_HTTPHEADER]) && is_array($curl_params[CURLOPT_HTTPHEADER])) {
+				$headers = '';
+				foreach ($curl_params[CURLOPT_HTTPHEADER] as $header) {
+					$headers .= $header . "\n";
+				}
+				$headers = trim($headers);
+				$outline['frss:CURLOPT_HTTPHEADER'] = $headers;
+			}
+		}
+
+		// Remove null or invalid attributes
+		$outline = array_filter($outline, static function (mixed $value) { return (is_string($value) || is_int($value) || is_bool($value)) && $value !== ''; });
 
 		$outlines[] = $outline;
 	}
diff --git a/app/views/helpers/feed/update.phtml b/app/views/helpers/feed/update.phtml
index 5d4f1cc4b..13a751c09 100644
--- a/app/views/helpers/feed/update.phtml
+++ b/app/views/helpers/feed/update.phtml
@@ -407,6 +407,8 @@
 					<option value="<?= FreshRSS_Feed::KIND_RSS ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_RSS ? 'selected="selected"' : '' ?>><?= _t('sub.feed.kind.rss') ?></option>
 					<option value="<?= FreshRSS_Feed::KIND_HTML_XPATH ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_HTML_XPATH ? 'selected="selected"' : '' ?> data-show="html_xpath"><?= _t('sub.feed.kind.html_xpath') ?></option>
 					<option value="<?= FreshRSS_Feed::KIND_XML_XPATH ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_XML_XPATH ? 'selected="selected"' : '' ?> data-show="html_xpath"><?= _t('sub.feed.kind.xml_xpath') ?></option>
+					<option value="<?= FreshRSS_Feed::KIND_JSONFEED ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_JSONFEED ? 'selected="selected"' : '' ?>><?= _t('sub.feed.kind.jsonfeed') ?></option>
+					<option value="<?= FreshRSS_Feed::KIND_JSON_DOTPATH ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_JSON_DOTPATH ? 'selected="selected"' : '' ?> data-show="json_dotpath"><?= _t('sub.feed.kind.json_dotpath') ?></option>
 				</select>
 			</div>
 		</div>
@@ -505,6 +507,90 @@
 				</div>
 			</div>
 		</fieldset>
+
+		<fieldset id="json_dotpath">
+			<?php
+				$jsonSettings = Minz_Helper::htmlspecialchars_utf8($this->feed->attributeArray('json_dotpath') ?? []);
+			?>
+			<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.help') ?></p>
+			<div class="form-group">
+				<label class="group-name" for="jsonItem"><small><?= _t('sub.feed.kind.json_dotpath.json') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItem" id="jsonItem" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['item'] ?? '' ?>"><?= $jsonSettings['item'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemTitle"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_title') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemTitle" id="jsonItemTitle" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemTitle'] ?? '' ?>"><?= $jsonSettings['itemTitle'] ?? '' ?></textarea>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemContent"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_content') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemContent" id="jsonItemContent" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemContent'] ?? '' ?>"><?= $jsonSettings['itemContent'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_content.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemUri"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_uri') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemUri" id="jsonItemUri" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemUri'] ?? '' ?>"><?= $jsonSettings['itemUri'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_uri.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemThumbnail"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_thumbnail') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemThumbnail" id="jsonItemThumbnail" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemThumbnail'] ?? '' ?>"><?= $jsonSettings['itemThumbnail'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_thumbnail.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemAuthor"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_author') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemAuthor" id="jsonItemAuthor" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemAuthor'] ?? '' ?>"><?= $jsonSettings['itemAuthor'] ?? '' ?></textarea>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemTimestamp"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_timestamp') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemTimestamp" id="jsonItemTimestamp" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemTimestamp'] ?? '' ?>"><?= $jsonSettings['itemTimestamp'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_timestamp.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemTimeFormat">
+					<?= _t('sub.feed.kind.json_dotpath.item_timeFormat') ?></label>
+				<div class="group-controls">
+					<textarea class="w100" name="jsonItemTimeFormat" id="jsonItemTimeFormat" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemTimeFormat'] ?? '' ?>"><?= $jsonSettings['itemTimeFormat'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_timeFormat.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemCategories"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_categories') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemCategories" id="jsonItemCategories" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemCategories'] ?? '' ?>"><?= $jsonSettings['itemCategories'] ?? '' ?></textarea>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemUid"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_uid') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemUid" id="jsonItemUid" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemUid'] ?? '' ?>"><?= $jsonSettings['itemUid'] ?? '' ?></textarea>
+				</div>
+			</div>
+		</fieldset>
+
 		<div class="form-group form-actions">
 			<div class="group-controls">
 				<button class="btn btn-important"><?= _t('gen.action.submit') ?></button>
@@ -614,6 +700,29 @@
 		</div>
 
 		<div class="form-group">
+			<label class="group-name" for="curl_method"><?= _t('sub.feed.method') ?></label>
+			<div class="group-controls">
+				<select class="number" name="curl_method" id="curl_method"><?php
+				$curl_method = 'GET';
+				if ($this->feed->attributeArray('curl_params') !== null && !empty($this->feed->attributeArray('curl_params')[CURLOPT_POST])) {
+					$curl_method = 'POST';
+				}
+				foreach (['GET' => 'GET', 'POST' => 'POST'] as $k => $v) {
+					echo '<option value="' . $k . ($curl_method === $k ? '" selected="selected' : '') . '">' . $v . '</option>';
+				}
+				?>
+				</select>
+				<div class="stick">
+					<input type="text" name="curl_fields" id="curl_fields" value="<?=
+						$this->feed->attributeArray('curl_params') !== null && !empty($this->feed->attributeArray('curl_params')[CURLOPT_POSTFIELDS]) ?
+							htmlentities($this->feed->attributeArray('curl_params')[CURLOPT_POSTFIELDS], ENT_COMPAT) : ''
+					?>" placeholder="<?= _t('sub.feed.method_postparams') ?>" />
+				</div>
+				<p class="help"><?= _i('help') ?> <?= _t('sub.feed.method_help') ?></p>
+			</div>
+		</div>
+
+		<div class="form-group">
 			<label class="group-name" for="timeout"><?= _t('sub.feed.timeout') ?></label>
 			<div class="group-controls">
 				<input type="number" name="timeout" id="timeout" class="w50" min="3" max="900" value="<?= $this->feed->attributeInt('timeout') ?>" placeholder="<?= _t('gen.short.by_default') ?>" />