JSONFeeds, JSON scraping, and POST requests for feeds (#5662)

* allow POST requests for feeds

* added json dotpath and jsonfeed subscriptions. No translation strings yet

* debug and fix jsonfeed parser

* bugfix params saved when editing feed

* added translations for JSON features

* Update docs for web scraping

* make fix-all
and revert unrelated changes, plus a few manual fixes, but there are still several type errors

* Fix some i18n

* refactor json parsing for both feed types

* cleanup unnecessary comment

* refactored generation of SimplePie for XPath and JSON feeds

* Fix merge error

* Update to newer FreshRSS code

* A bit of refactoring

* doc, whitespace

* JSON Feed is in two words

* Add support for array syntax

* Whitespace

* Add OPML export/import

* Work on i18n

* Accept application/feed+json

* Rework POST

* Fix update

* OPML for cURL options

* Fix types

* Fix Typos

---------

Co-authored-by: Erion Elmasllari <elmasllari@factorsixty.com>
Co-authored-by: Alexandre Alapetite <alexandre@alapetite.fr>

1 files changed, 109 insertions, 0 deletions

diff --git a/app/views/helpers/feed/update.phtml b/app/views/helpers/feed/update.phtml
index 5d4f1cc4b..13a751c09 100644
--- a/app/views/helpers/feed/update.phtml
+++ b/app/views/helpers/feed/update.phtml
@@ -407,6 +407,8 @@
 					<option value="<?= FreshRSS_Feed::KIND_RSS ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_RSS ? 'selected="selected"' : '' ?>><?= _t('sub.feed.kind.rss') ?></option>
 					<option value="<?= FreshRSS_Feed::KIND_HTML_XPATH ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_HTML_XPATH ? 'selected="selected"' : '' ?> data-show="html_xpath"><?= _t('sub.feed.kind.html_xpath') ?></option>
 					<option value="<?= FreshRSS_Feed::KIND_XML_XPATH ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_XML_XPATH ? 'selected="selected"' : '' ?> data-show="html_xpath"><?= _t('sub.feed.kind.xml_xpath') ?></option>
+					<option value="<?= FreshRSS_Feed::KIND_JSONFEED ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_JSONFEED ? 'selected="selected"' : '' ?>><?= _t('sub.feed.kind.jsonfeed') ?></option>
+					<option value="<?= FreshRSS_Feed::KIND_JSON_DOTPATH ?>" <?= $this->feed->kind() === FreshRSS_Feed::KIND_JSON_DOTPATH ? 'selected="selected"' : '' ?> data-show="json_dotpath"><?= _t('sub.feed.kind.json_dotpath') ?></option>
 				</select>
 			</div>
 		</div>
@@ -505,6 +507,90 @@
 				</div>
 			</div>
 		</fieldset>
+
+		<fieldset id="json_dotpath">
+			<?php
+				$jsonSettings = Minz_Helper::htmlspecialchars_utf8($this->feed->attributeArray('json_dotpath') ?? []);
+			?>
+			<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.help') ?></p>
+			<div class="form-group">
+				<label class="group-name" for="jsonItem"><small><?= _t('sub.feed.kind.json_dotpath.json') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItem" id="jsonItem" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['item'] ?? '' ?>"><?= $jsonSettings['item'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemTitle"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_title') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemTitle" id="jsonItemTitle" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemTitle'] ?? '' ?>"><?= $jsonSettings['itemTitle'] ?? '' ?></textarea>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemContent"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_content') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemContent" id="jsonItemContent" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemContent'] ?? '' ?>"><?= $jsonSettings['itemContent'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_content.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemUri"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_uri') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemUri" id="jsonItemUri" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemUri'] ?? '' ?>"><?= $jsonSettings['itemUri'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_uri.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemThumbnail"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_thumbnail') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemThumbnail" id="jsonItemThumbnail" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemThumbnail'] ?? '' ?>"><?= $jsonSettings['itemThumbnail'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_thumbnail.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemAuthor"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_author') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemAuthor" id="jsonItemAuthor" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemAuthor'] ?? '' ?>"><?= $jsonSettings['itemAuthor'] ?? '' ?></textarea>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemTimestamp"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_timestamp') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemTimestamp" id="jsonItemTimestamp" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemTimestamp'] ?? '' ?>"><?= $jsonSettings['itemTimestamp'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_timestamp.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemTimeFormat">
+					<?= _t('sub.feed.kind.json_dotpath.item_timeFormat') ?></label>
+				<div class="group-controls">
+					<textarea class="w100" name="jsonItemTimeFormat" id="jsonItemTimeFormat" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemTimeFormat'] ?? '' ?>"><?= $jsonSettings['itemTimeFormat'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.json_dotpath.item_timeFormat.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemCategories"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_categories') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemCategories" id="jsonItemCategories" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemCategories'] ?? '' ?>"><?= $jsonSettings['itemCategories'] ?? '' ?></textarea>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="jsonItemUid"><small><?= _t('sub.feed.kind.json_dotpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.json_dotpath.item_uid') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-json w100" name="jsonItemUid" id="jsonItemUid" rows="2" cols="64" spellcheck="false" data-leave-validation="<?= $jsonSettings['itemUid'] ?? '' ?>"><?= $jsonSettings['itemUid'] ?? '' ?></textarea>
+				</div>
+			</div>
+		</fieldset>
+
 		<div class="form-group form-actions">
 			<div class="group-controls">
 				<button class="btn btn-important"><?= _t('gen.action.submit') ?></button>
@@ -614,6 +700,29 @@
 		</div>
 
 		<div class="form-group">
+			<label class="group-name" for="curl_method"><?= _t('sub.feed.method') ?></label>
+			<div class="group-controls">
+				<select class="number" name="curl_method" id="curl_method"><?php
+				$curl_method = 'GET';
+				if ($this->feed->attributeArray('curl_params') !== null && !empty($this->feed->attributeArray('curl_params')[CURLOPT_POST])) {
+					$curl_method = 'POST';
+				}
+				foreach (['GET' => 'GET', 'POST' => 'POST'] as $k => $v) {
+					echo '<option value="' . $k . ($curl_method === $k ? '" selected="selected' : '') . '">' . $v . '</option>';
+				}
+				?>
+				</select>
+				<div class="stick">
+					<input type="text" name="curl_fields" id="curl_fields" value="<?=
+						$this->feed->attributeArray('curl_params') !== null && !empty($this->feed->attributeArray('curl_params')[CURLOPT_POSTFIELDS]) ?
+							htmlentities($this->feed->attributeArray('curl_params')[CURLOPT_POSTFIELDS], ENT_COMPAT) : ''
+					?>" placeholder="<?= _t('sub.feed.method_postparams') ?>" />
+				</div>
+				<p class="help"><?= _i('help') ?> <?= _t('sub.feed.method_help') ?></p>
+			</div>
+		</div>
+
+		<div class="form-group">
 			<label class="group-name" for="timeout"><?= _t('sub.feed.timeout') ?></label>
 			<div class="group-controls">
 				<input type="number" name="timeout" id="timeout" class="w50" min="3" max="900" value="<?= $this->feed->attributeInt('timeout') ?>" placeholder="<?= _t('gen.short.by_default') ?>" />