JSONFeeds, JSON scraping, and POST requests for feeds (#5662)

* allow POST requests for feeds

* added json dotpath and jsonfeed subscriptions. No translation strings yet

* debug and fix jsonfeed parser

* bugfix params saved when editing feed

* added translations for JSON features

* Update docs for web scraping

* make fix-all
and revert unrelated changes, plus a few manual fixes, but there are still several type errors

* Fix some i18n

* refactor json parsing for both feed types

* cleanup unnecessary comment

* refactored generation of SimplePie for XPath and JSON feeds

* Fix merge error

* Update to newer FreshRSS code

* A bit of refactoring

* doc, whitespace

* JSON Feed is in two words

* Add support for array syntax

* Whitespace

* Add OPML export/import

* Work on i18n

* Accept application/feed+json

* Rework POST

* Fix update

* OPML for cURL options

* Fix types

* Fix Typos

---------

Co-authored-by: Erion Elmasllari <elmasllari@factorsixty.com>
Co-authored-by: Alexandre Alapetite <alexandre@alapetite.fr>

1 files changed, 60 insertions, 0 deletions

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index aa1b30182..018bdb382 100644
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -178,6 +178,9 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController {
 			$useragent = Minz_Request::paramString('curl_params_useragent');
 			$proxy_address = Minz_Request::paramString('curl_params');
 			$proxy_type = Minz_Request::paramString('proxy_type');
+			$request_method = Minz_Request::paramString('curl_method');
+			$request_fields = Minz_Request::paramString('curl_fields', true);
+
 			$opts = [];
 			if ($proxy_type !== '') {
 				$opts[CURLOPT_PROXY] = $proxy_address;
@@ -198,6 +201,15 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController {
 			if ($useragent !== '') {
 				$opts[CURLOPT_USERAGENT] = $useragent;
 			}
+			if ($request_method === 'POST') {
+				$opts[CURLOPT_POST] = true;
+				if ($request_fields !== '') {
+					$opts[CURLOPT_POSTFIELDS] = $request_fields;
+					if (json_decode($request_fields, true) !== null) {
+						$opts[CURLOPT_HTTPHEADER] = ['Content-Type: application/json'];
+					}
+				}
+			}
 
 			$attributes = [
 				'curl_params' => empty($opts) ? null : $opts,
@@ -245,6 +257,44 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController {
 				if (!empty($xPathSettings)) {
 					$attributes['xpath'] = $xPathSettings;
 				}
+			} elseif ($feed_kind === FreshRSS_Feed::KIND_JSON_DOTPATH) {
+				$jsonSettings = [];
+				if (Minz_Request::paramString('jsonFeedTitle') !== '') {
+					$jsonSettings['feedTitle'] = Minz_Request::paramString('jsonFeedTitle', true);
+				}
+				if (Minz_Request::paramString('jsonItem') !== '') {
+					$jsonSettings['item'] = Minz_Request::paramString('jsonItem', true);
+				}
+				if (Minz_Request::paramString('jsonItemTitle') !== '') {
+					$jsonSettings['itemTitle'] = Minz_Request::paramString('jsonItemTitle', true);
+				}
+				if (Minz_Request::paramString('jsonItemContent') !== '') {
+					$jsonSettings['itemContent'] = Minz_Request::paramString('jsonItemContent', true);
+				}
+				if (Minz_Request::paramString('jsonItemUri') !== '') {
+					$jsonSettings['itemUri'] = Minz_Request::paramString('jsonItemUri', true);
+				}
+				if (Minz_Request::paramString('jsonItemAuthor') !== '') {
+					$jsonSettings['itemAuthor'] = Minz_Request::paramString('jsonItemAuthor', true);
+				}
+				if (Minz_Request::paramString('jsonItemTimestamp') !== '') {
+					$jsonSettings['itemTimestamp'] = Minz_Request::paramString('jsonItemTimestamp', true);
+				}
+				if (Minz_Request::paramString('jsonItemTimeFormat') !== '') {
+					$jsonSettings['itemTimeFormat'] = Minz_Request::paramString('jsonItemTimeFormat', true);
+				}
+				if (Minz_Request::paramString('jsonItemThumbnail') !== '') {
+					$jsonSettings['itemThumbnail'] = Minz_Request::paramString('jsonItemThumbnail', true);
+				}
+				if (Minz_Request::paramString('jsonItemCategories') !== '') {
+					$jsonSettings['itemCategories'] = Minz_Request::paramString('jsonItemCategories', true);
+				}
+				if (Minz_Request::paramString('jsonItemUid') !== '') {
+					$jsonSettings['itemUid'] = Minz_Request::paramString('jsonItemUid', true);
+				}
+				if (!empty($jsonSettings)) {
+					$attributes['json_dotpath'] = $jsonSettings;
+				}
 			}
 
 			try {
@@ -445,6 +495,16 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController {
 					if ($simplePie === null) {
 						throw new FreshRSS_Feed_Exception('XML+XPath parsing failed for [' . $feed->url(false) . ']');
 					}
+				} elseif ($feed->kind() === FreshRSS_Feed::KIND_JSON_DOTPATH) {
+					$simplePie = $feed->loadJson();
+					if ($simplePie === null) {
+						throw new FreshRSS_Feed_Exception('JSON dotpath parsing failed for [' . $feed->url(false) . ']');
+					}
+				} elseif ($feed->kind() === FreshRSS_Feed::KIND_JSONFEED) {
+					$simplePie = $feed->loadJson();
+					if ($simplePie === null) {
+						throw new FreshRSS_Feed_Exception('JSON Feed parsing failed for [' . $feed->url(false) . ']');
+					}
 				} else {
 					$simplePie = $feed->load(false, $feedIsNew);
 				}