JSONFeeds, JSON scraping, and POST requests for feeds (#5662)

* allow POST requests for feeds

* added json dotpath and jsonfeed subscriptions. No translation strings yet

* debug and fix jsonfeed parser

* bugfix params saved when editing feed

* added translations for JSON features

* Update docs for web scraping

* make fix-all
and revert unrelated changes, plus a few manual fixes, but there are still several type errors

* Fix some i18n

* refactor json parsing for both feed types

* cleanup unnecessary comment

* refactored generation of SimplePie for XPath and JSON feeds

* Fix merge error

* Update to newer FreshRSS code

* A bit of refactoring

* doc, whitespace

* JSON Feed is in two words

* Add support for array syntax

* Whitespace

* Add OPML export/import

* Work on i18n

* Accept application/feed+json

* Rework POST

* Fix update

* OPML for cURL options

* Fix types

* Fix Typos

---------

Co-authored-by: Erion Elmasllari <elmasllari@factorsixty.com>
Co-authored-by: Alexandre Alapetite <alexandre@alapetite.fr>

1 files changed, 53 insertions, 11 deletions

diff --git a/app/views/helpers/export/opml.phtml b/app/views/helpers/export/opml.phtml
index ce53bfc02..736854f46 100644
--- a/app/views/helpers/export/opml.phtml
+++ b/app/views/helpers/export/opml.phtml
@@ -3,7 +3,7 @@ declare(strict_types=1);
 
 /**
  * @param array<FreshRSS_Feed> $feeds
- * @return array<array<string,string|null>>
+ * @return array<array<string,string|bool|int>>
  */
 function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array {
 	$outlines = [];
@@ -20,15 +20,22 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array {
 			'description' => htmlspecialchars_decode($feed->description(), ENT_QUOTES),
 		];
 
+		switch ($feed->kind()) {
+			case FreshRSS_Feed::KIND_HTML_XPATH:
+				$outline['type'] = FreshRSS_Export_Service::TYPE_HTML_XPATH;
+				break;
+			case FreshRSS_Feed::KIND_XML_XPATH:
+				$outline['type'] = FreshRSS_Export_Service::TYPE_XML_XPATH;
+				break;
+			case FreshRSS_Feed::KIND_JSON_DOTPATH:
+				$outline['type'] = FreshRSS_Export_Service::TYPE_JSON_DOTPATH;
+				break;
+			case FreshRSS_Feed::KIND_JSONFEED:
+				$outline['type'] = FreshRSS_Export_Service::TYPE_JSONFEED;
+				break;
+		}
+
 		if ($feed->kind() === FreshRSS_Feed::KIND_HTML_XPATH || $feed->kind() === FreshRSS_Feed::KIND_XML_XPATH) {
-			switch ($feed->kind()) {
-				case FreshRSS_Feed::KIND_HTML_XPATH:
-					$outline['type'] = FreshRSS_Export_Service::TYPE_HTML_XPATH;
-					break;
-				case FreshRSS_Feed::KIND_XML_XPATH:
-					$outline['type'] = FreshRSS_Export_Service::TYPE_XML_XPATH;
-					break;
-			}
 			/** @var array<string,string> */
 			$xPathSettings = $feed->attributeArray('xpath') ?? [];
 			$outline['frss:xPathItem'] = $xPathSettings['item'] ?? null;
@@ -41,6 +48,19 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array {
 			$outline['frss:xPathItemThumbnail'] = $xPathSettings['itemThumbnail'] ?? null;
 			$outline['frss:xPathItemCategories'] = $xPathSettings['itemCategories'] ?? null;
 			$outline['frss:xPathItemUid'] = $xPathSettings['itemUid'] ?? null;
+		} elseif ($feed->kind() === FreshRSS_Feed::KIND_JSON_DOTPATH) {
+			/** @var array<string,string> */
+			$jsonSettings = $feed->attributeArray('json_dotpath') ?? [];
+			$outline['frss:jsonItem'] = $jsonSettings['item'] ?? null;
+			$outline['frss:jsonItemTitle'] = $jsonSettings['itemTitle'] ?? null;
+			$outline['frss:jsonItemContent'] = $jsonSettings['itemContent'] ?? null;
+			$outline['frss:jsonItemUri'] = $jsonSettings['itemUri'] ?? null;
+			$outline['frss:jsonItemAuthor'] = $jsonSettings['itemAuthor'] ?? null;
+			$outline['frss:jsonItemTimestamp'] = $jsonSettings['itemTimestamp'] ?? null;
+			$outline['frss:jsonItemTimeformat'] = $jsonSettings['itemTimeformat'] ?? null;
+			$outline['frss:jsonItemThumbnail'] = $jsonSettings['itemThumbnail'] ?? null;
+			$outline['frss:jsonItemCategories'] = $jsonSettings['itemCategories'] ?? null;
+			$outline['frss:jsonItemUid'] = $jsonSettings['itemUid'] ?? null;
 		}
 
 		if (!empty($feed->filtersAction('read'))) {
@@ -60,8 +80,30 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array {
 			$outline['frss:cssFullContentFilter'] = $feed->attributeString('path_entries_filter');
 		}
 
-		// Remove null attributes
-		$outline = array_filter($outline, static function (?string $value) { return $value !== null; });
+		$curl_params = $feed->attributeArray('curl_params');
+		if (!empty($curl_params)) {
+			$outline['frss:CURLOPT_COOKIE'] = $curl_params[CURLOPT_COOKIE] ?? null;
+			$outline['frss:CURLOPT_COOKIEFILE'] = $curl_params[CURLOPT_COOKIEFILE] ?? null;
+			$outline['frss:CURLOPT_FOLLOWLOCATION'] = $curl_params[CURLOPT_FOLLOWLOCATION] ?? null;
+			$outline['frss:CURLOPT_MAXREDIRS'] = $curl_params[CURLOPT_MAXREDIRS] ?? null;
+			$outline['frss:CURLOPT_POST'] = $curl_params[CURLOPT_POST] ?? null;
+			$outline['frss:CURLOPT_POSTFIELDS'] = $curl_params[CURLOPT_POSTFIELDS] ?? null;
+			$outline['frss:CURLOPT_PROXY'] = $curl_params[CURLOPT_PROXY] ?? null;
+			$outline['frss:CURLOPT_PROXYTYPE'] = $curl_params[CURLOPT_PROXYTYPE] ?? null;
+			$outline['frss:CURLOPT_USERAGENT'] = $curl_params[CURLOPT_USERAGENT] ?? null;
+
+			if (!empty($curl_params[CURLOPT_HTTPHEADER]) && is_array($curl_params[CURLOPT_HTTPHEADER])) {
+				$headers = '';
+				foreach ($curl_params[CURLOPT_HTTPHEADER] as $header) {
+					$headers .= $header . "\n";
+				}
+				$headers = trim($headers);
+				$outline['frss:CURLOPT_HTTPHEADER'] = $headers;
+			}
+		}
+
+		// Remove null or invalid attributes
+		$outline = array_filter($outline, static function (mixed $value) { return (is_string($value) || is_int($value) || is_bool($value)) && $value !== ''; });
 
 		$outlines[] = $outline;
 	}