JSONFeeds, JSON scraping, and POST requests for feeds (#5662)

* allow POST requests for feeds

* added json dotpath and jsonfeed subscriptions. No translation strings yet

* debug and fix jsonfeed parser

* bugfix params saved when editing feed

* added translations for JSON features

* Update docs for web scraping

* make fix-all
and revert unrelated changes, plus a few manual fixes, but there are still several type errors

* Fix some i18n

* refactor json parsing for both feed types

* cleanup unnecessary comment

* refactored generation of SimplePie for XPath and JSON feeds

* Fix merge error

* Update to newer FreshRSS code

* A bit of refactoring

* doc, whitespace

* JSON Feed is in two words

* Add support for array syntax

* Whitespace

* Add OPML export/import

* Work on i18n

* Accept application/feed+json

* Rework POST

* Fix update

* OPML for cURL options

* Fix types

* Fix Typos

---------

Co-authored-by: Erion Elmasllari <elmasllari@factorsixty.com>
Co-authored-by: Alexandre Alapetite <alexandre@alapetite.fr>

1 files changed, 76 insertions, 5 deletions

diff --git a/app/Models/Feed.php b/app/Models/Feed.php
index 024a7841a..40e78bfeb 100644
--- a/app/Models/Feed.php
+++ b/app/Models/Feed.php
@@ -30,6 +30,9 @@ class FreshRSS_Feed extends Minz_Model {
 	 */
 	public const KIND_JSON_XPATH = 20;
 
+	public const KIND_JSONFEED = 25;
+	public const KIND_JSON_DOTPATH = 30;
+
 	public const PRIORITY_IMPORTANT = 20;
 	public const PRIORITY_MAIN_STREAM = 10;
 	public const PRIORITY_CATEGORY = 0;
@@ -580,6 +583,78 @@ class FreshRSS_Feed extends Minz_Model {
 	}
 
 	/**
+	 * Given a feed content generated from a FreshRSS_View
+	 * returns a SimplePie initialized already with that content
+	 * @param string $feedContent the content of the feed, typically generated via FreshRSS_View::renderToString()
+	 */
+	private function simplePieFromContent(string $feedContent): SimplePie {
+		$simplePie = customSimplePie();
+		$simplePie->set_raw_data($feedContent);
+		$simplePie->init();
+		return $simplePie;
+	}
+
+	/** @return array<string,string> */
+	private function dotPathsForStandardJsonFeed(): array {
+		return [
+			'feedTitle' => 'title',
+			'item' => 'items',
+			'itemTitle' => 'title',
+			'itemContent' => 'content_text',
+			'itemContentHTML' => 'content_html',
+			'itemUri' => 'url',
+			'itemTimestamp' => 'date_published',
+			'itemTimeFormat' => DateTimeInterface::RFC3339_EXTENDED,
+			'itemThumbnail' => 'image',
+			'itemCategories' => 'tags',
+			'itemUid' => 'id',
+			'itemAttachment' => 'attachments',
+			'itemAttachmentUrl' => 'url',
+			'itemAttachmentType' => 'mime_type',
+			'itemAttachmentLength' => 'size_in_bytes',
+		];
+	}
+
+	/**
+	 * @throws FreshRSS_Context_Exception
+	 */
+	public function loadJson(): ?SimplePie {
+		if ($this->url == '') {
+			return null;
+		}
+		$feedSourceUrl = htmlspecialchars_decode($this->url, ENT_QUOTES);
+		if ($this->httpAuth != '') {
+			$feedSourceUrl = preg_replace('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $feedSourceUrl);
+		}
+		if ($feedSourceUrl == null) {
+			return null;
+		}
+
+		$cachePath = FreshRSS_Feed::cacheFilename($feedSourceUrl, $this->attributes(), $this->kind());
+		$httpAccept = 'json';
+		$json = httpGet($feedSourceUrl, $cachePath, $httpAccept, $this->attributes());
+		if (strlen($json) <= 0) {
+			return null;
+		}
+
+		//check if the content is actual JSON
+		$jf = json_decode($json, true);
+		if (json_last_error() !== JSON_ERROR_NONE) {
+			return null;
+		}
+
+		/** @var array<string,string> $json_dotpath */
+		$json_dotpath = $this->attributeArray('json_dotpath') ?? [];
+		$dotPaths = $this->kind() === FreshRSS_Feed::KIND_JSONFEED ? $this->dotPathsForStandardJsonFeed() : $json_dotpath;
+
+		$feedContent = FreshRSS_dotpath_Util::convertJsonToRss($jf, $feedSourceUrl, $dotPaths, $this->name());
+		if ($feedContent == null) {
+			return null;
+		}
+		return $this->simplePieFromContent($feedContent);
+	}
+
+	/**
 	 * @throws FreshRSS_Context_Exception
 	 */
 	public function loadHtmlXpath(): ?SimplePie {
@@ -719,11 +794,7 @@ class FreshRSS_Feed extends Minz_Model {
 			Minz_Log::warning($ex->getMessage());
 			return null;
 		}
-
-		$simplePie = customSimplePie();
-		$simplePie->set_raw_data($view->renderToString());
-		$simplePie->init();
-		return $simplePie;
+		return $this->simplePieFromContent($view->renderToString());
 	}
 
 	/**