From 9c97d8ca729e3cfb067445c0d3c9ad8284132aeb Mon Sep 17 00:00:00 2001 From: eta-orionis <3466670+eta-orionis@users.noreply.github.com> Date: Wed, 10 Jan 2024 08:23:45 +0100 Subject: JSONFeeds, JSON scraping, and POST requests for feeds (#5662) * allow POST requests for feeds * added json dotpath and jsonfeed subscriptions. No translation strings yet * debug and fix jsonfeed parser * bugfix params saved when editing feed * added translations for JSON features * Update docs for web scraping * make fix-all and revert unrelated changes, plus a few manual fixes, but there are still several type errors * Fix some i18n * refactor json parsing for both feed types * cleanup unnecessary comment * refactored generation of SimplePie for XPath and JSON feeds * Fix merge error * Update to newer FreshRSS code * A bit of refactoring * doc, whitespace * JSON Feed is in two words * Add support for array syntax * Whitespace * Add OPML export/import * Work on i18n * Accept application/feed+json * Rework POST * Fix update * OPML for cURL options * Fix types * Fix Typos --------- Co-authored-by: Erion Elmasllari Co-authored-by: Alexandre Alapetite --- app/views/helpers/export/opml.phtml | 64 +++++++++++++++++---- app/views/helpers/feed/update.phtml | 109 ++++++++++++++++++++++++++++++++++++ app/views/subscription/add.phtml | 105 ++++++++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 11 deletions(-) (limited to 'app/views') diff --git a/app/views/helpers/export/opml.phtml b/app/views/helpers/export/opml.phtml index ce53bfc02..736854f46 100644 --- a/app/views/helpers/export/opml.phtml +++ b/app/views/helpers/export/opml.phtml @@ -3,7 +3,7 @@ declare(strict_types=1); /** * @param array $feeds - * @return array> + * @return array> */ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array { $outlines = []; @@ -20,15 +20,22 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array { 'description' => htmlspecialchars_decode($feed->description(), ENT_QUOTES), ]; + switch ($feed->kind()) { + case FreshRSS_Feed::KIND_HTML_XPATH: + $outline['type'] = FreshRSS_Export_Service::TYPE_HTML_XPATH; + break; + case FreshRSS_Feed::KIND_XML_XPATH: + $outline['type'] = FreshRSS_Export_Service::TYPE_XML_XPATH; + break; + case FreshRSS_Feed::KIND_JSON_DOTPATH: + $outline['type'] = FreshRSS_Export_Service::TYPE_JSON_DOTPATH; + break; + case FreshRSS_Feed::KIND_JSONFEED: + $outline['type'] = FreshRSS_Export_Service::TYPE_JSONFEED; + break; + } + if ($feed->kind() === FreshRSS_Feed::KIND_HTML_XPATH || $feed->kind() === FreshRSS_Feed::KIND_XML_XPATH) { - switch ($feed->kind()) { - case FreshRSS_Feed::KIND_HTML_XPATH: - $outline['type'] = FreshRSS_Export_Service::TYPE_HTML_XPATH; - break; - case FreshRSS_Feed::KIND_XML_XPATH: - $outline['type'] = FreshRSS_Export_Service::TYPE_XML_XPATH; - break; - } /** @var array */ $xPathSettings = $feed->attributeArray('xpath') ?? []; $outline['frss:xPathItem'] = $xPathSettings['item'] ?? null; @@ -41,6 +48,19 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array { $outline['frss:xPathItemThumbnail'] = $xPathSettings['itemThumbnail'] ?? null; $outline['frss:xPathItemCategories'] = $xPathSettings['itemCategories'] ?? null; $outline['frss:xPathItemUid'] = $xPathSettings['itemUid'] ?? null; + } elseif ($feed->kind() === FreshRSS_Feed::KIND_JSON_DOTPATH) { + /** @var array */ + $jsonSettings = $feed->attributeArray('json_dotpath') ?? []; + $outline['frss:jsonItem'] = $jsonSettings['item'] ?? null; + $outline['frss:jsonItemTitle'] = $jsonSettings['itemTitle'] ?? null; + $outline['frss:jsonItemContent'] = $jsonSettings['itemContent'] ?? null; + $outline['frss:jsonItemUri'] = $jsonSettings['itemUri'] ?? null; + $outline['frss:jsonItemAuthor'] = $jsonSettings['itemAuthor'] ?? null; + $outline['frss:jsonItemTimestamp'] = $jsonSettings['itemTimestamp'] ?? null; + $outline['frss:jsonItemTimeformat'] = $jsonSettings['itemTimeformat'] ?? null; + $outline['frss:jsonItemThumbnail'] = $jsonSettings['itemThumbnail'] ?? null; + $outline['frss:jsonItemCategories'] = $jsonSettings['itemCategories'] ?? null; + $outline['frss:jsonItemUid'] = $jsonSettings['itemUid'] ?? null; } if (!empty($feed->filtersAction('read'))) { @@ -60,8 +80,30 @@ function feedsToOutlines(array $feeds, bool $excludeMutedFeeds = false): array { $outline['frss:cssFullContentFilter'] = $feed->attributeString('path_entries_filter'); } - // Remove null attributes - $outline = array_filter($outline, static function (?string $value) { return $value !== null; }); + $curl_params = $feed->attributeArray('curl_params'); + if (!empty($curl_params)) { + $outline['frss:CURLOPT_COOKIE'] = $curl_params[CURLOPT_COOKIE] ?? null; + $outline['frss:CURLOPT_COOKIEFILE'] = $curl_params[CURLOPT_COOKIEFILE] ?? null; + $outline['frss:CURLOPT_FOLLOWLOCATION'] = $curl_params[CURLOPT_FOLLOWLOCATION] ?? null; + $outline['frss:CURLOPT_MAXREDIRS'] = $curl_params[CURLOPT_MAXREDIRS] ?? null; + $outline['frss:CURLOPT_POST'] = $curl_params[CURLOPT_POST] ?? null; + $outline['frss:CURLOPT_POSTFIELDS'] = $curl_params[CURLOPT_POSTFIELDS] ?? null; + $outline['frss:CURLOPT_PROXY'] = $curl_params[CURLOPT_PROXY] ?? null; + $outline['frss:CURLOPT_PROXYTYPE'] = $curl_params[CURLOPT_PROXYTYPE] ?? null; + $outline['frss:CURLOPT_USERAGENT'] = $curl_params[CURLOPT_USERAGENT] ?? null; + + if (!empty($curl_params[CURLOPT_HTTPHEADER]) && is_array($curl_params[CURLOPT_HTTPHEADER])) { + $headers = ''; + foreach ($curl_params[CURLOPT_HTTPHEADER] as $header) { + $headers .= $header . "\n"; + } + $headers = trim($headers); + $outline['frss:CURLOPT_HTTPHEADER'] = $headers; + } + } + + // Remove null or invalid attributes + $outline = array_filter($outline, static function (mixed $value) { return (is_string($value) || is_int($value) || is_bool($value)) && $value !== ''; }); $outlines[] = $outline; } diff --git a/app/views/helpers/feed/update.phtml b/app/views/helpers/feed/update.phtml index 5d4f1cc4b..13a751c09 100644 --- a/app/views/helpers/feed/update.phtml +++ b/app/views/helpers/feed/update.phtml @@ -407,6 +407,8 @@ + + @@ -505,6 +507,90 @@ + +
+ feed->attributeArray('json_dotpath') ?? []); + ?> +

+
+ +
+ +

+
+
+
+ +
+ +
+
+
+ +
+ +

+
+
+
+ +
+ +

+
+
+
+ +
+ +

+
+
+
+ +
+ +
+
+
+ +
+ +

+
+
+
+ +
+ +

+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
@@ -613,6 +699,29 @@
+
+ +
+ +
+ +
+

+
+
+
diff --git a/app/views/subscription/add.phtml b/app/views/subscription/add.phtml index 2dfc8c705..fad1ee0ff 100644 --- a/app/views/subscription/add.phtml +++ b/app/views/subscription/add.phtml @@ -71,6 +71,8 @@ + +
@@ -164,6 +166,93 @@ +
+

+
+ +
+ +

+
+
+
+ +
+ +

+
+
+
+ +
+ +
+
+
+ +
+ +

+
+
+
+ +
+ +

+
+
+
+ +
+ +

+
+
+
+ +
+ +
+
+
+ +
+ +

+
+
+
+ +
+ +

+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
@@ -231,6 +320,22 @@ +
+ +
+ +
+ +
+

+
+
+
-- cgit v1.2.3