From 1fe66ad020ca8f0560bb9c6e311852ed77228f78 Mon Sep 17 00:00:00 2001
From: Alexandre Alapetite <alexandre@alapetite.fr>
Date: Mon, 28 Feb 2022 20:22:43 +0100
Subject: Implement Web scraping "HTML + XPath" (#4220)

* More PHP type hints for Fever
Follow-up of https://github.com/FreshRSS/FreshRSS/pull/4201
Related to https://github.com/FreshRSS/FreshRSS/issues/4200

* Detail

* Draft

* Progress

* More draft

* Fix thumbnail PHP type hint
https://github.com/FreshRSS/FreshRSS/issues/4215

* More types

* A bit more

* Refactor FreshRSS_Entry::fromArray

* Progress

* Starts to work

* Categories

* Fonctional

* Layout update

* Fix relative URLs

* Cache system

* Forgotten files

* Remove a debug line

* Automatic form validation of XPath expressions

* data-leave-validation

* Fix reload action

* Simpler examples

* Fix column type for PostgreSQL

* Enforce HTTP encoding

* Readme

* Fix get full content

* target="_blank"

* gitignore

* htmlspecialchars_utf8

* Implement HTML <base>
And fix/revert `xml:base` support in SimplePie https://github.com/simplepie/simplepie/commit/e49c578817aa504d8d05cd7f33857aeda9d41908

* SimplePie upstream PR merged
https://github.com/simplepie/simplepie/pull/723
---
 app/views/helpers/export/articles.phtml |   2 +-
 app/views/helpers/feed/update.phtml     | 104 ++++++++++++++++++++++++++++++++
 2 files changed, 105 insertions(+), 1 deletion(-)

(limited to 'app/views/helpers')

diff --git a/app/views/helpers/export/articles.phtml b/app/views/helpers/export/articles.phtml
index c131b8474..ad5210968 100644
--- a/app/views/helpers/export/articles.phtml
+++ b/app/views/helpers/export/articles.phtml
@@ -22,7 +22,7 @@ foreach ($this->entriesRaw as $entryRaw) {
 	if ($entryRaw == null) {
 		continue;
 	}
-	$entry = FreshRSS_EntryDAO::daoToEntry($entryRaw);
+	$entry = FreshRSS_Entry::fromArray($entryRaw);
 	if (!isset($this->feed)) {
 		$feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed());
 		if ($feed === null) {
diff --git a/app/views/helpers/feed/update.phtml b/app/views/helpers/feed/update.phtml
index 264881f77..f71be5135 100644
--- a/app/views/helpers/feed/update.phtml
+++ b/app/views/helpers/feed/update.phtml
@@ -373,6 +373,110 @@
 			</div>
 		</div>
 
+		<legend><?= _t('sub.feed.kind') ?></legend>
+		<div class="form-group">
+			<label class="group-name" for="feed_kind"><?= _t('sub.feed.kind') ?></label>
+			<div class="group-controls">
+				<select name="feed_kind" id="feed_kind" class="select-show">
+					<option value="<?= FreshRSS_Feed::KIND_RSS ?>" <?= $this->feed->kind() == FreshRSS_Feed::KIND_RSS ? 'selected="selected"' : '' ?>><?= _t('sub.feed.kind.rss') ?></option>
+					<option value="<?= FreshRSS_Feed::KIND_HTML_XPATH ?>" <?= $this->feed->kind() == FreshRSS_Feed::KIND_HTML_XPATH ? 'selected="selected"' : '' ?> data-show="html_xpath"><?= _t('sub.feed.kind.html_xpath') ?></option>
+				</select>
+			</div>
+		</div>
+
+		<fieldset id="html_xpath">
+			<?php
+				$xpath = Minz_Helper::htmlspecialchars_utf8($this->feed->attributes('xpath'));
+			?>
+			<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.help') ?></p>
+			<div class="form-group">
+				<label class="group-name" for="xPathFeedTitle"><small><?= _t('sub.feed.kind.html_xpath.xpath') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.feed_title') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathFeedTitle" id="xPathFeedTitle" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['feedTitle'] ?? '' ?>"><?= $xpath['feedTitle'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.feed_title.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="xPathItem"><small><?= _t('sub.feed.kind.html_xpath.xpath') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.item') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathItem" id="xPathItem" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['item'] ?? '' ?>"><?= $xpath['item'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.item.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="xPathItemTitle"><small><?= _t('sub.feed.kind.html_xpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.item_title') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathItemTitle" id="xPathItemTitle" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['itemTitle'] ?? '' ?>"><?= $xpath['itemTitle'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.item_title.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="xPathItemContent"><small><?= _t('sub.feed.kind.html_xpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.item_content') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathItemContent" id="xPathItemContent" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['itemContent'] ?? '' ?>"><?= $xpath['itemContent'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.item_content.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="xPathItemUri"><small><?= _t('sub.feed.kind.html_xpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.item_uri') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathItemUri" id="xPathItemUri" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['itemUri'] ?? '' ?>"><?= $xpath['itemUri'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.item_uri.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="xPathItemThumbnail"><small><?= _t('sub.feed.kind.html_xpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.item_thumbnail') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathItemThumbnail" id="xPathItemThumbnail" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['itemThumbnail'] ?? '' ?>"><?= $xpath['itemThumbnail'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.item_thumbnail.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="xPathItemAuthor"><small><?= _t('sub.feed.kind.html_xpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.item_author') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathItemAuthor" id="xPathItemAuthor" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['itemAuthor'] ?? '' ?>"><?= $xpath['itemAuthor'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.item_author.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="xPathItemTimestamp"><small><?= _t('sub.feed.kind.html_xpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.item_timestamp') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathItemTimestamp" id="xPathItemTimestamp" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['itemTimestamp'] ?? '' ?>"><?= $xpath['itemTimestamp'] ?? '' ?></textarea>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.kind.html_xpath.item_timestamp.help') ?></p>
+				</div>
+			</div>
+			<div class="form-group">
+				<label class="group-name" for="xPathItemCategories"><small><?= _t('sub.feed.kind.html_xpath.relative') ?></small><br />
+					<?= _t('sub.feed.kind.html_xpath.item_categories') ?></label>
+				<div class="group-controls">
+					<textarea class="valid-xpath" name="xPathItemCategories" id="xPathItemCategories" rows="2" cols="64" spellcheck="false"
+						data-leave-validation="<?= $xpath['itemCategories'] ?? '' ?>"><?= $xpath['itemCategories'] ?? '' ?></textarea>
+				</div>
+			</div>
+		</fieldset>
+		<div class="form-group form-actions">
+			<div class="group-controls">
+				<button class="btn btn-important"><?= _t('gen.action.submit') ?></button>
+				<button type="reset" class="btn"><?= _t('gen.action.cancel') ?></button>
+			</div>
+		</div>
+
 		<legend><?= _t('sub.feed.advanced') ?></legend>
 		<div class="form-group">
 			<label class="group-name" for="path_entries"><?= _t('sub.feed.css_path') ?></label>
-- 
cgit v1.2.3