New unicity policies for feeds with bad GUIDs (#4487)

New set of unicity criteria options.

New tolerance heuristic:

> `$invalidGuidsTolerance` (default 0.05) The maximum ratio (rounded) of invalid GUIDs to tolerate before degrading the unicity criteria.
> Example for 0.05 (5% rounded): tolerate 0 invalid GUIDs for up to 9 articles, 1 for 10, 2 for 30, 3 for 50, 4 for 70, 5 for 90, 6 for 110, etc.
> The default value of 5% rounded was chosen to allow 1 invalid GUID for feeds of 10 articles, which is a frequently observed amount of articles.

29 files changed, 365 insertions, 33 deletions

diff --git a/app/Controllers/feedController.php b/app/Controllers/feedController.php
index 2b757177c..26e3caa3c 100755
--- a/app/Controllers/feedController.php
+++ b/app/Controllers/feedController.php
@@ -716,7 +716,8 @@ class FreshRSS_feed_Controller extends FreshRSS_ActionController {
 					}
 				}
 			}
-			if (!empty($feedProperties)) {
+			if (!empty($feedProperties) || $feedIsNew) {
+				$feedProperties['attributes'] = $feed->attributes();
 				$ok = $feedDAO->updateFeed($feed->id(), $feedProperties);
 				if (!$ok && $feedIsNew) {
 					//Cancel adding new feed in case of database error at first actualize
diff --git a/app/Controllers/subscriptionController.php b/app/Controllers/subscriptionController.php
index 683f403bc..43f6aad11 100644
--- a/app/Controllers/subscriptionController.php
+++ b/app/Controllers/subscriptionController.php
@@ -108,6 +108,18 @@ class FreshRSS_subscription_Controller extends FreshRSS_ActionController {
 		FreshRSS_View::prependTitle($feed->name() . ' · ' . _t('sub.title.feed_management') . ' · ');
 
 		if (Minz_Request::isPost()) {
+			$unicityCriteria = Minz_Request::paramString('unicityCriteria');
+			if (in_array($unicityCriteria, ['id', '', null], strict: true)) {
+				$unicityCriteria = null;
+			}
+			if ($unicityCriteria === null && $feed->attributeBoolean('hasBadGuids')) {	// Legacy
+				$unicityCriteria = 'link';
+			}
+			$feed->_attribute('hasBadGuids', null);	// Remove legacy
+			$feed->_attribute('unicityCriteria', $unicityCriteria);
+
+			$feed->_attribute('unicityCriteriaForced', Minz_Request::paramBoolean('unicityCriteriaForced') ? true : null);
+
 			$user = Minz_Request::paramString('http_user_feed' . $id);
 			$pass = Minz_Request::paramString('http_pass_feed' . $id);
 
diff --git a/app/Models/Entry.php b/app/Models/Entry.php
index fe6702bcd..6e87fe5cd 100644
--- a/app/Models/Entry.php
+++ b/app/Models/Entry.php
@@ -456,7 +456,7 @@ HTML;
 	}
 
 	public function hash(): string {
-		if ($this->hash == '') {
+		if ($this->hash === '') {
 			//Do not include $this->date because it may be automatically generated when lacking
 			$this->hash = md5($this->link . $this->title . $this->authors(true) . $this->originalContent() . $this->tags(true));
 		}
@@ -481,16 +481,11 @@ HTML;
 			$this->date_added = $value;
 		}
 	}
+
 	public function _guid(string $value): void {
-		$value = trim($value);
-		if (empty($value)) {
-			$value = $this->link;
-			if (empty($value)) {
-				$value = $this->hash();
-			}
-		}
-		$this->guid = $value;
+		$this->guid = trim($value);
 	}
+
 	public function _title(string $value): void {
 		$this->hash = '';
 		$this->title = trim($value);
diff --git a/app/Models/Feed.php b/app/Models/Feed.php
index ad84c35a1..b5b599d5f 100644
--- a/app/Models/Feed.php
+++ b/app/Models/Feed.php
@@ -429,14 +429,61 @@ class FreshRSS_Feed extends Minz_Model {
 	}
 
 	/**
+	 * Decide the GUID of an entry based on the feed’s policy.
+	 * @param \SimplePie\Item $item The item to decide the GUID for.
+	 * @param bool $fallback Whether to automatically switch to the next policy in case of blank GUID.
+	 * @return string The decided GUID for the entry.
+	 */
+	protected function decideEntryGuid(\SimplePie\Item $item, bool $fallback = false): string {
+		$unicityCriteria = $this->attributeString('unicityCriteria');
+		if ($this->attributeBoolean('hasBadGuids')) {	// Legacy
+			$unicityCriteria = 'link';
+		}
+
+		$entryId = safe_ascii($item->get_id(false, false));
+
+		$guid = match ($unicityCriteria) {
+			null => $entryId,
+			'link' => $item->get_permalink() ?? '',
+			'sha1:link_published'               => sha1($item->get_permalink() . $item->get_date('U')),
+			'sha1:link_published_title'         => sha1($item->get_permalink() . $item->get_date('U') . $item->get_title()),
+			'sha1:link_published_title_content' => sha1($item->get_permalink() . $item->get_date('U') . $item->get_title() . $item->get_content()),
+			default => $entryId,
+		};
+
+		$blankHash = 'da39a3ee5e6b4b0d3255bfef95601890afd80709';	// sha1('')
+		if ($guid === $blankHash) {
+			$guid = '';
+		}
+
+		if ($fallback && $guid === '') {
+			if ($entryId !== '') {
+				$guid = $entryId;
+			} elseif (($item->get_permalink() ?? '') !== '') {
+				$guid = sha1($item->get_permalink() . $item->get_date('U'));
+			} elseif (($item->get_title() ?? '') !== '') {
+				$guid = sha1($item->get_permalink() . $item->get_date('U') . $item->get_title());
+			} else {
+				$guid = sha1($item->get_permalink() . $item->get_date('U') . $item->get_title() . $item->get_content());
+			}
+			if ($guid === $blankHash) {
+				$guid = '';
+			}
+		}
+
+		return $guid;
+	}
+
+	/**
+	 * @param float $invalidGuidsTolerance (default 0.05) The maximum ratio (rounded) of invalid GUIDs to tolerate before degrading the unicity criteria.
+	 * Example for 0.05 (5% rounded): tolerate 0 invalid GUIDs for up to 9 articles, 1 for 10, 2 for 30, 3 for 50, 4 for 70, 5 for 90, 6 for 110, etc.
+	 * The default value of 5% rounded was chosen to allow 1 invalid GUID for feeds of 10 articles, which is a frequently observed amount of articles.
 	 * @return array<string>
 	 */
-	public function loadGuids(\SimplePie\SimplePie $simplePie): array {
-		$hasUniqueGuids = true;
+	public function loadGuids(\SimplePie\SimplePie $simplePie, float $invalidGuidsTolerance = 0.05): array {
+		$invalidGuids = 0;
 		$testGuids = [];
 		$guids = [];
-		$links = [];
-		$hadBadGuids = $this->attributeBoolean('hasBadGuids');
 
 		$items = $simplePie->get_items();
 		if (empty($items)) {
@@ -447,33 +494,46 @@ class FreshRSS_Feed extends Minz_Model {
 			if ($item == null) {
 				continue;
 			}
-			$guid = safe_ascii($item->get_id(false, false));
-			$hasUniqueGuids &= empty($testGuids['_' . $guid]);
+			$guid = $this->decideEntryGuid($item, fallback: true);
+			if ($guid === '' || !empty($testGuids['_' . $guid])) {
+				$invalidGuids++;
+				Minz_Log::debug('Invalid GUID [' . $guid . '] for feed ' . $this->url);
+			}
 			$testGuids['_' . $guid] = true;
 			$guids[] = $guid;
-			$permalink = $item->get_permalink();
-			if ($permalink != null) {
-				$links[] = $permalink;
-			}
 		}
 
-		if ($hadBadGuids != !$hasUniqueGuids) {
-			if ($hadBadGuids) {
-				Minz_Log::warning('Feed has invalid GUIDs: ' . $this->url);
-			} else {
-				Minz_Log::warning('Feed has valid GUIDs again: ' . $this->url);
+		if ($invalidGuids > 0) {
+			Minz_Log::warning("Feed has {$invalidGuids} invalid GUIDs: " . $this->url);
+			if (!$this->attributeBoolean('unicityCriteriaForced') && $invalidGuids > round($invalidGuidsTolerance * count($items))) {
+				$unicityCriteria = $this->attributeString('unicityCriteria');
+				if ($this->attributeBoolean('hasBadGuids')) {	// Legacy
+					$unicityCriteria = 'link';
+				}
+
+				// Automatic fallback to next (degraded) unicity criteria
+				$newUnicityCriteria = match ($unicityCriteria) {
+					null => 'sha1:link_published',
+					'link' => 'sha1:link_published',
+					'sha1:link_published' => 'sha1:link_published_title',
+					default => $unicityCriteria,
+				};
+
+				if ($newUnicityCriteria !== $unicityCriteria) {
+					$this->_attribute('hasBadGuids', null);	// Remove legacy
+					$this->_attribute('unicityCriteria', $newUnicityCriteria);
+					Minz_Log::warning('Feed unicity policy degraded (' . ($unicityCriteria ?: 'id') . ' → ' . $newUnicityCriteria . '): ' . $this->url);
+					return $this->loadGuids($simplePie, $invalidGuidsTolerance);
+				}
 			}
-			$feedDAO = FreshRSS_Factory::createFeedDao();
-			$feedDAO->updateFeedAttribute($this, 'hasBadGuids', !$hasUniqueGuids);
+			$this->_error(true);
 		}
 
-		return $hasUniqueGuids ? $guids : $links;
+		return $guids;
 	}
 
 	/** @return Traversable<FreshRSS_Entry> */
 	public function loadEntries(\SimplePie\SimplePie $simplePie): Traversable {
-		$hasBadGuids = $this->attributeBoolean('hasBadGuids');
-
 		$items = $simplePie->get_items();
 		if (empty($items)) {
 			return;
@@ -487,7 +547,7 @@ class FreshRSS_Feed extends Minz_Model {
 			$title = html_only_entity_decode(strip_tags($item->get_title() ?? ''));
 			$authors = $item->get_authors();
 			$link = $item->get_permalink();
-			$date = @strtotime((string)($item->get_date() ?? '')) ?: 0;
+			$date = $item->get_date('U');
 
 			//Tag processing (tag == category)
 			$categories = $item->get_categories();
@@ -571,7 +631,7 @@ class FreshRSS_Feed extends Minz_Model {
 				}
 			}
 
-			$guid = safe_ascii($item->get_id(false, false));
+			$guid = $this->decideEntryGuid($item, fallback: true);
 			unset($item);
 
 			$authorNames = '';
@@ -587,7 +647,7 @@ class FreshRSS_Feed extends Minz_Model {
 
 			$entry = new FreshRSS_Entry(
 				$this->id(),
-				$hasBadGuids ? '' : $guid,
+				$guid,
 				$title == '' ? '' : $title,
 				$authorNames,
 				$content == '' ? '' : $content,
diff --git a/app/i18n/cs/sub.php b/app/i18n/cs/sub.php
index c60f47f95..6546a38af 100644
--- a/app/i18n/cs/sub.php
+++ b/app/i18n/cs/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Název',
 		'title_add' => 'Přidat kanál RSS',
 		'ttl' => 'Neobnovovat automaticky častěji než',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Adresa URL kanálu',
 		'useragent' => 'Nastavte uživatelský agent pro načítání tohoto kanálu',
 		'useragent_help' => 'Příklad: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/de/sub.php b/app/i18n/de/sub.php
index b8fba0a7e..30f77b398 100644
--- a/app/i18n/de/sub.php
+++ b/app/i18n/de/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Titel',
 		'title_add' => 'Einen RSS-Feed hinzufügen',
 		'ttl' => 'Aktualisiere automatisch nicht öfter als',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Feed-URL',
 		'useragent' => 'Browser User Agent für den Abruf des Feeds verwenden',
 		'useragent_help' => 'Beispiel: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/el/sub.php b/app/i18n/el/sub.php
index 47e442b13..02821ea6d 100644
--- a/app/i18n/el/sub.php
+++ b/app/i18n/el/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Title',	// TODO
 		'title_add' => 'Add an RSS feed',	// TODO
 		'ttl' => 'Do not automatically refresh more often than',	// TODO
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Feed URL',	// TODO
 		'useragent' => 'Set the user agent for fetching this feed',	// TODO
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',	// TODO
diff --git a/app/i18n/en-us/sub.php b/app/i18n/en-us/sub.php
index 93e7bf21b..ef7931681 100644
--- a/app/i18n/en-us/sub.php
+++ b/app/i18n/en-us/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Title',	// IGNORE
 		'title_add' => 'Add an RSS feed',	// IGNORE
 		'ttl' => 'Do not automatically refresh more often than',	// IGNORE
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// IGNORE
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// IGNORE
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// IGNORE
+			'id' => 'Standard ID (default)',	// IGNORE
+			'link' => 'Link',	// IGNORE
+			'sha1:link_published' => 'Link + Date',	// IGNORE
+			'sha1:link_published_title' => 'Link + Date + Title',	// IGNORE
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// IGNORE
+		),
 		'url' => 'Feed URL',	// IGNORE
 		'useragent' => 'Set the user agent for fetching this feed',	// IGNORE
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',	// IGNORE
diff --git a/app/i18n/en/sub.php b/app/i18n/en/sub.php
index ca4da3409..0a6385428 100644
--- a/app/i18n/en/sub.php
+++ b/app/i18n/en/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Title',
 		'title_add' => 'Add an RSS feed',
 		'ttl' => 'Do not automatically refresh more often than',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',
+			'link' => 'Link',
+			'sha1:link_published' => 'Link + Date',
+			'sha1:link_published_title' => 'Link + Date + Title',
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',
+		),
 		'url' => 'Feed URL',
 		'useragent' => 'Set the user agent for fetching this feed',
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/es/sub.php b/app/i18n/es/sub.php
index 5615d5ac9..f5c557db1 100644
--- a/app/i18n/es/sub.php
+++ b/app/i18n/es/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Título',
 		'title_add' => 'Añadir fuente RSS',
 		'ttl' => 'No actualizar de forma automática con una frecuencia mayor a',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'URL de la fuente',
 		'useragent' => 'Selecciona el agente de usuario por recuperar la fuente',
 		'useragent_help' => 'Ejemplo: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/fa/sub.php b/app/i18n/fa/sub.php
index 8ad8f3090..93ff0f205 100644
--- a/app/i18n/fa/sub.php
+++ b/app/i18n/fa/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => ' عنوان',
 		'title_add' => ' یک فید RSS اضافه کنید',
 		'ttl' => ' به‌طور خودکار بیشتر از آن رفرش نکنید',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => ' URL فید',
 		'useragent' => ' عامل کاربر را برای واکشی این فید تنظیم کنید',
 		'useragent_help' => ' مثال: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/fr/sub.php b/app/i18n/fr/sub.php
index 2c6ec36cb..24678f377 100644
--- a/app/i18n/fr/sub.php
+++ b/app/i18n/fr/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Titre',
 		'title_add' => 'Ajouter un flux RSS',
 		'ttl' => 'Ne pas automatiquement rafraîchir plus souvent que',
+		'unicityCriteria' => array(
+			'_' => 'Critère d’unicité des articles',
+			'forced' => '<span title="Bloque le critère d’unicité même en cas de doublons">forcé</span>',
+			'help' => 'Utile pour les flux invalides.<br />⚠️ Changer le critère peut créer des doublons.',
+			'id' => 'ID standard (défaut)',
+			'link' => 'Lien',
+			'sha1:link_published' => 'Lien + Date',
+			'sha1:link_published_title' => 'Lien + Date + Titre',
+			'sha1:link_published_title_content' => 'Lien + Date + Titre + Contenu',
+		),
 		'url' => 'URL du flux',
 		'useragent' => 'Sélectionner l’agent utilisateur pour télécharger ce flux',
 		'useragent_help' => 'Exemple : <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/he/sub.php b/app/i18n/he/sub.php
index 4accafe33..eb4df468c 100644
--- a/app/i18n/he/sub.php
+++ b/app/i18n/he/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'כותרת',
 		'title_add' => 'הוספת הזנה',
 		'ttl' => 'אין לרענן אוטומטית יותר מ',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'הזנה URL',
 		'useragent' => 'Set the user agent for fetching this feed',	// TODO
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',	// TODO
diff --git a/app/i18n/hu/sub.php b/app/i18n/hu/sub.php
index 7b9a33d98..397907f0f 100644
--- a/app/i18n/hu/sub.php
+++ b/app/i18n/hu/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Cím',
 		'title_add' => 'RSS hírforrás hozzáadása',
 		'ttl' => 'Ne frissítsd automatikusan többször mint',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Hírforrás URL',
 		'useragent' => 'Állíts be egy user agent-et ehhez a hírforráshoz',
 		'useragent_help' => 'Példa: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/id/sub.php b/app/i18n/id/sub.php
index d8fde0251..f192dacec 100644
--- a/app/i18n/id/sub.php
+++ b/app/i18n/id/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Title',	// TODO
 		'title_add' => 'Add an RSS feed',	// TODO
 		'ttl' => 'Do not automatically refresh more often than',	// TODO
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Feed URL',	// TODO
 		'useragent' => 'Set the user agent for fetching this feed',	// TODO
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',	// TODO
diff --git a/app/i18n/it/sub.php b/app/i18n/it/sub.php
index e36a144ec..35b8fab60 100644
--- a/app/i18n/it/sub.php
+++ b/app/i18n/it/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Titolo',
 		'title_add' => 'Aggiungi RSS feed',
 		'ttl' => 'Non aggiornare automaticamente piu di',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'URL del feed',
 		'useragent' => 'Imposta lo user agent per recuperare questo feed',
 		'useragent_help' => 'Esempio: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/ja/sub.php b/app/i18n/ja/sub.php
index 00234c0af..a1a02d2a9 100644
--- a/app/i18n/ja/sub.php
+++ b/app/i18n/ja/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'タイトル',
 		'title_add' => 'RSS フィードを追加する',
 		'ttl' => '自動更新の頻度',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'フィードのURL',
 		'useragent' => 'フィードを読み込む際のユーザーエージェントを設定してください',
 		'useragent_help' => '例: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/ko/sub.php b/app/i18n/ko/sub.php
index e28f22f74..917d71b68 100644
--- a/app/i18n/ko/sub.php
+++ b/app/i18n/ko/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => '제목',
 		'title_add' => 'RSS 피드 추가',
 		'ttl' => '다음 시간이 지나기 전에 새로고침 금지',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => '피드 URL',
 		'useragent' => '이 피드를 가져올 때 사용할 유저 에이전트 설정',
 		'useragent_help' => '예시: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/lv/sub.php b/app/i18n/lv/sub.php
index d81da3718..ea67d3005 100644
--- a/app/i18n/lv/sub.php
+++ b/app/i18n/lv/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Tituls',
 		'title_add' => 'Pievienot RSS barotni',
 		'ttl' => 'Automātiski neatjaunināt biežāk par',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Barotnes URL',
 		'useragent' => 'Lietotāja aģenta iestatīšana šīs barotnes iegūšanai',
 		'useragent_help' => 'Piemērs: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/nl/sub.php b/app/i18n/nl/sub.php
index 817fa8b10..ae0b47391 100644
--- a/app/i18n/nl/sub.php
+++ b/app/i18n/nl/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Titel',
 		'title_add' => 'Voeg een RSS-feed toe',
 		'ttl' => 'Vernieuw automatisch niet vaker dan',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Feed-url',
 		'useragent' => 'Stelt de useragent in om deze feed op te halen',
 		'useragent_help' => 'Voorbeeld: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/oc/sub.php b/app/i18n/oc/sub.php
index 6b8ea93bb..c0f0ae245 100644
--- a/app/i18n/oc/sub.php
+++ b/app/i18n/oc/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Títol',
 		'title_add' => 'Ajustar un flux RSS',
 		'ttl' => 'Actualizar pas automaticament mai sovent que',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Flux URL',
 		'useragent' => 'Definir un user agent per recuperar aqueste flux',
 		'useragent_help' => 'Exemple : <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/pl/sub.php b/app/i18n/pl/sub.php
index 4f99c2866..30cd09a50 100644
--- a/app/i18n/pl/sub.php
+++ b/app/i18n/pl/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Tytuł',
 		'title_add' => 'Dodaj kanał',
 		'ttl' => 'Nie odświeżaj automatycznie częściej niż',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Adres kanału',
 		'useragent' => 'Ciąg user agent używany podczas pobierania kanału',
 		'useragent_help' => 'Przykład: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/pt-br/sub.php b/app/i18n/pt-br/sub.php
index d1c2f08e5..4d6f074e7 100644
--- a/app/i18n/pt-br/sub.php
+++ b/app/i18n/pt-br/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Título',
 		'title_add' => 'Adicionar o RSS feed',
 		'ttl' => 'Não atualize automaticamente mais que',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'URL do Feed',
 		'useragent' => 'Defina um usuário para buscar este feed',
 		'useragent_help' => 'Exemplo: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/ru/sub.php b/app/i18n/ru/sub.php
index 946f53177..569e9202d 100644
--- a/app/i18n/ru/sub.php
+++ b/app/i18n/ru/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Заголовок',
 		'title_add' => 'Добавить RSS-ленту',
 		'ttl' => 'Не обновлять автоматически чаще, чем каждые',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'URL ленты',
 		'useragent' => 'Указать юзерагент для извлечения лент',
 		'useragent_help' => 'Пример: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/sk/sub.php b/app/i18n/sk/sub.php
index a019db2a9..ef4060352 100644
--- a/app/i18n/sk/sub.php
+++ b/app/i18n/sk/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Nadpis',
 		'title_add' => 'Pridať kanál RSS',
 		'ttl' => 'Automaticky neaktualizovať častejšie ako',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Odkaz kanála',
 		'useragent' => 'Nastaviť používateľského agenta na sťahovanie tohto kanála',
 		'useragent_help' => 'Príklad: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/tr/sub.php b/app/i18n/tr/sub.php
index 2bdc551a8..293a33c1e 100644
--- a/app/i18n/tr/sub.php
+++ b/app/i18n/tr/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Başlık',
 		'title_add' => 'RSS akışı ekle',
 		'ttl' => 'Şu kadar süreden fazla otomatik yenileme yapma',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Akış URL',
 		'useragent' => 'Bu akışı yüklemek için user agent kullan',
 		'useragent_help' => 'Örnek: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/zh-cn/sub.php b/app/i18n/zh-cn/sub.php
index 274ef677b..8731d2ec2 100644
--- a/app/i18n/zh-cn/sub.php
+++ b/app/i18n/zh-cn/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => '标题',
 		'title_add' => '添加订阅源',
 		'ttl' => '最小自动更新间隔',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => '源地址',
 		'useragent' => '设置用于获取此源的 User Agent',
 		'useragent_help' => '例：<kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/zh-tw/sub.php b/app/i18n/zh-tw/sub.php
index 8cbc0f9ae..4934b5e66 100644
--- a/app/i18n/zh-tw/sub.php
+++ b/app/i18n/zh-tw/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => '標題',
 		'title_add' => '添加訂閱源',
 		'ttl' => '最小自動更新間隔',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => '源地址',
 		'useragent' => '設置用於獲取此源的 User Agent',
 		'useragent_help' => '例：<kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/views/helpers/feed/update.phtml b/app/views/helpers/feed/update.phtml
index 780e9667a..001c04131 100644
--- a/app/views/helpers/feed/update.phtml
+++ b/app/views/helpers/feed/update.phtml
@@ -101,6 +101,30 @@
 			</div>
 
 			<div class="form-group">
+				<label class="group-name" for="unicityCriteria"><?= _t('sub.feed.unicityCriteria') ?></label>
+				<?php
+					$unicityCriteria = $this->feed->attributeString('unicityCriteria');
+					if ($this->feed->attributeBoolean('hasBadGuids')) {	// Legacy
+						$unicityCriteria = 'link';
+					}
+				?>
+				<div class="group-controls">
+					<select class="w50" name="unicityCriteria" id="unicityCriteria" required="required">
+						<option value="id" <?= $unicityCriteria == null ? 'selected="selected"' : '' ?>><?= _t('sub.feed.unicityCriteria.id') ?></option>
+						<option value="link" <?= $unicityCriteria === 'link' ? 'selected="selected"' : '' ?>><?= _t('sub.feed.unicityCriteria.link') ?></option>
+						<option value="sha1:link_published" <?= $unicityCriteria === 'sha1:link_published' ? 'selected="selected"' : '' ?>><?= _t('sub.feed.unicityCriteria.sha1:link_published') ?></option>
+						<option value="sha1:link_published_title" <?= $unicityCriteria === 'sha1:link_published_title' ? 'selected="selected"' : '' ?>><?= _t('sub.feed.unicityCriteria.sha1:link_published_title') ?></option>
+						<option value="sha1:link_published_title_content" <?= $unicityCriteria === 'sha1:link_published_title_content' ? 'selected="selected"' : '' ?>><?= _t('sub.feed.unicityCriteria.sha1:link_published_title_content') ?></option>
+					</select>
+					<label for="unicityCriteriaForced" class="inline">
+						<input type="checkbox" name="unicityCriteriaForced" id="unicityCriteriaForced" value="1"<?= $this->feed->attributeBoolean('unicityCriteriaForced') ? ' checked="checked"' : '' ?> />
+						<?= _t('sub.feed.unicityCriteria.forced') ?>
+					</label>
+					<p class="help"><?= _i('help') ?> <?= _t('sub.feed.unicityCriteria.help') ?></p>
+				</div>
+			</div>
+
+			<div class="form-group">
 				<label class="group-name" for="ttl"><?= _t('sub.feed.ttl') ?></label>
 				<div class="group-controls">
 					<select class="w50" name="ttl" id="ttl" required="required"><?php