New unicity policies for feeds with bad GUIDs (#4487)

New set of unicity criteria options.

New tolerance heuristic:

> `$invalidGuidsTolerance` (default 0.05) The maximum ratio (rounded) of invalid GUIDs to tolerate before degrading the unicity criteria.
> Example for 0.05 (5% rounded): tolerate 0 invalid GUIDs for up to 9 articles, 1 for 10, 2 for 30, 3 for 50, 4 for 70, 5 for 90, 6 for 110, etc.
> The default value of 5% rounded was chosen to allow 1 invalid GUID for feeds of 10 articles, which is a frequently observed amount of articles.

24 files changed, 240 insertions, 0 deletions

diff --git a/app/i18n/cs/sub.php b/app/i18n/cs/sub.php
index c60f47f95..6546a38af 100644
--- a/app/i18n/cs/sub.php
+++ b/app/i18n/cs/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Název',
 		'title_add' => 'Přidat kanál RSS',
 		'ttl' => 'Neobnovovat automaticky častěji než',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Adresa URL kanálu',
 		'useragent' => 'Nastavte uživatelský agent pro načítání tohoto kanálu',
 		'useragent_help' => 'Příklad: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/de/sub.php b/app/i18n/de/sub.php
index b8fba0a7e..30f77b398 100644
--- a/app/i18n/de/sub.php
+++ b/app/i18n/de/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Titel',
 		'title_add' => 'Einen RSS-Feed hinzufügen',
 		'ttl' => 'Aktualisiere automatisch nicht öfter als',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Feed-URL',
 		'useragent' => 'Browser User Agent für den Abruf des Feeds verwenden',
 		'useragent_help' => 'Beispiel: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/el/sub.php b/app/i18n/el/sub.php
index 47e442b13..02821ea6d 100644
--- a/app/i18n/el/sub.php
+++ b/app/i18n/el/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Title',	// TODO
 		'title_add' => 'Add an RSS feed',	// TODO
 		'ttl' => 'Do not automatically refresh more often than',	// TODO
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Feed URL',	// TODO
 		'useragent' => 'Set the user agent for fetching this feed',	// TODO
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',	// TODO
diff --git a/app/i18n/en-us/sub.php b/app/i18n/en-us/sub.php
index 93e7bf21b..ef7931681 100644
--- a/app/i18n/en-us/sub.php
+++ b/app/i18n/en-us/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Title',	// IGNORE
 		'title_add' => 'Add an RSS feed',	// IGNORE
 		'ttl' => 'Do not automatically refresh more often than',	// IGNORE
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// IGNORE
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// IGNORE
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// IGNORE
+			'id' => 'Standard ID (default)',	// IGNORE
+			'link' => 'Link',	// IGNORE
+			'sha1:link_published' => 'Link + Date',	// IGNORE
+			'sha1:link_published_title' => 'Link + Date + Title',	// IGNORE
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// IGNORE
+		),
 		'url' => 'Feed URL',	// IGNORE
 		'useragent' => 'Set the user agent for fetching this feed',	// IGNORE
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',	// IGNORE
diff --git a/app/i18n/en/sub.php b/app/i18n/en/sub.php
index ca4da3409..0a6385428 100644
--- a/app/i18n/en/sub.php
+++ b/app/i18n/en/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Title',
 		'title_add' => 'Add an RSS feed',
 		'ttl' => 'Do not automatically refresh more often than',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',
+			'link' => 'Link',
+			'sha1:link_published' => 'Link + Date',
+			'sha1:link_published_title' => 'Link + Date + Title',
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',
+		),
 		'url' => 'Feed URL',
 		'useragent' => 'Set the user agent for fetching this feed',
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/es/sub.php b/app/i18n/es/sub.php
index 5615d5ac9..f5c557db1 100644
--- a/app/i18n/es/sub.php
+++ b/app/i18n/es/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Título',
 		'title_add' => 'Añadir fuente RSS',
 		'ttl' => 'No actualizar de forma automática con una frecuencia mayor a',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'URL de la fuente',
 		'useragent' => 'Selecciona el agente de usuario por recuperar la fuente',
 		'useragent_help' => 'Ejemplo: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/fa/sub.php b/app/i18n/fa/sub.php
index 8ad8f3090..93ff0f205 100644
--- a/app/i18n/fa/sub.php
+++ b/app/i18n/fa/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => ' عنوان',
 		'title_add' => ' یک فید RSS اضافه کنید',
 		'ttl' => ' به‌طور خودکار بیشتر از آن رفرش نکنید',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => ' URL فید',
 		'useragent' => ' عامل کاربر را برای واکشی این فید تنظیم کنید',
 		'useragent_help' => ' مثال: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/fr/sub.php b/app/i18n/fr/sub.php
index 2c6ec36cb..24678f377 100644
--- a/app/i18n/fr/sub.php
+++ b/app/i18n/fr/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Titre',
 		'title_add' => 'Ajouter un flux RSS',
 		'ttl' => 'Ne pas automatiquement rafraîchir plus souvent que',
+		'unicityCriteria' => array(
+			'_' => 'Critère d’unicité des articles',
+			'forced' => '<span title="Bloque le critère d’unicité même en cas de doublons">forcé</span>',
+			'help' => 'Utile pour les flux invalides.<br />⚠️ Changer le critère peut créer des doublons.',
+			'id' => 'ID standard (défaut)',
+			'link' => 'Lien',
+			'sha1:link_published' => 'Lien + Date',
+			'sha1:link_published_title' => 'Lien + Date + Titre',
+			'sha1:link_published_title_content' => 'Lien + Date + Titre + Contenu',
+		),
 		'url' => 'URL du flux',
 		'useragent' => 'Sélectionner l’agent utilisateur pour télécharger ce flux',
 		'useragent_help' => 'Exemple : <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/he/sub.php b/app/i18n/he/sub.php
index 4accafe33..eb4df468c 100644
--- a/app/i18n/he/sub.php
+++ b/app/i18n/he/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'כותרת',
 		'title_add' => 'הוספת הזנה',
 		'ttl' => 'אין לרענן אוטומטית יותר מ',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'הזנה URL',
 		'useragent' => 'Set the user agent for fetching this feed',	// TODO
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',	// TODO
diff --git a/app/i18n/hu/sub.php b/app/i18n/hu/sub.php
index 7b9a33d98..397907f0f 100644
--- a/app/i18n/hu/sub.php
+++ b/app/i18n/hu/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Cím',
 		'title_add' => 'RSS hírforrás hozzáadása',
 		'ttl' => 'Ne frissítsd automatikusan többször mint',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Hírforrás URL',
 		'useragent' => 'Állíts be egy user agent-et ehhez a hírforráshoz',
 		'useragent_help' => 'Példa: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/id/sub.php b/app/i18n/id/sub.php
index d8fde0251..f192dacec 100644
--- a/app/i18n/id/sub.php
+++ b/app/i18n/id/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Title',	// TODO
 		'title_add' => 'Add an RSS feed',	// TODO
 		'ttl' => 'Do not automatically refresh more often than',	// TODO
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Feed URL',	// TODO
 		'useragent' => 'Set the user agent for fetching this feed',	// TODO
 		'useragent_help' => 'Example: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',	// TODO
diff --git a/app/i18n/it/sub.php b/app/i18n/it/sub.php
index e36a144ec..35b8fab60 100644
--- a/app/i18n/it/sub.php
+++ b/app/i18n/it/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Titolo',
 		'title_add' => 'Aggiungi RSS feed',
 		'ttl' => 'Non aggiornare automaticamente piu di',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'URL del feed',
 		'useragent' => 'Imposta lo user agent per recuperare questo feed',
 		'useragent_help' => 'Esempio: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/ja/sub.php b/app/i18n/ja/sub.php
index 00234c0af..a1a02d2a9 100644
--- a/app/i18n/ja/sub.php
+++ b/app/i18n/ja/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'タイトル',
 		'title_add' => 'RSS フィードを追加する',
 		'ttl' => '自動更新の頻度',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'フィードのURL',
 		'useragent' => 'フィードを読み込む際のユーザーエージェントを設定してください',
 		'useragent_help' => '例: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/ko/sub.php b/app/i18n/ko/sub.php
index e28f22f74..917d71b68 100644
--- a/app/i18n/ko/sub.php
+++ b/app/i18n/ko/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => '제목',
 		'title_add' => 'RSS 피드 추가',
 		'ttl' => '다음 시간이 지나기 전에 새로고침 금지',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => '피드 URL',
 		'useragent' => '이 피드를 가져올 때 사용할 유저 에이전트 설정',
 		'useragent_help' => '예시: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/lv/sub.php b/app/i18n/lv/sub.php
index d81da3718..ea67d3005 100644
--- a/app/i18n/lv/sub.php
+++ b/app/i18n/lv/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Tituls',
 		'title_add' => 'Pievienot RSS barotni',
 		'ttl' => 'Automātiski neatjaunināt biežāk par',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Barotnes URL',
 		'useragent' => 'Lietotāja aģenta iestatīšana šīs barotnes iegūšanai',
 		'useragent_help' => 'Piemērs: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/nl/sub.php b/app/i18n/nl/sub.php
index 817fa8b10..ae0b47391 100644
--- a/app/i18n/nl/sub.php
+++ b/app/i18n/nl/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Titel',
 		'title_add' => 'Voeg een RSS-feed toe',
 		'ttl' => 'Vernieuw automatisch niet vaker dan',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Feed-url',
 		'useragent' => 'Stelt de useragent in om deze feed op te halen',
 		'useragent_help' => 'Voorbeeld: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/oc/sub.php b/app/i18n/oc/sub.php
index 6b8ea93bb..c0f0ae245 100644
--- a/app/i18n/oc/sub.php
+++ b/app/i18n/oc/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Títol',
 		'title_add' => 'Ajustar un flux RSS',
 		'ttl' => 'Actualizar pas automaticament mai sovent que',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Flux URL',
 		'useragent' => 'Definir un user agent per recuperar aqueste flux',
 		'useragent_help' => 'Exemple : <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/pl/sub.php b/app/i18n/pl/sub.php
index 4f99c2866..30cd09a50 100644
--- a/app/i18n/pl/sub.php
+++ b/app/i18n/pl/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Tytuł',
 		'title_add' => 'Dodaj kanał',
 		'ttl' => 'Nie odświeżaj automatycznie częściej niż',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Adres kanału',
 		'useragent' => 'Ciąg user agent używany podczas pobierania kanału',
 		'useragent_help' => 'Przykład: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/pt-br/sub.php b/app/i18n/pt-br/sub.php
index d1c2f08e5..4d6f074e7 100644
--- a/app/i18n/pt-br/sub.php
+++ b/app/i18n/pt-br/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Título',
 		'title_add' => 'Adicionar o RSS feed',
 		'ttl' => 'Não atualize automaticamente mais que',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'URL do Feed',
 		'useragent' => 'Defina um usuário para buscar este feed',
 		'useragent_help' => 'Exemplo: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/ru/sub.php b/app/i18n/ru/sub.php
index 946f53177..569e9202d 100644
--- a/app/i18n/ru/sub.php
+++ b/app/i18n/ru/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Заголовок',
 		'title_add' => 'Добавить RSS-ленту',
 		'ttl' => 'Не обновлять автоматически чаще, чем каждые',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'URL ленты',
 		'useragent' => 'Указать юзерагент для извлечения лент',
 		'useragent_help' => 'Пример: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/sk/sub.php b/app/i18n/sk/sub.php
index a019db2a9..ef4060352 100644
--- a/app/i18n/sk/sub.php
+++ b/app/i18n/sk/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Nadpis',
 		'title_add' => 'Pridať kanál RSS',
 		'ttl' => 'Automaticky neaktualizovať častejšie ako',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Odkaz kanála',
 		'useragent' => 'Nastaviť používateľského agenta na sťahovanie tohto kanála',
 		'useragent_help' => 'Príklad: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/tr/sub.php b/app/i18n/tr/sub.php
index 2bdc551a8..293a33c1e 100644
--- a/app/i18n/tr/sub.php
+++ b/app/i18n/tr/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => 'Başlık',
 		'title_add' => 'RSS akışı ekle',
 		'ttl' => 'Şu kadar süreden fazla otomatik yenileme yapma',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => 'Akış URL',
 		'useragent' => 'Bu akışı yüklemek için user agent kullan',
 		'useragent_help' => 'Örnek: <kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/zh-cn/sub.php b/app/i18n/zh-cn/sub.php
index 274ef677b..8731d2ec2 100644
--- a/app/i18n/zh-cn/sub.php
+++ b/app/i18n/zh-cn/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => '标题',
 		'title_add' => '添加订阅源',
 		'ttl' => '最小自动更新间隔',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => '源地址',
 		'useragent' => '设置用于获取此源的 User Agent',
 		'useragent_help' => '例：<kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',
diff --git a/app/i18n/zh-tw/sub.php b/app/i18n/zh-tw/sub.php
index 8cbc0f9ae..4934b5e66 100644
--- a/app/i18n/zh-tw/sub.php
+++ b/app/i18n/zh-tw/sub.php
@@ -228,6 +228,16 @@ return array(
 		'title' => '標題',
 		'title_add' => '添加訂閱源',
 		'ttl' => '最小自動更新間隔',
+		'unicityCriteria' => array(
+			'_' => 'Article unicity criteria',	// TODO
+			'forced' => '<span title="Block the unicity criteria, even when the feed has duplicate articles">forced</span>',	// TODO
+			'help' => 'Relevant for invalid feeds.<br />⚠️ Changing the policy will create duplicates.',	// TODO
+			'id' => 'Standard ID (default)',	// TODO
+			'link' => 'Link',	// TODO
+			'sha1:link_published' => 'Link + Date',	// TODO
+			'sha1:link_published_title' => 'Link + Date + Title',	// TODO
+			'sha1:link_published_title_content' => 'Link + Date + Title + Content',	// TODO
+		),
 		'url' => '源地址',
 		'useragent' => '設置用於獲取此源的 User Agent',
 		'useragent_help' => '例：<kbd>Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0)</kbd>',