From cf899d8d25c57b05dff89b89e2c7e56808f83c50 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Mon, 5 Nov 2018 18:10:38 +0100 Subject: TT-RSS import (#2099) * TT-RSS import Import of Tiny Tiny RSS favourites https://github.com/FreshRSS/FreshRSS/issues/2018#issuecomment-432710462 * Fallback feed_url * Simpler JSON * TT-RSS import custom labels * Fix syntax --- app/Controllers/importExportController.php | 162 ++++++++++++++++++++++++----- app/Models/EntryDAO.php | 9 ++ app/Models/FeedDAO.php | 8 +- app/Models/TagDAO.php | 2 +- app/views/helpers/export/articles.phtml | 18 +++- 5 files changed, 171 insertions(+), 28 deletions(-) (limited to 'app') diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php index 0fb5ba651..c7b384540 100644 --- a/app/Controllers/importExportController.php +++ b/app/Controllers/importExportController.php @@ -109,6 +109,17 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { } } } + foreach ($list_files['ttrss_starred'] as $article_file) { + $json = $this->ttrssXmlToJson($article_file); + if (!$this->importJson($json, true)) { + $ok = false; + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during TT-RSS articles import' . "\n"); + } else { + Minz_Log::warning('Error during TT-RSS articles import'); + } + } + } return $ok; } @@ -165,17 +176,22 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { private static function guessFileType($filename) { if (substr_compare($filename, '.zip', -4) === 0) { return 'zip'; - } elseif (substr_compare($filename, '.opml', -5) === 0 || - substr_compare($filename, '.xml', -4) === 0) { + } elseif (substr_compare($filename, '.opml', -5) === 0) { return 'opml'; - } elseif (substr_compare($filename, '.json', -5) === 0 && - strpos($filename, 'starred') !== false) { - return 'json_starred'; } elseif (substr_compare($filename, '.json', -5) === 0) { - return 'json_feed'; - } else { - return 'unknown'; + if (strpos($filename, 'starred') !== false) { + return 'json_starred'; + } else { + return 'json_feed'; + } + } elseif (substr_compare($filename, '.xml', -4) === 0) { + if (preg_match('/Tiny|tt-?rss/i', $filename)) { + return 'ttrss_starred'; + } else { + return 'opml'; + } } + return 'unknown'; } /** @@ -364,6 +380,43 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { return !$error; } + private function ttrssXmlToJson($xml) { + $table = (array)simplexml_load_string($xml, null, LIBXML_NOCDATA); + $table['items'] = isset($table['article']) ? $table['article'] : array(); + unset($table['article']); + for ($i = count($table['items']) - 1; $i >= 0; $i--) { + $item = (array)($table['items'][$i]); + $item['updated'] = isset($item['updated']) ? strtotime($item['updated']) : ''; + $item['published'] = $item['updated']; + $item['content'] = array('content' => isset($item['content']) ? $item['content'] : ''); + $item['categories'] = isset($item['tag_cache']) ? array($item['tag_cache']) : array(); + if (!empty($item['marked'])) { + $item['categories'][] = 'user/-/state/com.google/starred'; + } + if (!empty($item['published'])) { + $item['categories'][] = 'user/-/state/com.google/broadcast'; + } + if (!empty($item['label_cache'])) { + $labels_cache = json_decode($item['label_cache'], true); + if (is_array($labels_cache)) { + foreach ($labels_cache as $label_cache) { + if (!empty($label_cache[1])) { + $item['categories'][] = 'user/-/label/' . trim($label_cache[1]); + } + } + } + } + $item['alternate'][0]['href'] = isset($item['link']) ? $item['link'] : ''; + $item['origin'] = array( + 'title' => isset($item['feed_title']) ? $item['feed_title'] : '', + 'feedUrl' => isset($item['feed_url']) ? $item['feed_url'] : '', + ); + $item['id'] = isset($item['guid']) ? $item['guid'] : (isset($item['feed_url']) ? $item['feed_url'] : $item['published']); + $table['items'][$i] = $item; + } + return json_encode($table); + } + /** * This method import a JSON-based file (Google Reader format). * @@ -405,7 +458,7 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { // Oops, no more place! Minz_Log::warning(_t('feedback.sub.feed.over_max', $limits['max_feeds'])); } else { - $feed = $this->addFeedJson($item['origin'], $google_compliant); + $feed = $this->addFeedJson($item['origin']); } if ($feed == null) { @@ -425,6 +478,15 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { } } + $tagDAO = FreshRSS_Factory::createTagDao(); + $labels = $tagDAO->listTags(); + $knownLabels = array(); + foreach ($labels as $label) { + $knownLabels[$label->name()]['id'] = $label->id(); + $knownLabels[$label->name()]['articles'] = array(); + } + unset($labels); + // For each feed, check existing GUIDs already in database. $existingHashForGuids = array(); foreach ($newFeedGuids as $feedId => $newGuids) { @@ -443,19 +505,36 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { $feed_id = $article_to_feed[$item['id']]; $author = isset($item['author']) ? $item['author'] : ''; - $key_content = ($google_compliant && !isset($item['content'])) ? 'summary' : 'content'; + $is_starred = $starred; $tags = $item['categories']; - if ($google_compliant) { - // Remove tags containing "/state/com.google" which are useless. - $tags = array_filter($tags, function($var) { - return strpos($var, '/state/com.google') !== false; - }); + $labels = array(); + for ($i = count($tags) - 1; $i >= 0; $i --) { + $tag = trim($tags[$i]); + if (strpos($tag, 'user/-/') !== false) { + if ($tag === 'user/-/state/com.google/starred') { + $is_starred = true; + } elseif (strpos($tag, 'user/-/label/') === 0) { + $tag = trim(substr($tag, 13)); + if ($tag != '') { + $labels[] = $tag; + } + } + unset($tags[$i]); + } + } + + $url = $item['alternate'][0]['href']; + if (!empty($item['content']['content'])) { + $content = $item['content']['content']; + } elseif (!empty($item['summary']['content'])) { + $content = $item['summary']['content']; } + $content = sanitizeHTML($content, $url); $entry = new FreshRSS_Entry( $feed_id, $item['id'], $item['title'], $author, - $item[$key_content]['content'], $item['alternate'][0]['href'], - $item['published'], $is_read, $starred + $content, $url, + $item['published'], $is_read, $is_starred ); $entry->_id(min(time(), $entry->date(true)) . uSecString()); $entry->_tags($tags); @@ -478,8 +557,21 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { } else { $ok = $this->entryDAO->addEntry($values); } - $error |= ($ok === false); + foreach ($labels as $labelName) { + if (empty($knownLabels[$labelName]['id'])) { + $labelId = $tagDAO->addTag(array('name' => $labelName)); + $knownLabels[$labelName]['id'] = $labelId; + $knownLabels[$labelName]['articles'] = array(); + } + $knownLabels[$labelName]['articles'][] = array( + //'id' => $entry->id(), //ID changes after commitNewEntries() + 'id_feed' => $entry->feed(), + 'guid' => $entry->guid(), + ); + } + + $error |= ($ok === false); } $this->entryDAO->commit(); @@ -488,6 +580,20 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { $this->feedDAO->updateCachedValues(); $this->entryDAO->commit(); + $this->entryDAO->beginTransaction(); + foreach ($knownLabels as $labelName => $knownLabel) { + $labelId = $knownLabel['id']; + foreach ($knownLabel['articles'] as $article) { + $entryId = $this->entryDAO->searchIdByGuid($article['id_feed'], $article['guid']); + if ($entryId != null) { + $tagDAO->tagEntry($labelId, $entryId); + } else { + Minz_Log::warning('Could not add label "' . $labelName . '" to entry "' . $article['guid'] . '" in feed ' . $article['id_feed']); + } + } + } + $this->entryDAO->commit(); + return !$error; } @@ -495,16 +601,24 @@ class FreshRSS_importExport_Controller extends Minz_ActionController { * This method import a JSON-based feed (Google Reader format). * * @param array $origin represents a feed. - * @param boolean $google_compliant takes care of some specific values if true. * @return FreshRSS_Feed if feed is in database at the end of the process, * else null. */ - private function addFeedJson($origin, $google_compliant) { + private function addFeedJson($origin) { $return = null; - $key = $google_compliant ? 'htmlUrl' : 'feedUrl'; - $url = $origin[$key]; - $name = $origin['title']; - $website = $origin['htmlUrl']; + if (!empty($origin['feedUrl'])) { + $url = $origin['feedUrl']; + } elseif (!empty($origin['htmlUrl'])) { + $url = $origin['htmlUrl']; + } else { + return null; + } + if (!empty($origin['htmlUrl'])) { + $website = $origin['htmlUrl']; + } elseif (!empty($origin['feedUrl'])) { + $website = $origin['feedUrl']; + } + $name = empty($origin['title']) ? '' : $origin['title']; try { // Create a Feed object and add it in database. diff --git a/app/Models/EntryDAO.php b/app/Models/EntryDAO.php index a01c2227b..708d01a69 100644 --- a/app/Models/EntryDAO.php +++ b/app/Models/EntryDAO.php @@ -671,6 +671,15 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable { return isset($entries[0]) ? $entries[0] : null; } + public function searchIdByGuid($id_feed, $guid) { + $sql = 'SELECT id FROM `' . $this->prefix . 'entry` WHERE id_feed=? AND guid=?'; + $stm = $this->bd->prepare($sql); + $values = array($id_feed, $guid); + $stm->execute($values); + $res = $stm->fetchAll(PDO::FETCH_COLUMN, 0); + return isset($res[0]) ? $res[0] : null; + } + protected function sqlConcat($s1, $s2) { return 'CONCAT(' . $s1 . ',' . $s2 . ')'; //MySQL } diff --git a/app/Models/FeedDAO.php b/app/Models/FeedDAO.php index e579f5881..7f00642f4 100644 --- a/app/Models/FeedDAO.php +++ b/app/Models/FeedDAO.php @@ -465,9 +465,15 @@ UPDATE `{$this->prefix}feed` SQL; $stm = $this->bd->prepare($sql); if (!($stm && $stm->execute(array(':new_value' => FreshRSS_Feed::TTL_DEFAULT, ':old_value' => -2)))) { + $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); + Minz_Log::error('SQL warning updateTTL 1: ' . $info[2] . ' ' . $sql); + $sql2 = 'ALTER TABLE `' . $this->prefix . 'feed` ADD COLUMN ttl INT NOT NULL DEFAULT ' . FreshRSS_Feed::TTL_DEFAULT; //v0.7.3 $stm = $this->bd->prepare($sql2); - $stm->execute(); + if (!($stm && $stm->execute())) { + $info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo(); + Minz_Log::error('SQL error updateTTL 2: ' . $info[2] . ' ' . $sql2); + } } else { $stm->execute(array(':new_value' => -3600, ':old_value' => -1)); } diff --git a/app/Models/TagDAO.php b/app/Models/TagDAO.php index 1b59c8971..b55d2b35d 100644 --- a/app/Models/TagDAO.php +++ b/app/Models/TagDAO.php @@ -266,7 +266,7 @@ class FreshRSS_TagDAO extends Minz_ModelPdo implements FreshRSS_Searchable { if (is_array($entries) && count($entries) > 0) { $sql .= ' AND et.id_entry IN (' . str_repeat('?,', count($entries) - 1). '?)'; foreach ($entries as $entry) { - $values[] = $entry->id(); + $values[] = is_array($entry) ? $entry['id'] : $entry->id(); } } $stm = $this->bd->prepare($sql); diff --git a/app/views/helpers/export/articles.phtml b/app/views/helpers/export/articles.phtml index b8958f527..59a2c7ad7 100644 --- a/app/views/helpers/export/articles.phtml +++ b/app/views/helpers/export/articles.phtml @@ -16,6 +16,12 @@ $articles = array( echo rtrim(json_encode($articles, $options), " ]}\n\r\t"), "\n"; $first = true; +$tagDAO = FreshRSS_Factory::createTagDao(); +$entryIdsTagNames = $tagDAO->getEntryIdsTagNames($this->entriesRaw); +if ($entryIdsTagNames == false) { + $entryIdsTagNames = array(); +} + foreach ($this->entriesRaw as $entryRaw) { if (empty($entryRaw)) { continue; @@ -32,13 +38,14 @@ foreach ($this->entriesRaw as $entryRaw) { $article = array( 'id' => $entry->guid(), + 'timestampUsec' => '' . $entry->id(), 'categories' => array_values($entry->tags()), 'title' => $entry->title(), - 'author' => $entry->authors(true), //TODO: Make an array like tags? + 'author' => $entry->authors(true), 'published' => $entry->date(true), 'updated' => $entry->date(true), 'alternate' => array(array( - 'href' => $entry->link(), + 'href' => htmlspecialchars_decode($entry->link(), ENT_QUOTES), 'type' => 'text/html', )), 'content' => array( @@ -51,6 +58,13 @@ foreach ($this->entriesRaw as $entryRaw) { 'feedUrl' => $feed == null ? '' : $feed->url(), ) ); + if ($entry->isFavorite()) { + $article['categories'][] = 'user/-/state/com.google/starred'; + } + $tagNames = isset($entryIdsTagNames['e_' . $entry->id()]) ? $entryIdsTagNames['e_' . $entry->id()] : array(); + foreach ($tagNames as $tagName) { + $article['categories'][] = 'user/-/label/' . $tagName; + } $line = json_encode($article, $options); if ($line != '') { -- cgit v1.2.3