diff options
Diffstat (limited to 'app/Controllers/importExportController.php')
| -rw-r--r-- | app/Controllers/importExportController.php | 716 |
1 files changed, 716 insertions, 0 deletions
diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php new file mode 100644 index 000000000..6ae89defb --- /dev/null +++ b/app/Controllers/importExportController.php @@ -0,0 +1,716 @@ +<?php + +/** + * Controller to handle every import and export actions. + */ +class FreshRSS_importExport_Controller extends Minz_ActionController { + /** + * This action is called before every other action in that class. It is + * the common boiler plate for every action. It is triggered by the + * underlying framework. + */ + public function firstAction() { + if (!FreshRSS_Auth::hasAccess()) { + Minz_Error::error(403); + } + + require_once(LIB_PATH . '/lib_opml.php'); + + $this->catDAO = new FreshRSS_CategoryDAO(); + $this->entryDAO = FreshRSS_Factory::createEntryDao(); + $this->feedDAO = FreshRSS_Factory::createFeedDao(); + } + + /** + * This action displays the main page for import / export system. + */ + public function indexAction() { + $this->view->feeds = $this->feedDAO->listFeeds(); + Minz_View::prependTitle(_t('sub.import_export.title') . ' ยท '); + } + + public function importFile($name, $path, $username = null) { + require_once(LIB_PATH . '/lib_opml.php'); + + $this->catDAO = new FreshRSS_CategoryDAO($username); + $this->entryDAO = FreshRSS_Factory::createEntryDao($username); + $this->feedDAO = FreshRSS_Factory::createFeedDao($username); + + $type_file = self::guessFileType($name); + + $list_files = array( + 'opml' => array(), + 'json_starred' => array(), + 'json_feed' => array() + ); + + // We try to list all files according to their type + $list = array(); + if ($type_file === 'zip' && extension_loaded('zip')) { + $zip = zip_open($path); + if (!is_resource($zip)) { + // zip_open cannot open file: something is wrong + throw new FreshRSS_Zip_Exception($zip); + } + while (($zipfile = zip_read($zip)) !== false) { + if (!is_resource($zipfile)) { + // zip_entry() can also return an error code! + throw new FreshRSS_Zip_Exception($zipfile); + } else { + $type_zipfile = self::guessFileType(zip_entry_name($zipfile)); + if ($type_file !== 'unknown') { + $list_files[$type_zipfile][] = zip_entry_read( + $zipfile, + zip_entry_filesize($zipfile) + ); + } + } + } + zip_close($zip); + } elseif ($type_file === 'zip') { + // ZIP extension is not loaded + throw new FreshRSS_ZipMissing_Exception(); + } elseif ($type_file !== 'unknown') { + $list_files[$type_file][] = file_get_contents($path); + } + + // Import file contents. + // OPML first(so categories and feeds are imported) + // Starred articles then so the "favourite" status is already set + // And finally all other files. + $ok = true; + foreach ($list_files['opml'] as $opml_file) { + if (!$this->importOpml($opml_file)) { + $ok = false; + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during OPML import' . "\n"); + } else { + Minz_Log::warning('Error during OPML import'); + } + } + } + foreach ($list_files['json_starred'] as $article_file) { + if (!$this->importJson($article_file, true)) { + $ok = false; + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during JSON stars import' . "\n"); + } else { + Minz_Log::warning('Error during JSON stars import'); + } + } + } + foreach ($list_files['json_feed'] as $article_file) { + if (!$this->importJson($article_file)) { + $ok = false; + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during JSON feeds import' . "\n"); + } else { + Minz_Log::warning('Error during JSON feeds import'); + } + } + } + + return $ok; + } + + /** + * This action handles import action. + * + * It must be reached by a POST request. + * + * Parameter is: + * - file (default: nothing!) + * Available file types are: zip, json or xml. + */ + public function importAction() { + if (!Minz_Request::isPost()) { + Minz_Request::forward(array('c' => 'importExport', 'a' => 'index'), true); + } + + $file = $_FILES['file']; + $status_file = $file['error']; + + if ($status_file !== 0) { + Minz_Log::warning('File cannot be uploaded. Error code: ' . $status_file); + Minz_Request::bad(_t('feedback.import_export.file_cannot_be_uploaded'), + array('c' => 'importExport', 'a' => 'index')); + } + + @set_time_limit(300); + + $error = false; + try { + $error = !$this->importFile($file['name'], $file['tmp_name']); + } catch (FreshRSS_ZipMissing_Exception $zme) { + Minz_Request::bad(_t('feedback.import_export.no_zip_extension'), + array('c' => 'importExport', 'a' => 'index')); + } catch (FreshRSS_Zip_Exception $ze) { + Minz_Log::warning('ZIP archive cannot be imported. Error code: ' . $ze->zipErrorCode()); + Minz_Request::bad(_t('feedback.import_export.zip_error'), + array('c' => 'importExport', 'a' => 'index')); + } + + // And finally, we get import status and redirect to the home page + Minz_Session::_param('actualize_feeds', true); + $content_notif = $error === true ? _t('feedback.import_export.feeds_imported_with_errors') : + _t('feedback.import_export.feeds_imported'); + Minz_Request::good($content_notif); + } + + /** + * This method tries to guess the file type based on its name. + * + * Itis a *very* basic guess file type function. Only based on filename. + * That's could be improved but should be enough for what we have to do. + */ + private static function guessFileType($filename) { + if (substr_compare($filename, '.zip', -4) === 0) { + return 'zip'; + } elseif (substr_compare($filename, '.opml', -5) === 0 || + substr_compare($filename, '.xml', -4) === 0) { + return 'opml'; + } elseif (substr_compare($filename, '.json', -5) === 0 && + strpos($filename, 'starred') !== false) { + return 'json_starred'; + } elseif (substr_compare($filename, '.json', -5) === 0) { + return 'json_feed'; + } else { + return 'unknown'; + } + } + + /** + * This method parses and imports an OPML file. + * + * @param string $opml_file the OPML file content. + * @return boolean false if an error occured, true otherwise. + */ + private function importOpml($opml_file) { + $opml_array = array(); + try { + $opml_array = libopml_parse_string($opml_file, false); + } catch (LibOPML_Exception $e) { + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during OPML parsing: ' . $e->getMessage() . "\n"); + } else { + Minz_Log::warning($e->getMessage()); + } + return false; + } + + $this->catDAO->checkDefault(); + + return $this->addOpmlElements($opml_array['body']); + } + + /** + * This method imports an OPML file based on its body. + * + * @param array $opml_elements an OPML element (body or outline). + * @param string $parent_cat the name of the parent category. + * @return boolean false if an error occured, true otherwise. + */ + private function addOpmlElements($opml_elements, $parent_cat = null) { + $ok = true; + + $nb_feeds = count($this->feedDAO->listFeeds()); + $nb_cats = count($this->catDAO->listCategories(false)); + $limits = FreshRSS_Context::$system_conf->limits; + + foreach ($opml_elements as $elt) { + if (isset($elt['xmlUrl'])) { + // If xmlUrl exists, it means it is a feed + if (FreshRSS_Context::$isCli && $nb_feeds >= $limits['max_feeds']) { + Minz_Log::warning(_t('feedback.sub.feed.over_max', + $limits['max_feeds'])); + $ok = false; + continue; + } + + if ($this->addFeedOpml($elt, $parent_cat)) { + $nb_feeds++; + } else { + $ok = false; + } + } else { + // No xmlUrl? It should be a category! + $limit_reached = ($nb_cats >= $limits['max_categories']); + if (!FreshRSS_Context::$isCli && $limit_reached) { + Minz_Log::warning(_t('feedback.sub.category.over_max', + $limits['max_categories'])); + $ok = false; + continue; + } + + if ($this->addCategoryOpml($elt, $parent_cat, $limit_reached)) { + $nb_cats++; + } else { + $ok = false; + } + } + } + + return $ok; + } + + /** + * This method imports an OPML feed element. + * + * @param array $feed_elt an OPML element (must be a feed element). + * @param string $parent_cat the name of the parent category. + * @return boolean false if an error occured, true otherwise. + */ + private function addFeedOpml($feed_elt, $parent_cat) { + if ($parent_cat == null) { + // This feed has no parent category so we get the default one + $this->catDAO->checkDefault(); + $default_cat = $this->catDAO->getDefault(); + $parent_cat = $default_cat->name(); + } + + $cat = $this->catDAO->searchByName($parent_cat); + if ($cat == null) { + // If there is not $cat, it means parent category does not exist in + // database. + // If it happens, take the default category. + $this->catDAO->checkDefault(); + $cat = $this->catDAO->getDefault(); + } + + // We get different useful information + $url = Minz_Helper::htmlspecialchars_utf8($feed_elt['xmlUrl']); + $name = Minz_Helper::htmlspecialchars_utf8($feed_elt['text']); + $website = ''; + if (isset($feed_elt['htmlUrl'])) { + $website = Minz_Helper::htmlspecialchars_utf8($feed_elt['htmlUrl']); + } + $description = ''; + if (isset($feed_elt['description'])) { + $description = Minz_Helper::htmlspecialchars_utf8($feed_elt['description']); + } + + $error = false; + try { + // Create a Feed object and add it in DB + $feed = new FreshRSS_Feed($url); + $feed->_category($cat->id()); + $feed->_name($name); + $feed->_website($website); + $feed->_description($description); + + // Call the extension hook + $feed = Minz_ExtensionManager::callHook('feed_before_insert', $feed); + if ($feed != null) { + // addFeedObject checks if feed is already in DB so nothing else to + // check here + $id = $this->feedDAO->addFeedObject($feed); + $error = ($id === false); + } else { + $error = true; + } + } catch (FreshRSS_Feed_Exception $e) { + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during OPML feed import: ' . $e->getMessage() . "\n"); + } else { + Minz_Log::warning($e->getMessage()); + } + $error = true; + } + + if ($error) { + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during OPML feed import from URL: ' . $url . ' in category ' . $cat->id() . "\n"); + } else { + Minz_Log::warning('Error during OPML feed import from URL: ' . $url . ' in category ' . $cat->id()); + } + } + + return !$error; + } + + /** + * This method imports an OPML category element. + * + * @param array $cat_elt an OPML element (must be a category element). + * @param string $parent_cat the name of the parent category. + * @param boolean $cat_limit_reached indicates if category limit has been reached. + * if yes, category is not added (but we try for feeds!) + * @return boolean false if an error occured, true otherwise. + */ + private function addCategoryOpml($cat_elt, $parent_cat, $cat_limit_reached) { + // Create a new Category object + $catName = Minz_Helper::htmlspecialchars_utf8($cat_elt['text']); + $cat = new FreshRSS_Category($catName); + + $error = true; + if (FreshRSS_Context::$isCli || !$cat_limit_reached) { + $id = $this->catDAO->addCategoryObject($cat); + $error = ($id === false); + } + if ($error) { + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during OPML category import from URL: ' . $catName . "\n"); + } else { + Minz_Log::warning('Error during OPML category import from URL: ' . $catName); + } + } + + if (isset($cat_elt['@outlines'])) { + // Our cat_elt contains more categories or more feeds, so we + // add them recursively. + // Note: FreshRSS does not support yet category arborescence + $error &= !$this->addOpmlElements($cat_elt['@outlines'], $catName); + } + + return !$error; + } + + /** + * This method import a JSON-based file (Google Reader format). + * + * @param string $article_file the JSON file content. + * @param boolean $starred true if articles from the file must be starred. + * @return boolean false if an error occured, true otherwise. + */ + private function importJson($article_file, $starred = false) { + $article_object = json_decode($article_file, true); + if ($article_object == null) { + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error trying to import a non-JSON file' . "\n"); + } else { + Minz_Log::warning('Try to import a non-JSON file'); + } + return false; + } + + $is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0; + + $google_compliant = strpos($article_object['id'], 'com.google') !== false; + + $error = false; + $article_to_feed = array(); + + $nb_feeds = count($this->feedDAO->listFeeds()); + $limits = FreshRSS_Context::$system_conf->limits; + + // First, we check feeds of articles are in DB (and add them if needed). + foreach ($article_object['items'] as $item) { + $key = $google_compliant ? 'htmlUrl' : 'feedUrl'; + $feed = new FreshRSS_Feed($item['origin'][$key]); + $feed = $this->feedDAO->searchByUrl($feed->url()); + + if ($feed == null) { + // Feed does not exist in DB,we should to try to add it. + if ((!FreshRSS_Context::$isCli) && ($nb_feeds >= $limits['max_feeds'])) { + // Oops, no more place! + Minz_Log::warning(_t('feedback.sub.feed.over_max', $limits['max_feeds'])); + } else { + $feed = $this->addFeedJson($item['origin'], $google_compliant); + } + + if ($feed == null) { + // Still null? It means something went wrong. + $error = true; + } else { + $nb_feeds++; + } + } + + if ($feed != null) { + $article_to_feed[$item['id']] = $feed->id(); + } + } + + $newGuids = array(); + foreach ($article_object['items'] as $item) { + $newGuids[] = safe_ascii($item['id']); + } + // For this feed, check existing GUIDs already in database. + $existingHashForGuids = $this->entryDAO->listHashForFeedGuids($feed->id(), $newGuids); + unset($newGuids); + + // Then, articles are imported. + $this->entryDAO->beginTransaction(); + foreach ($article_object['items'] as $item) { + if (!isset($article_to_feed[$item['id']])) { + // Related feed does not exist for this entry, do nothing. + continue; + } + + $feed_id = $article_to_feed[$item['id']]; + $author = isset($item['author']) ? $item['author'] : ''; + $key_content = ($google_compliant && !isset($item['content'])) ? + 'summary' : 'content'; + $tags = $item['categories']; + if ($google_compliant) { + // Remove tags containing "/state/com.google" which are useless. + $tags = array_filter($tags, function($var) { + return strpos($var, '/state/com.google') !== false; + }); + } + + $entry = new FreshRSS_Entry( + $feed_id, $item['id'], $item['title'], $author, + $item[$key_content]['content'], $item['alternate'][0]['href'], + $item['published'], $is_read, $starred + ); + $entry->_id(min(time(), $entry->date(true)) . uSecString()); + $entry->_tags($tags); + + $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); + if ($entry == null) { + // An extension has returned a null value, there is nothing to insert. + continue; + } + + $values = $entry->toArray(); + if (isset($existingHashForGuids[$entry->guid()])) { + $id = $this->entryDAO->updateEntry($values); + } else { + $id = $this->entryDAO->addEntry($values); + } + + if (!$error && ($id === false)) { + $error = true; + } + } + $this->entryDAO->commit(); + + return !$error; + } + + /** + * This method import a JSON-based feed (Google Reader format). + * + * @param array $origin represents a feed. + * @param boolean $google_compliant takes care of some specific values if true. + * @return FreshRSS_Feed if feed is in database at the end of the process, + * else null. + */ + private function addFeedJson($origin, $google_compliant) { + $return = null; + $key = $google_compliant ? 'htmlUrl' : 'feedUrl'; + $url = $origin[$key]; + $name = $origin['title']; + $website = $origin['htmlUrl']; + + try { + // Create a Feed object and add it in database. + $feed = new FreshRSS_Feed($url); + $feed->_category(FreshRSS_CategoryDAO::defaultCategoryId); + $feed->_name($name); + $feed->_website($website); + + // Call the extension hook + $feed = Minz_ExtensionManager::callHook('feed_before_insert', $feed); + if ($feed != null) { + // addFeedObject checks if feed is already in DB so nothing else to + // check here. + $id = $this->feedDAO->addFeedObject($feed); + + if ($id !== false) { + $feed->_id($id); + $return = $feed; + } + } + } catch (FreshRSS_Feed_Exception $e) { + if (FreshRSS_Context::$isCli) { + fwrite(STDERR, 'FreshRSS error during JSON feed import: ' . $e->getMessage() . "\n"); + } else { + Minz_Log::warning($e->getMessage()); + } + } + + return $return; + } + + public function exportFile($export_opml = true, $export_starred = false, $export_feeds = array(), $maxFeedEntries = 50, $username = null) { + require_once(LIB_PATH . '/lib_opml.php'); + + $this->catDAO = new FreshRSS_CategoryDAO($username); + $this->entryDAO = FreshRSS_Factory::createEntryDao($username); + $this->feedDAO = FreshRSS_Factory::createFeedDao($username); + + $this->entryDAO->disableBuffering(); + + if ($export_feeds === true) { + //All feeds + $export_feeds = $this->feedDAO->listFeedsIds(); + } + if (!is_array($export_feeds)) { + $export_feeds = array(); + } + + $day = date('Y-m-d'); + + $export_files = array(); + if ($export_opml) { + $export_files["feeds_${day}.opml.xml"] = $this->generateOpml(); + } + + if ($export_starred) { + $export_files["starred_${day}.json"] = $this->generateEntries('starred'); + } + + foreach ($export_feeds as $feed_id) { + $feed = $this->feedDAO->searchById($feed_id); + if ($feed) { + $filename = "feed_${day}_" . $feed->category() . '_' + . $feed->id() . '.json'; + $export_files[$filename] = $this->generateEntries('feed', $feed, $maxFeedEntries); + } + } + + $nb_files = count($export_files); + if ($nb_files > 1) { + // If there are more than 1 file to export, we need a ZIP archive. + try { + $this->sendZip($export_files); + } catch (Exception $e) { + throw new FreshRSS_ZipMissing_Exception($e); + } + } elseif ($nb_files === 1) { + // Only one file? Guess its type and export it. + $filename = key($export_files); + $type = self::guessFileType($filename); + $this->sendFile('freshrss_' . $filename, $export_files[$filename], $type); + } + return $nb_files; + } + + /** + * This action handles export action. + * + * This action must be reached by a POST request. + * + * Parameters are: + * - export_opml (default: false) + * - export_starred (default: false) + * - export_feeds (default: array()) a list of feed ids + */ + public function exportAction() { + if (!Minz_Request::isPost()) { + Minz_Request::forward(array('c' => 'importExport', 'a' => 'index'), true); + } + $this->view->_useLayout(false); + + $nb_files = 0; + try { + $nb_files = $this->exportFile( + Minz_Request::param('export_opml', false), + Minz_Request::param('export_starred', false), + Minz_Request::param('export_feeds', array()) + ); + } catch (FreshRSS_ZipMissing_Exception $zme) { + # Oops, there is no ZIP extension! + Minz_Request::bad(_t('feedback.import_export.export_no_zip_extension'), + array('c' => 'importExport', 'a' => 'index')); + } + + if ($nb_files < 1) { + // Nothing to do... + Minz_Request::forward(array('c' => 'importExport', 'a' => 'index'), true); + } + } + + /** + * This method returns the OPML file based on user subscriptions. + * + * @return string the OPML file content. + */ + private function generateOpml() { + $list = array(); + foreach ($this->catDAO->listCategories() as $key => $cat) { + $list[$key]['name'] = $cat->name(); + $list[$key]['feeds'] = $this->feedDAO->listByCategory($cat->id()); + } + + $this->view->categories = $list; + return $this->view->helperToString('export/opml'); + } + + /** + * This method returns a JSON file content. + * + * @param string $type must be "starred" or "feed" + * @param FreshRSS_Feed $feed feed of which we want to get entries. + * @return string the JSON file content. + */ + private function generateEntries($type, $feed = NULL, $maxFeedEntries = 50) { + $this->view->categories = $this->catDAO->listCategories(); + + if ($type == 'starred') { + $this->view->list_title = _t('sub.import_export.starred_list'); + $this->view->type = 'starred'; + $unread_fav = $this->entryDAO->countUnreadReadFavorites(); + $this->view->entriesRaw = $this->entryDAO->listWhereRaw( + 's', '', FreshRSS_Entry::STATE_ALL, 'ASC', $unread_fav['all'] + ); + } elseif ($type === 'feed' && $feed != null) { + $this->view->list_title = _t('sub.import_export.feed_list', $feed->name()); + $this->view->type = 'feed/' . $feed->id(); + $this->view->entriesRaw = $this->entryDAO->listWhereRaw( + 'f', $feed->id(), FreshRSS_Entry::STATE_ALL, 'ASC', + $maxFeedEntries + ); + $this->view->feed = $feed; + } + + return $this->view->helperToString('export/articles'); + } + + /** + * This method zips a list of files and returns it by HTTP. + * + * @param array $files list of files where key is filename and value the content. + * @throws Exception if Zip extension is not loaded. + */ + private function sendZip($files) { + if (!extension_loaded('zip')) { + throw new Exception(); + } + + // From https://stackoverflow.com/questions/1061710/php-zip-files-on-the-fly + $zip_file = tempnam('tmp', 'zip'); + $zip = new ZipArchive(); + $zip->open($zip_file, ZipArchive::OVERWRITE); + + foreach ($files as $filename => $content) { + $zip->addFromString($filename, $content); + } + + // Close and send to user + $zip->close(); + header('Content-Type: application/zip'); + header('Content-Length: ' . filesize($zip_file)); + $day = date('Y-m-d'); + header('Content-Disposition: attachment; filename="freshrss_' . $day . '_export.zip"'); + readfile($zip_file); + unlink($zip_file); + } + + /** + * This method returns a single file (OPML or JSON) by HTTP. + * + * @param string $filename + * @param string $content + * @param string $type the file type (opml, json_feed or json_starred). + * If equals to unknown, nothing happens. + */ + private function sendFile($filename, $content, $type) { + if ($type === 'unknown') { + return; + } + + $content_type = ''; + if ($type === 'opml') { + $content_type = 'application/xml'; + } elseif ($type === 'json_feed' || $type === 'json_starred') { + $content_type = 'application/json'; + } + + header('Content-Type: ' . $content_type . '; charset=utf-8'); + header('Content-disposition: attachment; filename=' . $filename); + print($content); + } +} |