Move translation tools into the cli folder (#1673)

Translation tools must be used on cli. It is better to have them in the cli folder.

9 files changed, 677 insertions, 0 deletions

diff --git a/cli/check.translation.php b/cli/check.translation.php
new file mode 100644
index 000000000..6ebd12973
--- /dev/null
+++ b/cli/check.translation.php
@@ -0,0 +1,106 @@
+<?php
+
+require_once __DIR__ . '/i18n/I18nFile.php';
+require_once __DIR__ . '/i18n/I18nCompletionValidator.php';
+require_once __DIR__ . '/i18n/I18nUsageValidator.php';
+
+$i18nFile = new I18nFile();
+$i18nData = $i18nFile->load();
+
+$options = getopt("dhl:r");
+
+if (array_key_exists('h', $options)) {
+	help();
+}
+if (array_key_exists('l', $options)) {
+	$languages = array($options['l']);
+} else {
+	$languages = $i18nData->getAvailableLanguages();
+}
+$displayResults = array_key_exists('d', $options);
+$displayReport = array_key_exists('r', $options);
+
+$isValidated = true;
+$result = array();
+$report = array();
+
+foreach ($languages as $language) {
+	if ($language === $i18nData::REFERENCE_LANGUAGE) {
+		$i18nValidator = new I18nUsageValidator($i18nData->getReferenceLanguage(), findUsedTranslations());
+		$isValidated = $i18nValidator->validate(include __DIR__ . '/i18n/ignore/' . $language . '.php') && $isValidated;
+	} else {
+		$i18nValidator = new I18nCompletionValidator($i18nData->getReferenceLanguage(), $i18nData->getLanguage($language));
+		if (file_exists(__DIR__ . '/i18n/ignore/' . $language . '.php')) {
+			$isValidated = $i18nValidator->validate(include __DIR__ . '/i18n/ignore/' . $language . '.php') && $isValidated;
+		} else {
+			$isValidated = $i18nValidator->validate(null) && $isValidated;
+		}
+	}
+
+	$report[$language] = sprintf('%-5s - %s', $language, $i18nValidator->displayReport());
+	$result[$language] = $i18nValidator->displayResult();
+}
+
+if ($displayResults) {
+	foreach ($result as $lang => $value) {
+		echo 'Language: ', $lang, PHP_EOL;
+		print_r($value);
+		echo PHP_EOL;
+	}
+}
+
+if ($displayReport) {
+	foreach ($report as $value) {
+		echo $value;
+	}
+}
+
+if (!$isValidated) {
+	exit(1);
+}
+
+/**
+ * Find used translation keys in the project
+ *
+ * Iterates through all php and phtml files in the whole project and extracts all
+ * translation keys used.
+ *
+ * @return array
+ */
+function findUsedTranslations() {
+	$directory = new RecursiveDirectoryIterator(__DIR__ . '/..');
+	$iterator = new RecursiveIteratorIterator($directory);
+	$regex = new RegexIterator($iterator, '/^.+\.(php|phtml)$/i', RecursiveRegexIterator::GET_MATCH);
+	$usedI18n = array();
+	foreach (array_keys(iterator_to_array($regex)) as $file) {
+		$fileContent = file_get_contents($file);
+		preg_match_all('/_t\([\'"](?P<strings>[^\'"]+)[\'"]/', $fileContent, $matches);
+		$usedI18n = array_merge($usedI18n, $matches['strings']);
+	}
+	return $usedI18n;
+}
+
+/**
+ * Output help message.
+ */
+function help() {
+	$help = <<<HELP
+NAME
+	%s
+
+SYNOPSIS
+	php %s [OPTION]...
+
+DESCRIPTION
+	Check if translation files have missing keys or missing translations.
+
+	-d	display results.
+	-h	display this help and exit.
+	-l=LANG	filter by LANG.
+	-r	display completion report.
+
+HELP;
+	$file = str_replace(__DIR__ . '/', '', __FILE__);
+	echo sprintf($help, $file, $file);
+	exit;
+}
diff --git a/cli/i18n/I18nCompletionValidator.php b/cli/i18n/I18nCompletionValidator.php
new file mode 100644
index 000000000..2cb71acd5
--- /dev/null
+++ b/cli/i18n/I18nCompletionValidator.php
@@ -0,0 +1,49 @@
+<?php
+
+require_once __DIR__ . '/I18nValidatorInterface.php';
+
+class I18nCompletionValidator implements I18nValidatorInterface {
+
+	private $reference;
+	private $language;
+	private $totalEntries = 0;
+	private $passEntries = 0;
+	private $result = '';
+
+	public function __construct($reference, $language) {
+		$this->reference = $reference;
+		$this->language = $language;
+	}
+
+	public function displayReport() {
+		return sprintf('Translation is %5.1f%% complete.', $this->passEntries / $this->totalEntries * 100) . PHP_EOL;
+	}
+
+	public function displayResult() {
+		return $this->result;
+	}
+
+	public function validate($ignore) {
+		foreach ($this->reference as $file => $data) {
+			foreach ($data as $key => $value) {
+				$this->totalEntries++;
+				if (is_array($ignore) && in_array($key, $ignore)) {
+					$this->passEntries++;
+					continue;
+				}
+				if (!array_key_exists($key, $this->language[$file])) {
+					$this->result .= sprintf('Missing key %s', $key) . PHP_EOL;
+					continue;
+				}
+				if ($value === $this->language[$file][$key]) {
+					$this->result .= sprintf('Untranslated key %s - %s', $key, $value) . PHP_EOL;
+					continue;
+				}
+				$this->passEntries++;
+			}
+		}
+
+		return $this->totalEntries === $this->passEntries;
+	}
+
+}
diff --git a/cli/i18n/I18nData.php b/cli/i18n/I18nData.php
new file mode 100644
index 000000000..cd8ba0765
--- /dev/null
+++ b/cli/i18n/I18nData.php
@@ -0,0 +1,118 @@
+<?php
+
+class I18nData {
+
+	const REFERENCE_LANGUAGE = 'en';
+
+	private $data = array();
+	private $originalData = array();
+
+	public function __construct($data) {
+		$this->data = $data;
+		$this->originalData = $data;
+	}
+
+	public function getData() {
+		return $this->data;
+	}
+
+	/**
+	 * Return the available languages
+	 *
+	 * @return array
+	 */
+	public function getAvailableLanguages() {
+		$languages = array_keys($this->data);
+		sort($languages);
+
+		return $languages;
+	}
+
+	/**
+	 * Add a new language. It's a copy of the reference language.
+	 *
+	 * @param string $language
+	 */
+	public function addLanguage($language) {
+		if (array_key_exists($language, $this->data)) {
+			throw new Exception('The selected language already exist.');
+		}
+		$this->data[$language] = $this->data[static::REFERENCE_LANGUAGE];
+	}
+
+	/**
+	 * Add a key in the reference language
+	 *
+	 * @param string $key
+	 * @param string $value
+	 */
+	public function addKey($key, $value) {
+		if (array_key_exists($key, $this->data[static::REFERENCE_LANGUAGE][$this->getFilenamePrefix($key)])) {
+			throw new Exception('The selected key already exist.');
+		}
+		$this->data[static::REFERENCE_LANGUAGE][$this->getFilenamePrefix($key)][$key] = $value;
+	}
+
+	/**
+	 * Duplicate a key from the reference language to all other languages
+	 *
+	 * @param string $key
+	 */
+	public function duplicateKey($key) {
+		if (!array_key_exists($key, $this->data[static::REFERENCE_LANGUAGE][$this->getFilenamePrefix($key)])) {
+			throw new Exception('The selected key does not exist.');
+		}
+		$value = $this->data[static::REFERENCE_LANGUAGE][$this->getFilenamePrefix($key)][$key];
+		foreach ($this->getAvailableLanguages() as $language) {
+			if (static::REFERENCE_LANGUAGE === $language) {
+				continue;
+			}
+			if (array_key_exists($key, $this->data[$language][$this->getFilenamePrefix($key)])) {
+				throw new Exception(sprintf('The selected key already exist in %s.', $language));
+			}
+			$this->data[$language][$this->getFilenamePrefix($key)][$key] = $value;
+		}
+	}
+
+	/**
+	 * Remove a key in all languages
+	 *
+	 * @param string $key
+	 */
+	public function removeKey($key) {
+		if (!array_key_exists($key, $this->data[static::REFERENCE_LANGUAGE][$this->getFilenamePrefix($key)])) {
+			throw new Exception('The selected key does not exist.');
+		}
+		foreach ($this->getAvailableLanguages() as $language) {
+			if (array_key_exists($key, $this->data[$language][$this->getFilenamePrefix($key)])) {
+				unset($this->data[$language][$this->getFilenamePrefix($key)][$key]);
+			}
+		}
+	}
+
+	/**
+	 * Check if the data has changed
+	 *
+	 * @return bool
+	 */
+	public function hasChanged() {
+		return $this->data !== $this->originalData;
+	}
+
+	public function getLanguage($language) {
+		return $this->data[$language];
+	}
+
+	public function getReferenceLanguage() {
+		return $this->getLanguage(static::REFERENCE_LANGUAGE);
+	}
+
+	/**
+	 * @param string $key
+	 * @return string
+	 */
+	private function getFilenamePrefix($key) {
+		return preg_replace('/\..*/', '.php', $key);
+	}
+
+}
diff --git a/cli/i18n/I18nFile.php b/cli/i18n/I18nFile.php
new file mode 100644
index 000000000..d6489ee21
--- /dev/null
+++ b/cli/i18n/I18nFile.php
@@ -0,0 +1,92 @@
+<?php
+
+require_once __DIR__ . '/I18nData.php';
+
+class i18nFile {
+
+	private $i18nPath;
+
+	public function __construct() {
+		$this->i18nPath = __DIR__ . '/../../app/i18n';
+	}
+
+	public function load() {
+		$dirs = new DirectoryIterator($this->i18nPath);
+		foreach ($dirs as $dir) {
+			if ($dir->isDot()) {
+				continue;
+			}
+			$files = new DirectoryIterator($dir->getPathname());
+			foreach ($files as $file) {
+				if (!$file->isFile()) {
+					continue;
+				}
+				$i18n[$dir->getFilename()][$file->getFilename()] = $this->flatten(include $file->getPathname(), $file->getBasename('.php'));
+			}
+		}
+
+		return new I18nData($i18n);
+	}
+
+	public function dump(I18nData $i18n) {
+		foreach ($i18n->getData() as $language => $file) {
+			$dir = $this->i18nPath . DIRECTORY_SEPARATOR . $language;
+			if (!file_exists($dir)) {
+				mkdir($dir);
+			}
+			foreach ($file as $name => $content) {
+				$filename = $dir . DIRECTORY_SEPARATOR . $name;
+				$fullContent = var_export($this->unflatten($content), true);
+				file_put_contents($filename, sprintf('<?php return %s;', $fullContent));
+			}
+		}
+	}
+
+	/**
+	 * Flatten an array of translation
+	 *
+	 * @param array $translation
+	 * @param string $prefix
+	 * @return array
+	 */
+	private function flatten($translation, $prefix = '') {
+		$a = array();
+
+		if ('' !== $prefix) {
+			$prefix .= '.';
+		}
+
+		foreach ($translation as $key => $value) {
+			if (is_array($value)) {
+				$a += $this->flatten($value, $prefix . $key);
+			} else {
+				$a[$prefix . $key] = $value;
+			}
+		}
+
+		return $a;
+	}
+
+	/**
+	 * Unflatten an array of translation
+	 *
+	 * The first key is dropped since it represents the filename and we have
+	 * no use of it.
+	 *
+	 * @param array $translation
+	 * @return array
+	 */
+	private function unflatten($translation) {
+		$a = array();
+
+		ksort($translation);
+		foreach ($translation as $compoundKey => $value) {
+			$keys = explode('.', $compoundKey);
+			array_shift($keys);
+			eval("\$a['" . implode("']['", $keys) . "'] = '" . $value . "';");
+		}
+
+		return $a;
+	}
+
+}
diff --git a/cli/i18n/I18nUsageValidator.php b/cli/i18n/I18nUsageValidator.php
new file mode 100644
index 000000000..8ab934971
--- /dev/null
+++ b/cli/i18n/I18nUsageValidator.php
@@ -0,0 +1,47 @@
+<?php
+
+require_once __DIR__ . '/I18nValidatorInterface.php';
+
+class I18nUsageValidator implements I18nValidatorInterface {
+
+	private $code;
+	private $reference;
+	private $totalEntries = 0;
+	private $failedEntries = 0;
+	private $result = '';
+
+	public function __construct($reference, $code) {
+		$this->code = $code;
+		$this->reference = $reference;
+	}
+
+	public function displayReport() {
+		return sprintf('%5.1f%% of translation keys are unused.', $this->failedEntries / $this->totalEntries * 100) . PHP_EOL;
+	}
+
+	public function displayResult() {
+		return $this->result;
+	}
+
+	public function validate($ignore) {
+		foreach ($this->reference as $file => $data) {
+			foreach ($data as $key => $value) {
+				$this->totalEntries++;
+				if (preg_match('/\._$/', $key) && in_array(preg_replace('/\._$/', '', $key), $this->code)) {
+					continue;
+				}
+				if (is_array($ignore) && in_array($key, $ignore)) {
+					continue;
+				}
+				if (!in_array($key, $this->code)) {
+					$this->result .= sprintf('Unused key %s - %s', $key, $value) . PHP_EOL;
+					$this->failedEntries++;
+					continue;
+				}
+			}
+		}
+
+		return 0 === $this->failedEntries;
+	}
+
+}
diff --git a/cli/i18n/I18nValidatorInterface.php b/cli/i18n/I18nValidatorInterface.php
new file mode 100644
index 000000000..edfe7aac0
--- /dev/null
+++ b/cli/i18n/I18nValidatorInterface.php
@@ -0,0 +1,26 @@
+<?php
+
+interface I18nValidatorInterface {
+
+	/**
+	 * Display the validation result.
+	 * Empty if there are no errors.
+	 *
+	 * @return array
+	 */
+	public function displayResult();
+
+	/**
+	 * @param array $ignore Keys to ignore for validation
+	 * @return bool
+	 */
+	public function validate($ignore);
+
+	/**
+	 * Display the validation report.
+	 *
+	 * @return array
+	 */
+	public function displayReport();
+
+}
diff --git a/cli/i18n/ignore/en.php b/cli/i18n/ignore/en.php
new file mode 100644
index 000000000..e231afdda
--- /dev/null
+++ b/cli/i18n/ignore/en.php
@@ -0,0 +1,105 @@
+<?php
+
+return array(
+	'admin.check_install.cache.nok',
+	'admin.check_install.cache.ok',
+	'admin.check_install.categories.nok',
+	'admin.check_install.categories.ok',
+	'admin.check_install.connection.nok',
+	'admin.check_install.connection.ok',
+	'admin.check_install.ctype.nok',
+	'admin.check_install.ctype.ok',
+	'admin.check_install.curl.nok',
+	'admin.check_install.curl.ok',
+	'admin.check_install.data.nok',
+	'admin.check_install.data.ok',
+	'admin.check_install.dom.nok',
+	'admin.check_install.dom.ok',
+	'admin.check_install.entries.nok',
+	'admin.check_install.entries.ok',
+	'admin.check_install.favicons.nok',
+	'admin.check_install.favicons.ok',
+	'admin.check_install.feeds.nok',
+	'admin.check_install.feeds.ok',
+	'admin.check_install.fileinfo.nok',
+	'admin.check_install.fileinfo.ok',
+	'admin.check_install.json.nok',
+	'admin.check_install.json.ok',
+	'admin.check_install.minz.nok',
+	'admin.check_install.minz.ok',
+	'admin.check_install.pcre.nok',
+	'admin.check_install.pcre.ok',
+	'admin.check_install.pdo.nok',
+	'admin.check_install.pdo.ok',
+	'admin.check_install.php.nok',
+	'admin.check_install.php.ok',
+	'admin.check_install.tables.nok',
+	'admin.check_install.tables.ok',
+	'admin.check_install.tokens.nok',
+	'admin.check_install.tokens.ok',
+	'admin.check_install.users.nok',
+	'admin.check_install.users.ok',
+	'admin.check_install.zip.nok',
+	'admin.check_install.zip.ok',
+	'conf.query.get_all',
+	'conf.query.get_category',
+	'conf.query.get_favorite',
+	'conf.query.get_feed',
+	'conf.query.order_asc',
+	'conf.query.order_desc',
+	'conf.query.state_0',
+	'conf.query.state_1',
+	'conf.query.state_2',
+	'conf.query.state_3',
+	'conf.query.state_4',
+	'conf.query.state_5',
+	'conf.query.state_6',
+	'conf.query.state_7',
+	'conf.query.state_8',
+	'conf.query.state_9',
+	'conf.query.state_10',
+	'conf.query.state_11',
+	'conf.query.state_12',
+	'conf.query.state_13',
+	'conf.query.state_14',
+	'conf.query.state_15',
+	'conf.sharing.blogotext',
+	'conf.sharing.diaspora',
+	'conf.sharing.email',
+	'conf.sharing.facebook',
+	'conf.sharing.g+',
+	'conf.sharing.print',
+	'conf.sharing.shaarli',
+	'conf.sharing.twitter',
+	'conf.sharing.wallabag',
+	'gen.lang.cz',
+	'gen.lang.de',
+	'gen.lang.en',
+	'gen.lang.es',
+	'gen.lang.fr',
+	'gen.lang.it',
+	'gen.lang.kr',
+	'gen.lang.nl',
+	'gen.lang.pt-br',
+	'gen.lang.ru',
+	'gen.lang.tr',
+	'gen.lang.zh-cn',
+	'gen.share.blogotext',
+	'gen.share.diaspora',
+	'gen.share.email',
+	'gen.share.facebook',
+	'gen.share.g+',
+	'gen.share.movim',
+	'gen.share.print',
+	'gen.share.shaarli',
+	'gen.share.twitter',
+	'gen.share.wallabag',
+	'gen.share.wallabagv2',
+	'gen.share.jdh',
+	'gen.share.Known',
+	'gen.share.gnusocial',
+	'index.menu.non-starred',
+	'index.menu.read',
+	'index.menu.starred',
+	'index.menu.unread',
+);
diff --git a/cli/i18n/ignore/fr.php b/cli/i18n/ignore/fr.php
new file mode 100644
index 000000000..0ac2e8758
--- /dev/null
+++ b/cli/i18n/ignore/fr.php
@@ -0,0 +1,55 @@
+<?php
+
+return array(
+	'admin.extensions.title',
+	'admin.stats.number_entries',
+	'admin.user.articles_and_size',
+	'conf.display.width.large',
+	'conf.sharing.blogotext',
+	'conf.sharing.diaspora',
+	'conf.sharing.facebook',
+	'conf.sharing.g+',
+	'conf.sharing.print',
+	'conf.sharing.shaarli',
+	'conf.sharing.twitter',
+	'conf.sharing.wallabag',
+	'conf.shortcut.navigation',
+	'conf.user.articles_and_size',
+	'gen.freshrss._',
+	'gen.lang.cz',
+	'gen.lang.de',
+	'gen.lang.en',
+	'gen.lang.es',
+	'gen.lang.fr',
+	'gen.lang.it',
+	'gen.lang.kr',
+	'gen.lang.nl',
+	'gen.lang.pt-br',
+	'gen.lang.ru',
+	'gen.lang.tr',
+	'gen.lang.zh-cn',
+	'gen.menu.admin',
+	'gen.menu.configuration',
+	'gen.menu.extensions',
+	'gen.menu.logs',
+	'gen.share.blogotext',
+	'gen.share.diaspora',
+	'gen.share.facebook',
+	'gen.share.g+',
+	'gen.share.movim',
+	'gen.share.shaarli',
+	'gen.share.twitter',
+	'gen.share.wallabag',
+	'gen.share.wallabagv2',
+	'gen.share.jdh',
+	'gen.share.gnusocial',
+	'index.about.agpl3',
+	'index.about.version',
+	'index.log._',
+	'index.log.title',
+	'install.title',
+	'install.this_is_the_end',
+	'sub.bookmarklet.title',
+	'sub.feed.description',
+	'sub.feed.number_entries',
+);
diff --git a/cli/manipulate.translation.php b/cli/manipulate.translation.php
new file mode 100644
index 000000000..aace5723a
--- /dev/null
+++ b/cli/manipulate.translation.php
@@ -0,0 +1,79 @@
+<?php
+
+$options = getopt("h");
+
+if (array_key_exists('h', $options)) {
+	help();
+}
+
+if (1 === $argc || 4 < $argc) {
+	help();
+}
+
+require_once __DIR__ . '/i18n/I18nFile.php';
+
+$i18nFile = new I18nFile();
+$i18nData = $i18nFile->load();
+
+switch ($argv[1]) {
+	case 'add_language' :
+		$i18nData->addLanguage($argv[2]);
+		break;
+	case 'add_key' :
+		if (3 === $argc) {
+			help();
+		}
+		$i18nData->addKey($argv[2], $argv[3]);
+		break;
+	case 'duplicate_key' :
+		$i18nData->duplicateKey($argv[2]);
+		break;
+	case 'delete_key' :
+		$i18nData->removeKey($argv[2]);
+		break;
+	default :
+		help();
+}
+
+if ($i18nData->hasChanged()) {
+	$i18nFile->dump($i18nData);
+}
+
+/**
+ * Output help message.
+ */
+function help() {
+	$help = <<<HELP
+NAME
+	%s
+
+SYNOPSIS
+	php %s [OPTION] [OPERATION] [KEY] [VALUE]
+
+DESCRIPTION
+	Manipulate translation files. Available operations are 
+	Check if translation files have missing keys or missing translations.
+
+	-h	display this help and exit.
+
+OPERATION
+	add_language
+		add a new language by duplicating the referential. This operation
+		needs only a KEY.
+
+	add_key	add a new key in the referential. This operation needs a KEY and
+		a VALUE.
+
+	duplicate_key
+		duplicate a referential key in other languages. This operation
+		needs only a KEY.
+
+	delete_key
+		delete a referential key from all languages. This operation needs
+		only a KEY.
+
+HELP;
+	$file = str_replace(__DIR__ . '/', '', __FILE__);
+	echo sprintf($help, $file, $file);
+	exit;
+}