From 127b7f0a3aad7012055c058e8aba0d27192a8cbc Mon Sep 17 00:00:00 2001 From: Alexis Degrugillier Date: Sat, 8 Jan 2022 08:00:26 -0500 Subject: Change i18n process (#4131) Before, the ignore info were stored in a different file which was a bit cumbersome for new comers. Now, this info is stored directly in the translation file as a comment. Before, there was no way of telling translators that a previously translated string was in need of a new translation. Now, the dirty information is there to convey that info. --- cli/check.translation.php | 22 +- cli/i18n/I18nCompletionValidator.php | 21 +- cli/i18n/I18nData.php | 113 ++--- cli/i18n/I18nFile.php | 70 ++- cli/i18n/I18nFileInterface.php | 10 - cli/i18n/I18nIgnoreFile.php | 63 --- cli/i18n/I18nUsageValidator.php | 5 +- cli/i18n/I18nValidatorInterface.php | 3 +- cli/i18n/I18nValue.php | 75 ++++ cli/i18n/ignore/cz.php | 50 --- cli/i18n/ignore/de.php | 73 ---- cli/i18n/ignore/en-us.php | 804 ----------------------------------- cli/i18n/ignore/en.php | 110 ----- cli/i18n/ignore/es.php | 57 --- cli/i18n/ignore/fr.php | 77 ---- cli/i18n/ignore/he.php | 49 --- cli/i18n/ignore/it.php | 48 --- cli/i18n/ignore/ja.php | 50 --- cli/i18n/ignore/ko.php | 62 --- cli/i18n/ignore/nl.php | 69 --- cli/i18n/ignore/oc.php | 72 ---- cli/i18n/ignore/pl.php | 53 --- cli/i18n/ignore/pt-br.php | 60 --- cli/i18n/ignore/ru.php | 52 --- cli/i18n/ignore/sk.php | 73 ---- cli/i18n/ignore/tr.php | 56 --- cli/i18n/ignore/zh-cn.php | 51 --- cli/manipulate.translation.php | 39 +- 28 files changed, 212 insertions(+), 2075 deletions(-) delete mode 100644 cli/i18n/I18nFileInterface.php delete mode 100644 cli/i18n/I18nIgnoreFile.php create mode 100644 cli/i18n/I18nValue.php delete mode 100644 cli/i18n/ignore/cz.php delete mode 100644 cli/i18n/ignore/de.php delete mode 100644 cli/i18n/ignore/en-us.php delete mode 100644 cli/i18n/ignore/en.php delete mode 100644 cli/i18n/ignore/es.php delete mode 100644 cli/i18n/ignore/fr.php delete mode 100644 cli/i18n/ignore/he.php delete mode 100644 cli/i18n/ignore/it.php delete mode 100644 cli/i18n/ignore/ja.php delete mode 100644 cli/i18n/ignore/ko.php delete mode 100644 cli/i18n/ignore/nl.php delete mode 100644 cli/i18n/ignore/oc.php delete mode 100644 cli/i18n/ignore/pl.php delete mode 100644 cli/i18n/ignore/pt-br.php delete mode 100644 cli/i18n/ignore/ru.php delete mode 100644 cli/i18n/ignore/sk.php delete mode 100644 cli/i18n/ignore/tr.php delete mode 100644 cli/i18n/ignore/zh-cn.php (limited to 'cli') diff --git a/cli/check.translation.php b/cli/check.translation.php index 12655fc8e..38bb83af5 100644 --- a/cli/check.translation.php +++ b/cli/check.translation.php @@ -3,12 +3,10 @@ require_once __DIR__ . '/i18n/I18nCompletionValidator.php'; require_once __DIR__ . '/i18n/I18nData.php'; require_once __DIR__ . '/i18n/I18nFile.php'; -require_once __DIR__ . '/i18n/I18nIgnoreFile.php'; require_once __DIR__ . '/i18n/I18nUsageValidator.php'; $i18nFile = new I18nFile(); -$i18nIgnoreFile = new I18nIgnoreFile(); -$i18nData = new I18nData($i18nFile->load(), $i18nIgnoreFile->load()); +$i18nData = new I18nData($i18nFile->load()); $options = getopt("dhl:r"); @@ -30,14 +28,10 @@ $report = array(); foreach ($languages as $language) { if ($language === $i18nData::REFERENCE_LANGUAGE) { $i18nValidator = new I18nUsageValidator($i18nData->getReferenceLanguage(), findUsedTranslations()); - $isValidated = $i18nValidator->validate(include __DIR__ . '/i18n/ignore/' . $language . '.php') && $isValidated; + $isValidated = $i18nValidator->validate() && $isValidated; } else { $i18nValidator = new I18nCompletionValidator($i18nData->getReferenceLanguage(), $i18nData->getLanguage($language)); - if (file_exists(__DIR__ . '/i18n/ignore/' . $language . '.php')) { - $isValidated = $i18nValidator->validate(include __DIR__ . '/i18n/ignore/' . $language . '.php') && $isValidated; - } else { - $isValidated = $i18nValidator->validate(null) && $isValidated; - } + $isValidated = $i18nValidator->validate() && $isValidated; } $report[$language] = sprintf('%-5s - %s', $language, $i18nValidator->displayReport()); @@ -87,12 +81,14 @@ function findUsedTranslations() { * Output help message. */ function help() { - $help = <<result; } - /** - * @param array|null $ignore - */ - public function validate($ignore) { + public function validate() { foreach ($this->reference as $file => $data) { - foreach ($data as $key => $value) { + foreach ($data as $refKey => $refValue) { $this->totalEntries++; - if (is_array($ignore) && in_array($key, $ignore)) { - $this->passEntries++; + if (!array_key_exists($refKey, $this->language[$file])) { + $this->result .= "Missing key $refKey" . PHP_EOL; continue; } - if (!array_key_exists($key, $this->language[$file])) { - $this->result .= sprintf('Missing key %s', $key) . PHP_EOL; + + $value = $this->language[$file][$refKey]; + if ($value->isIgnore()) { + $this->passEntries++; continue; } - if ($value === $this->language[$file][$key]) { - $this->result .= sprintf('Untranslated key %s - %s', $key, $value) . PHP_EOL; + if ($refValue->equal($value)) { + $this->result .= "Untranslated key $refKey - $refValue" . PHP_EOL; continue; } $this->passEntries++; diff --git a/cli/i18n/I18nData.php b/cli/i18n/I18nData.php index 2e5e373f8..6656d45cd 100644 --- a/cli/i18n/I18nData.php +++ b/cli/i18n/I18nData.php @@ -4,68 +4,33 @@ class I18nData { const REFERENCE_LANGUAGE = 'en'; - private $data = array(); - private $ignore = array(); + private $data = []; - public function __construct($data, $ignore) { + public function __construct(array $data) { $this->data = $data; - $this->ignore = $ignore; - $this->synchonizeKeys(); + $this->addMissingKeysFromReference(); + $this->removeExtraKeysFromOtherLanguages(); + $this->processValueStates(); } public function getData() { - $output = array(); - $reference = $this->getReferenceLanguage(); - $languages = $this->getNonReferenceLanguages(); - - foreach ($reference as $file => $values) { - foreach ($values as $key => $value) { - $output[static::REFERENCE_LANGUAGE][$file][$key] = $value; - foreach ($languages as $language) { - if ($this->data[$language][$file][$key] !== $value) { - // This value is translated, there is no need to flag it. - $output[$language][$file][$key] = $this->data[$language][$file][$key]; - } elseif (array_key_exists($language, $this->ignore) && in_array($key, $this->ignore[$language])) { - // This value is ignored, there is no need to flag it. - $output[$language][$file][$key] = $this->data[$language][$file][$key]; - } else { - // This value is not translated nor ignored, it must be flagged. - $output[$language][$file][$key] = "{$value} -> todo"; - } - } - } - } - - return $output; - } - - public function getIgnore() { - $ignore = array(); - - foreach ($this->ignore as $language => $keys) { - sort($keys); - $ignore[$language] = $keys; - } - - return $ignore; - } - - private function synchonizeKeys() { - $this->addMissingKeysFromReference(); - $this->removeExtraKeysFromOtherLanguages(); - $this->removeUnknownIgnoreKeys(); + return $this->data; } private function addMissingKeysFromReference() { $reference = $this->getReferenceLanguage(); $languages = $this->getNonReferenceLanguages(); - foreach ($reference as $file => $values) { - foreach ($values as $key => $value) { + foreach ($reference as $file => $refValues) { + foreach ($refValues as $key => $refValue) { foreach ($languages as $language) { if (!array_key_exists($key, $this->data[$language][$file])) { - $this->data[$language][$file][$key] = $value; + $this->data[$language][$file][$key] = clone $refValue; + } + $value = $this->data[$language][$file][$key]; + if ($refValue->equal($value) && !$value->isIgnore()) { + $value->markAsTodo(); } } } @@ -85,12 +50,22 @@ class I18nData { } } - private function removeUnknownIgnoreKeys() { + private function processValueStates() { $reference = $this->getReferenceLanguage(); - foreach ($this->ignore as $language => $keys) { - foreach ($keys as $index => $key) { - if (!array_key_exists($this->getFilenamePrefix($key), $reference) || !array_key_exists($key, $reference[$this->getFilenamePrefix($key)])) { - unset($this->ignore[$language][$index]); + $languages = $this->getNonReferenceLanguages(); + + foreach ($reference as $file => $refValues) { + foreach ($refValues as $key => $refValue) { + foreach ($languages as $language) { + $value = $this->data[$language][$file][$key]; + if ($refValue->equal($value) && !$value->isIgnore()) { + $value->markAsTodo(); + continue; + } + if (!$refValue->equal($value) && $value->isTodo()) { + $value->markAsDirty(); + continue; + } } } } @@ -220,10 +195,13 @@ class I18nData { // To create an array, we need to change the key by appending an empty section. foreach ($this->getAvailableLanguages() as $language) { $parentValue = $this->data[$language][$this->getFilenamePrefix($parentKey)][$parentKey]; - $this->data[$language][$this->getFilenamePrefix($this->getEmptySibling($parentKey))][$this->getEmptySibling($parentKey)] = $parentValue; + $this->data[$language][$this->getFilenamePrefix($this->getEmptySibling($parentKey))][$this->getEmptySibling($parentKey)] = + new I18nValue($parentValue); } } + $value = new I18nValue($value); + $value->markAsTodo(); foreach ($this->getAvailableLanguages() as $language) { if (!array_key_exists($key, $this->data[$language][$this->getFilenamePrefix($key)])) { $this->data[$language][$this->getFilenamePrefix($key)][$key] = $value; @@ -251,6 +229,8 @@ class I18nData { !array_key_exists($key, $this->data[static::REFERENCE_LANGUAGE][$this->getFilenamePrefix($key)])) { throw new Exception('The selected key does not exist for the selected language.'); } + + $value = new I18nValue($value); if (static::REFERENCE_LANGUAGE === $language) { $previousValue = $this->data[static::REFERENCE_LANGUAGE][$this->getFilenamePrefix($key)][$key]; foreach ($this->getAvailableLanguages() as $lang) { @@ -282,9 +262,6 @@ class I18nData { if (array_key_exists($key, $this->data[$language][$this->getFilenamePrefix($key)])) { unset($this->data[$language][$this->getFilenamePrefix($key)][$key]); } - if (array_key_exists($language, $this->ignore) && $position = array_search($key, $this->ignore[$language])) { - unset($this->ignore[$language][$position]); - } } if ($this->isOnlyChild($key)) { @@ -305,24 +282,16 @@ class I18nData { * @param boolean $reverse */ public function ignore($key, $language, $reverse = false) { - if (!array_key_exists($language, $this->ignore)) { - $this->ignore[$language] = array(); - } - - $index = array_search($key, $this->ignore[$language]); - if (false !== $index && $reverse) { - unset($this->ignore[$language][$index]); - return; - } - if (false !== $index && !$reverse) { - return; + $value = $this->data[$language][$this->getFilenamePrefix($key)][$key]; + if ($reverse) { + $value->markAsIgnore(); + } else { + $value->unmarkAsIgnore(); } - - $this->ignore[$language][] = $key; } /** - *Ignore all unmodified keys from a language, or reverse it. + * Ignore all unmodified keys from a language, or reverse it. * * @param string $language * @param boolean $reverse @@ -332,7 +301,7 @@ class I18nData { foreach ($this->getReferenceLanguage() as $file => $ref_language) { foreach ($ref_language as $key => $ref_value) { if (array_key_exists($key, $my_language[$file])) { - if($ref_value == $my_language[$file][$key]) { + if($ref_value->equal($my_language[$file][$key])) { $this->ignore($key, $language, $reverse); } } diff --git a/cli/i18n/I18nFile.php b/cli/i18n/I18nFile.php index 222d07692..50b2d5023 100644 --- a/cli/i18n/I18nFile.php +++ b/cli/i18n/I18nFile.php @@ -1,8 +1,8 @@ isFile()) { continue; } - $i18n[$dir->getFilename()][$file->getFilename()] = $this->flatten(include $file->getPathname(), $file->getBasename('.php')); + + $i18n[$dir->getFilename()][$file->getFilename()] = $this->flatten($this->process($file->getPathname()), $file->getBasename('.php')); } } @@ -42,6 +43,35 @@ class I18nFile implements I18nFileInterface{ } } + /** + * Process the content of an i18n file + * + * @param string $filename + * @return array + */ + private function process(string $filename) { + $content = file_get_contents($filename); + $content = str_replace(' todo\',', + ' -> dirty\',', + ' -> ignore\',', + ], $content); + + $content = eval($content); + + if (is_array($content)) { + return $content; + } + + return []; + } + /** * Flatten an array of translation * @@ -49,7 +79,7 @@ class I18nFile implements I18nFileInterface{ * @param string $prefix * @return array */ - private function flatten($translation, $prefix = '') { + private function flatten(array $translation, string $prefix = '') { $a = array(); if ('' !== $prefix) { @@ -60,7 +90,7 @@ class I18nFile implements I18nFileInterface{ if (is_array($value)) { $a += $this->flatten($value, $prefix . $key); } else { - $a[$prefix . $key] = $value; + $a[$prefix . $key] = new I18nValue($value); } } @@ -76,7 +106,7 @@ class I18nFile implements I18nFileInterface{ * @param array $translation * @return array */ - private function unflatten($translation) { + private function unflatten(array $translation) { $a = array(); ksort($translation, SORT_NATURAL); @@ -99,25 +129,43 @@ class I18nFile implements I18nFileInterface{ * @param array $translation * @return string */ - private function format($translation) { + private function format(array $translation) { $translation = var_export($this->unflatten($translation), true); $patterns = array( + '/ -> todo\',/', + '/ -> dirty\',/', + '/ -> ignore\',/', '/array \(/', '/=>\s*array/', '/(\w) {2}/', '/ {2}/', - '/ -> todo\',/', ); $replacements = array( + "',\t// TODO", // Double quoting is mandatory to have a tab instead of the \t string + "',\t// DIRTY", // Double quoting is mandatory to have a tab instead of the \t string + "',\t// IGNORE", // Double quoting is mandatory to have a tab instead of the \t string 'array(', '=> array', '$1 ', "\t", // Double quoting is mandatory to have a tab instead of the \t string - "',\t// TODO - Translation", // Double quoting is mandatory to have a tab instead of the \t string ); $translation = preg_replace($patterns, $replacements, $translation); - // Double quoting is mandatory to have new lines instead of \n strings - return sprintf("i18nPath = __DIR__ . '/ignore'; - } - - public function dump(array $i18n) { - foreach ($i18n as $language => $content) { - $filename = $this->i18nPath . DIRECTORY_SEPARATOR . $language . '.php'; - file_put_contents($filename, $this->format($content)); - } - } - - public function load() { - $i18n = array(); - $files = new DirectoryIterator($this->i18nPath); - foreach ($files as $file) { - if (!$file->isFile()) { - continue; - } - $i18n[$file->getBasename('.php')] = (include $file->getPathname()); - } - - return $i18n; - } - - /** - * Format an array of translation - * - * It takes an array of translation and format it to be dumped in a - * translation file. The array is first converted to a string then some - * formatting regexes are applied to match the original content. - * - * @param array $translation - * @return string - */ - private function format($translation) { - $translation = var_export(($translation), true); - $patterns = array( - '/array \(/', - '/=>\s*array/', - '/ {2}/', - '/\d+ => /', - ); - $replacements = array( - 'array(', - '=> array', - "\t", // Double quoting is mandatory to have a tab instead of the \t string - '', - ); - $translation = preg_replace($patterns, $replacements, $translation); - - // Double quoting is mandatory to have new lines instead of \n strings - return sprintf("result; } - public function validate($ignore) { + public function validate() { foreach ($this->reference as $file => $data) { foreach ($data as $key => $value) { $this->totalEntries++; if (preg_match('/\._$/', $key) && in_array(preg_replace('/\._$/', '', $key), $this->code)) { continue; } - if (is_array($ignore) && in_array($key, $ignore)) { - continue; - } if (!in_array($key, $this->code)) { $this->result .= sprintf('Unused key %s - %s', $key, $value) . PHP_EOL; $this->failedEntries++; diff --git a/cli/i18n/I18nValidatorInterface.php b/cli/i18n/I18nValidatorInterface.php index 80fcb22ad..d5681912b 100644 --- a/cli/i18n/I18nValidatorInterface.php +++ b/cli/i18n/I18nValidatorInterface.php @@ -11,10 +11,9 @@ interface I18nValidatorInterface { public function displayResult(); /** - * @param array $ignore Keys to ignore for validation * @return bool */ - public function validate($ignore); + public function validate(); /** * Display the validation report. diff --git a/cli/i18n/I18nValue.php b/cli/i18n/I18nValue.php new file mode 100644 index 000000000..e691b8574 --- /dev/null +++ b/cli/i18n/I18nValue.php @@ -0,0 +1,75 @@ + ', $data); + + $this->value = array_shift($data); + if (count($data) === 0) { + return; + } + + $state = array_shift($data); + if (in_array($state, self::STATES)) { + $this->state = $state; + } + } + + public function __clone() { + $this->markAsTodo(); + } + + public function equal(I18nValue $value) { + return $this->value === $value->getValue(); + } + + public function isIgnore() { + return $this->state === self::STATE_IGNORE; + } + + public function isTodo() { + return $this->state === self::STATE_TODO; + } + + public function markAsDirty() { + $this->state = self::STATE_DIRTY; + } + + public function markAsIgnore() { + $this->state = self::STATE_IGNORE; + } + + public function markAsTodo() { + $this->state = self::STATE_TODO; + } + + public function unmarkAsIgnore() { + if ($this->state === self::STATE_IGNORE) { + $this->state = null; + } + } + + public function __toString() { + if ($this->state === null) { + return $this->value; + } + + return "{$this->value} -> {$this->state}"; + } + + public function getValue() { + return $this->value; + } +} diff --git a/cli/i18n/ignore/cz.php b/cli/i18n/ignore/cz.php deleted file mode 100644 index 8cb761736..000000000 --- a/cli/i18n/ignore/cz.php +++ /dev/null @@ -1,50 +0,0 @@ -load(), $ignore->load()); +$i18nData = new I18nData($data->load()); switch ($options['a']) { case 'add' : @@ -81,7 +79,6 @@ switch ($options['a']) { } $data->dump($i18nData->getData()); -$ignore->dump($i18nData->getIgnore()); /** * Output error message. @@ -99,19 +96,20 @@ ERROR; * Output help message. */ function help() { - $help = <<