ShippedPoMerger.php

Go to the documentation of this file.

<?php
 
declare(strict_types=1);
 
namespace ILIAS\Language\ComponentTranslation;
 
use ILIAS\Language\ComponentTranslation\Catalog\TranslationCatalog;
use ILIAS\Language\ComponentTranslation\Catalog\TranslationEntry;
use RuntimeException;
 
final class ShippedPoMerger
{
    public static function merge(
        LanguageFileDirectoryManager $language_file_directory_manager,
        string $ilias_absolute_path,
        string $lang_key,
        ?string $client_data_dir,
        string $backup_directory,
        ?\Closure $after_module = null
    ): array {
        $result = [
            'written' => [],
            'skipped' => [],
            'invalid_markup' => [],
            'not_merged' => [],
            'unwritten_database' => [],
            'unmarked_overlay' => [],
        ];
        if ($client_data_dir === null) {
            return $result;
        }
 
        $shipped_files = MigratedLanguageFileSync::findShippedModuleFiles($language_file_directory_manager, $ilias_absolute_path, $lang_key);
        $backup_names = self::backupFileNames($shipped_files);
        // the fixed lock order, see MigratedLanguageFileSync::withOverlayLock()
        ksort($shipped_files, SORT_STRING);
        foreach ($shipped_files as $module => $shipped_po) {
            $module = (string) $module;
            if (!MigratedLanguageFileSync::hasOverlay($language_file_directory_manager, $lang_key, $module, $client_data_dir)) {
                continue;
            }
            if ($backup_names[$module] === null) {
                $result['skipped'][$module] = sprintf('The shipped file name "%s" is not unique among the modules, it cannot be backed up.', basename($shipped_po));
                continue;
            }
            $outcome = MigratedLanguageFileSync::withOverlayLock(
                $language_file_directory_manager,
                $lang_key,
                $module,
                $client_data_dir,
                static function () use ($language_file_directory_manager, $ilias_absolute_path, $lang_key, $module, $shipped_po, $client_data_dir, $backup_directory, $backup_names, $after_module): array {
                    try {
                        // the template is shared by all languages: its own lock, after the overlay lock
                        $outcome = MigratedLanguageFileSync::withTemplateLock(
                            $language_file_directory_manager,
                            $module,
                            $client_data_dir,
                            static fn(): array => self::mergeModule(
                                $language_file_directory_manager,
                                $ilias_absolute_path,
                                $lang_key,
                                $module,
                                $shipped_po,
                                $client_data_dir,
                                rtrim($backup_directory, '/') . '/' . $backup_names[$module]
                            )
                        );
                    } catch (RuntimeException|\InvalidArgumentException $e) {
                        return ['skipped' => $e->getMessage()];
                    }
                    if ($outcome['written'] && $after_module !== null) {
                        // the files are written - a failing database write must not stop the others
                        try {
                            $after_module($module, $outcome['merged'], $outcome['merged_remarks']);
                        } catch (\Throwable $t) {
                            $outcome['database_error'] = $t->getMessage();
                        }
                    }
                    return $outcome;
                },
                $ilias_absolute_path
            );
 
            if (isset($outcome['skipped'])) {
                $result['skipped'][$module] = $outcome['skipped'];
                continue;
            }
            if ($outcome['invalid_markup'] !== []) {
                $result['invalid_markup'][$module] = $outcome['invalid_markup'];
            }
            if ($outcome['not_merged'] !== []) {
                $result['not_merged'][$module] = $outcome['not_merged'];
            }
            if ($outcome['written']) {
                $result['written'][$module] = $shipped_po;
            }
            if (isset($outcome['database_error'])) {
                $result['unwritten_database'][$module] = $outcome['database_error'];
            }
            if ($outcome['unmarked'] !== null) {
                $result['unmarked_overlay'][$module] = $outcome['unmarked'];
            }
        }
 
        return $result;
    }
 
    public static function backupShippedPoFiles(
        LanguageFileDirectoryManager $language_file_directory_manager,
        string $ilias_absolute_path,
        string $lang_key,
        string $directory,
        string $confine_to_directory
    ): array {
        $shipped_files = MigratedLanguageFileSync::findShippedModuleFiles($language_file_directory_manager, $ilias_absolute_path, $lang_key);
        $failed = [];
        foreach (self::backupFileNames($shipped_files) as $module => $name) {
            try {
                if ($name === null) {
                    throw new RuntimeException('file name not unique');
                }
                self::backup($shipped_files[$module], rtrim($directory, '/') . '/' . $name, $confine_to_directory);
            } catch (RuntimeException) {
                $failed[] = (string) $module;
            }
        }
 
        return $failed;
    }
 
    public static function findShippedChangesSinceBackup(
        LanguageFileDirectoryManager $language_file_directory_manager,
        string $ilias_absolute_path,
        string $lang_key,
        string $directory
    ): array {
        $shipped_files = MigratedLanguageFileSync::findShippedModuleFiles($language_file_directory_manager, $ilias_absolute_path, $lang_key);
        $result = ['changes' => [], 'without_backup' => []];
        foreach (self::backupFileNames($shipped_files) as $module => $name) {
            $module = (string) $module;
            $backup = $name === null ? null : rtrim($directory, '/') . '/' . $name;
            try {
                if ($backup === null || !is_file($backup) || !is_readable($backup)) {
                    throw new RuntimeException('no backup');
                }
                $former = self::valuesOf(MigratedLanguageFileSync::loadShippedModuleEntries($backup, $module));
                $current = self::valuesOf(MigratedLanguageFileSync::loadShippedModuleEntries($shipped_files[$module], $module));
            } catch (RuntimeException) {
                $result['without_backup'][] = $module;
                continue;
            }
            $changes = array_diff_assoc($current, $former);
            if ($changes !== []) {
                $result['changes'][$module] = $changes;
            }
        }
 
        return $result;
    }
 
    private static function mergeModule(
        LanguageFileDirectoryManager $language_file_directory_manager,
        string $ilias_absolute_path,
        string $lang_key,
        string $module,
        string $shipped_po,
        string $client_data_dir,
        string $backup_file
    ): array {
        $outcome = [
            'written' => false,
            'merged' => [],
            'merged_remarks' => [],
            'invalid_markup' => [],
            'not_merged' => [],
            'unmarked' => null,
        ];
        // read under the lock - it may have changed since hasOverlay()
        $overlay = MigratedLanguageFileSync::readOverlayCatalog($language_file_directory_manager, $lang_key, $module, $client_data_dir, $ilias_absolute_path);
        $directory = MigratedLanguageFileSync::findDirectory($language_file_directory_manager, $module);
        if ($overlay === null || $directory === null) {
            return $outcome;
        }
        $shipped = TranslationCatalog::fromPoFile($shipped_po);
        // an overlay written under another plural rule: its forms by meaning, see withPluralRuleOf()
        $overlay = MigratedLanguageFileSync::withPluralRuleOf($overlay, $shipped, $module);
        $overlay_entries = MigratedLanguageFileSync::moduleEntries($overlay, $module);
 
        // Everything that has to be writable, checked before anything is written
        self::assertWritableFile($shipped_po, $ilias_absolute_path);
        self::assertWritableFile($backup_file, $ilias_absolute_path);
        $template_file = MigratedLanguageFilePaths::shippedTemplatePath($ilias_absolute_path, $directory);
        $template = null;
        foreach ($overlay_entries as $identifier => $entry) {
            if (
                !MigratedLanguageFileSync::isRemarkOnly($entry)
                && PluralFormKey::parse((string) $identifier) === null
                && MigratedLanguageFileSync::findModuleEntry($shipped, $module, (string) $identifier) === null
            ) {
                if (!is_file($template_file)) {
                    throw new RuntimeException(sprintf('The template "%s" for the entries added locally is missing.', $template_file));
                }
                self::assertWritableFile($template_file, $ilias_absolute_path);
                $template = TranslationCatalog::fromPoFile($template_file);
                break;
            }
        }
 
        $plural_forms = MigratedLanguageFileSync::pluralFormsOf($shipped);
        $policy = new TranslationMarkupPolicy();
        $template_changed = false;
        foreach ($overlay_entries as $identifier => $overlay_entry) {
            $identifier = (string) $identifier;
            $shipped_entry = MigratedLanguageFileSync::findModuleEntry($shipped, $module, $identifier);
            $remark = LocalChangeComments::getRemark($overlay_entry);
            $has_value = !MigratedLanguageFileSync::isRemarkOnly($overlay_entry);
 
            if ($has_value) {
                // a new identifier shaped like a plural form (see PluralFormKey) would be read as one: not added
                $message = $shipped_entry === null && PluralFormKey::parse($identifier) !== null
                    ? null
                    : self::mergedMessage($overlay_entry, $shipped_entry, $template, $module, $plural_forms);
                if ($message === null) {
                    $outcome['not_merged'][] = $identifier;
                    continue;
                }
                // only a value the shipped .po does not carry yet is checked, see findInvalidChangedValues()
                $shipped_values = $shipped_entry === null ? [] : MigratedLanguageFileSync::flatValuesOf($shipped_entry, $plural_forms);
                $invalid = $policy->findInvalidChangedValues(self::flatValues($identifier, $message), $shipped_values, $shipped_values);
                if ($invalid !== []) {
                    $outcome['invalid_markup'] += $invalid;
                    continue;
                }
 
                $target = $shipped_entry ?? new TranslationEntry(null, $identifier);
                if ($message['plural_id'] === null) {
                    $target->translate($message['forms'][0]);
                } else {
                    $target->setPlural($message['plural_id'], $message['forms']);
                }
                $target->removeFlag('fuzzy');
                if ($shipped_entry === null) {
                    $shipped->addInIdOrder($target);
                    $template_changed = self::addToTemplate($template, $module, $identifier, $message['plural_id']) || $template_changed;
                }
                $outcome['merged'][] = $identifier;
                $shipped_entry = $target;
            }
 
            if ($remark !== null && $remark !== '') {
                if ($shipped_entry === null) {
                    // the remark of an entry without value that is not shipped - nothing to put it on
                    continue;
                }
                // as stored: compare the cleaned remark
                $notes = array_map('trim', MigratedLanguageFileSync::shippedNotesOf($shipped_entry));
                if (!in_array(trim(TranslationEntry::plainComment($remark)), $notes, true)) {
                    $shipped_entry->addTranslatorNote($remark);
                }
                $outcome['merged_remarks'][] = $identifier;
            }
        }
 
        if ($outcome['merged'] === [] && $outcome['merged_remarks'] === []) {
            return $outcome;
        }
 
        $po_content = self::checkedPoContent($shipped, true);
        $template_content = $template_changed && $template !== null ? self::checkedPoContent($template, false) : null;
        $previous_content = @file_get_contents($shipped_po);
        if ($previous_content === false) {
            throw new RuntimeException(sprintf('Could not read "%s".', $shipped_po));
        }
        if ($po_content === $previous_content && $template_content === null) {
            // taken over before (the overlay keeps it until "setup update"): nothing to write - only
            // a marking that failed then is made up for (a marked entry stays as it is)
            $outcome['unmarked'] = self::markMerged($language_file_directory_manager, $lang_key, $module, $outcome['merged'], $client_data_dir, $ilias_absolute_path);
            $outcome['merged'] = [];
            $outcome['merged_remarks'] = [];
            return $outcome;
        }
        // confined to the ILIAS directory: a symbolic link out of it is refused, nothing is written
        self::backup($shipped_po, $backup_file, $ilias_absolute_path, $previous_content);
        if ($template_content !== null) {
            // the template first: an additional entry there is harmless should the .po fail
            AtomicFileWriter::write($template_file, $template_content, $ilias_absolute_path);
        }
        AtomicFileWriter::write($shipped_po, $po_content, $ilias_absolute_path);
        $outcome['written'] = true;
        // the overlay keeps its values and the taken-over entries are marked, see the class docblock
        $outcome['unmarked'] = self::markMerged($language_file_directory_manager, $lang_key, $module, $outcome['merged'], $client_data_dir, $ilias_absolute_path);
 
        return $outcome;
    }
 
    private static function markMerged(
        LanguageFileDirectoryManager $language_file_directory_manager,
        string $lang_key,
        string $module,
        array $identifiers,
        string $client_data_dir,
        string $ilias_absolute_path
    ): ?string {
        try {
            MigratedLanguageFileSync::markMergedIntoShipped(
                $language_file_directory_manager,
                $lang_key,
                $module,
                $identifiers,
                $client_data_dir,
                $ilias_absolute_path
            );
        } catch (RuntimeException|\InvalidArgumentException $e) {
            return $e->getMessage();
        }
 
        return null;
    }
 
    private static function mergedMessage(
        TranslationEntry $overlay_entry,
        ?TranslationEntry $shipped_entry,
        ?TranslationCatalog $template,
        string $module,
        PluralForms $plural_forms
    ): ?array {
        if (!$overlay_entry->isPlural() && !$shipped_entry?->isPlural()) {
            return ['plural_id' => null, 'forms' => [$overlay_entry->getTranslation()]];
        }
        if ($overlay_entry->isPlural() && $shipped_entry !== null && !$shipped_entry->isPlural()) {
            return null;
        }
 
        if ($overlay_entry->isPlural()) {
            $forms = array_values(MigratedLanguageFileSync::flatValuesOf($overlay_entry, $plural_forms));
        } else {
            $forms = array_values(MigratedLanguageFileSync::flatValuesOf($shipped_entry, $plural_forms));
            $forms[$plural_forms->defaultFormIndex()] = $overlay_entry->getTranslation();
        }
        if ($forms[0] === '' && implode('', $forms) !== '') {
            return null;
        }
        $template_entry = $template === null ? null : MigratedLanguageFileSync::findModuleEntry($template, $module, $overlay_entry->getId());
 
        return [
            'plural_id' => $shipped_entry?->getPluralId()
                ?? $template_entry?->getPluralId()
                ?? $overlay_entry->getId() . '_plural',
            'forms' => $forms,
        ];
    }
 
    private static function addToTemplate(?TranslationCatalog $template, string $module, string $identifier, ?string $plural_id): bool
    {
        if ($template === null || MigratedLanguageFileSync::findModuleEntry($template, $module, $identifier) !== null) {
            return false;
        }
        $entry = new TranslationEntry(null, $identifier);
        if ($plural_id !== null) {
            $entry->setPlural($plural_id, ['', '']);
        }
        $template->addInIdOrder($entry);
 
        return true;
    }
 
    private static function flatValues(string $identifier, array $message): array
    {
        if ($message['plural_id'] === null) {
            return [$identifier => $message['forms'][0]];
        }
        $values = [];
        foreach ($message['forms'] as $form => $value) {
            $values[PluralFormKey::of($identifier, $form)] = $value;
        }
 
        return $values;
    }
 
    private static function checkedPoContent(TranslationCatalog $catalog, bool $compile): string
    {
        $content = $catalog->toPoString();
        if (!mb_check_encoding($content, 'UTF-8')) {
            throw new RuntimeException('The merged file would not be valid UTF-8.');
        }
        TranslationCatalog::fromPoString($content);
        if ($compile) {
            $catalog->toMoString();
        }
 
        return $content;
    }
 
    private static function backup(string $shipped_po, string $backup_file, string $confine_to_directory, ?string $content = null): void
    {
        $content ??= @file_get_contents($shipped_po);
        if ($content === false) {
            throw new RuntimeException(sprintf('Could not read "%s".', $shipped_po));
        }
        if (!is_dir(dirname($backup_file))) {
            throw new RuntimeException(sprintf('The backup directory "%s" does not exist.', dirname($backup_file)));
        }
        AtomicFileWriter::write($backup_file, $content, $confine_to_directory);
    }
 
    private static function assertWritableFile(string $file, string $confine_to_directory): void
    {
        $root = realpath($confine_to_directory);
        $directory = realpath(dirname($file));
        if (
            $root === false
            || $directory === false
            || ($directory !== $root && !str_starts_with($directory, rtrim($root, '/') . '/'))
            || !is_dir(dirname($file))
            || !is_writable(dirname($file))
            || is_link($file)
            || (file_exists($file) && (!is_file($file) || !is_writable($file)))
        ) {
            throw new RuntimeException(sprintf('"%s" is not writable or does not resolve to a path below "%s".', $file, $confine_to_directory));
        }
    }
 
    private static function backupFileNames(array $shipped_files): array
    {
        $counts = array_count_values(array_map('basename', $shipped_files));
        $names = [];
        foreach ($shipped_files as $module => $file) {
            $names[(string) $module] = $counts[basename($file)] === 1 ? basename($file) : null;
        }
 
        return $names;
    }
 
    private static function valuesOf(array $entries): array
    {
        return array_map(static fn(array $entry): string => $entry['value'], $entries);
    }
}