Concatenate/merge followups

.coveragerc

@@ -1,6 +1,7 @@
 [report]
 exclude_lines =
     NotImplemented
+    if TYPE_CHECKING:
     pragma: no cover
+    raise OptionError
     warnings.warn
-    if TYPE_CHECKING:

babel/messages/catalog.py

@@ -800,7 +800,7 @@ def add_conflict(self, message: Message, filename: str, project: str, version: s
         """
         key = self._key_for(message.id, message.context)
         self._conflicts[key].append({
-            'message': message,
+            'message': message.clone(),  # The catalog copy is mutable
             'filename': filename,
             'project': project,
             'version': version,
@@ -813,8 +813,9 @@ def get_conflicts(self, id: _MessageID, context: str | None = None) -> list[Conf
         :param context: optional message context (msgctxt)
         :return: list of :class:`ConflictInfo` dicts, or an empty list if none
         """
-        key = self._key_for(id, context)
-        return self._conflicts.get(key, [])
+        # Guard against catalogs serialized prior to conflicts existing
+        conflicts = getattr(self, '_conflicts', None) or {}
+        return conflicts.get(self._key_for(id, context), [])
 
     def add(
         self,

babel/messages/frontend.py

@@ -149,15 +149,15 @@ def __init__(self, dist=None):
         self.help = 0
         self.finalized = 0
 
-    def initialize_options(self):
+    def initialize_options(self):  # pragma: no cover
         pass
 
     def ensure_finalized(self):
         if not self.finalized:
             self.finalize_options()
         self.finalized = 1
 
-    def finalize_options(self):
+    def finalize_options(self):  # pragma: no cover
         raise RuntimeError(
             f"abstract method -- subclass {self.__class__} must override",
         )
@@ -886,6 +886,17 @@ def run(self):
             return
 
 
+def _is_translated(message) -> bool:
+    """Whether a message actually carries a translation.
+
+    An empty msgstr (or a plural tuple of empty msgstrs) is not a definition,
+    it just means the message is known but untranslated.
+    """
+    if isinstance(message.string, (list, tuple)):
+        return any(message.string)
+    return bool(message.string)
+
+
 class ConcatenateCatalog(CommandMixin):
     description = 'concatenates the specified PO files into single one'
     user_options = [
@@ -954,46 +965,73 @@ def finalize_options(self):
     def _collect_message_info(self):
         templates: list[tuple[str, Catalog]] = []
         message_counts: Counter[_MessageID] = Counter()
-        message_strings: dict[_MessageID, set[str | tuple[str, ...]]] = defaultdict(set)
+        message_strings: dict[_MessageID, set[_MessageID]] = defaultdict(set)
 
         for filename in self.input_files:
-            with open(filename) as pofile:
+            with open(filename, 'rb') as pofile:
                 template = read_po(pofile)
             for message in template:
                 if not message.id:
                     continue
-                message_counts[message.id] += 1
-                message_strings[message.id].add(
-                    message.string if isinstance(message.string, str) else tuple(message.string),
-                )
+                key = template._key_for(message.id, message.context)
+                message_counts[key] += 1
+                if _is_translated(message):
+                    message_strings[key].add(
+                        message.string if isinstance(message.string, str) else tuple(message.string),
+                    )
             templates.append((filename, template))
 
         return templates, message_counts, message_strings
 
     def run(self):
-        catalog = Catalog(fuzzy=False)
         templates, message_counts, message_strings = self._collect_message_info()
 
-        for path, template in templates:
-            if catalog.locale is None:
-                catalog.locale = template.locale
+        # Take the header from the first input preferring one that has a locale.
+        header_source = next(
+            (template for _, template in templates if template.locale is not None),
+            templates[0][1],
+        )
+        catalog = Catalog(fuzzy=False)
+        catalog.header_comment = header_source.header_comment
+        catalog.mime_headers = header_source.mime_headers
 
+        if len({template.charset for _, template in templates}) > 1:
+            # If charsets disagree, use UTF-8 as a common denominator.
+            catalog.charset = 'utf-8'
+
+        for path, template in templates:
             for message in template:
                 if not message.id:
                     continue
 
-                count = message_counts[message.id]
+                key = template._key_for(message.id, message.context)
+                count = message_counts[key]
                 if count <= self.more_than or (self.less_than is not None and count >= self.less_than):
                     continue
 
-                if count > 1 and not self.use_first and len(message_strings[message.id]) > 1:
-                    filename = os.path.basename(path)
-                    catalog.add_conflict(message, filename, template.project, template.version)
-                    message.flags |= {'fuzzy'}
+                translated = _is_translated(message)
+                conflicting = translated and not self.use_first and len(message_strings[key]) > 1
 
+                current = catalog.get(message.id, message.context)
+                was_translated = current is not None and _is_translated(current)
+                was_fuzzy = current is not None and current.fuzzy
                 catalog[message.id] = message
+                stored = catalog.get(message.id, message.context)
+
+                if translated and not was_translated and stored.pluralizable == message.pluralizable:
+                    stored.string = message.string
+                    was_fuzzy = message.fuzzy
+
+                if was_fuzzy:
+                    stored.flags.add('fuzzy')
+                elif current is not None:
+                    stored.flags.discard('fuzzy')
+
+                if conflicting:
+                    filename = os.path.basename(path)
+                    catalog.add_conflict(message, filename, template.project, template.version)
+                    stored.flags.add('fuzzy')
 
-        catalog.fuzzy = any(message.fuzzy for message in catalog)
 
         output_file = self.output_file
         if not output_file or output_file == '-':
@@ -1082,38 +1120,61 @@ def finalize_options(self):
 
         self._finalize_width()
 
+        if self.compendium is None:
+            self.compendium = []
+        elif isinstance(self.compendium, str):
+            self.compendium = [self.compendium]
+
     def _get_messages_from_compendiums(self, compendium_paths):
         for file_path in compendium_paths:
-            with open(file_path) as pofile:
+            with open(file_path, 'rb') as pofile:
                 catalog = read_po(pofile)
                 for message in catalog:
                     yield message, file_path
 
     def run(self):
         def_file, ref_file = self.input_files
 
-        with open(def_file) as pofile:
+        with open(def_file, 'rb') as pofile:
             catalog = read_po(pofile)
-        with open(ref_file) as pofile:
+        with open(ref_file, 'rb') as pofile:
             ref_catalog = read_po(pofile)
         catalog.update(
             ref_catalog,
             no_fuzzy_matching=self.no_fuzzy_matching,
         )
 
         for message, compendium_path in self._get_messages_from_compendiums(self.compendium):
-            if (current := catalog.get(message.id)) and (not current.string or current.fuzzy or self.compendium_overwrite):
-                if self.compendium_overwrite and not current.fuzzy and current.string:
-                    catalog.obsolete[message.id] = current.clone()
+            if not message.id or not _is_translated(message):
+                # Only translated compendium entries are of any use.
+                continue
 
-                current.string = message.string
-                if current.fuzzy:
-                    current.flags.remove('fuzzy')
+            current = catalog.get(message.id, message.context)
+            if current is None:
+                # The compendium does not add messages missing from the template.
+                continue
+
+            if current.pluralizable != message.pluralizable:
+                continue
+
+            if _is_translated(current) and not current.fuzzy:
+                if not self.compendium_overwrite:
+                    # Keep existing translations unless explicitly overwriting them.
+                    continue
+
+                # Preserve the translation being replaced as an obsolete message.
+                key = catalog._key_for(current.id, current.context)
+                catalog.obsolete[key] = current.clone()
+
+            current.string = message.string
+            if message.fuzzy:
+                current.flags.add('fuzzy')
+            else:
+                current.flags.discard('fuzzy')
 
-                if not self.no_compendium_comment:
-                    current.auto_comments.append(compendium_path)
+            if not self.no_compendium_comment:
+                current.auto_comments.append(compendium_path)
 
-        catalog.fuzzy = any(message.fuzzy for message in catalog)
         output_path = def_file if self.update else self.output_file
 
         if self.update and self.backup:

babel/messages/pofile.py

@@ -29,6 +29,8 @@
 _unescape_re = re.compile(r'\\([\\abfnrtv"])')
 _unescape_map = {'a': '\a', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', 'v': '\v'}
 
+CONFLICT_MARKER = "#-#-#-#-#"
+
 
 def unescape(string: str) -> str:
     r"""Reverse `escape` the given string.
@@ -644,36 +646,38 @@ def _format_comment(comment, prefix=''):
         for line in comment_wrapper.wrap(comment):
             yield f"#{prefix} {line.strip()}\n"
 
-    def _format_conflict_comment(file, project, version, prefix=''):
-        comment = f"#-#-#-#-#  {file} ({project} {version})  #-#-#-#-#"
-        yield f"{normalize(comment, prefix=prefix, width=width)}\n"
-
-    def _format_conflict(key: str | tuple[str, str], conflicts: list[ConflictInfo], prefix=''):
+    def _get_conflict_string(conflicts: list[ConflictInfo], plural_index: int | None = None) -> str:
+        parts = []
         for conflict in conflicts:
-            message = conflict['message']
-            if message.context:
-                yield from _format_conflict_comment(conflict['filename'], conflict['project'], conflict['version'], prefix=prefix)
-                yield f"{prefix}msgctxt {normalize(message.context, prefix=prefix, width=width)}\n"
+            parts.append(
+                f"{CONFLICT_MARKER}  "
+                f"{conflict['filename']} ({conflict['project']} {conflict['version']})"
+                f"  {CONFLICT_MARKER}",
+            )
+            string = conflict['message'].string
+            if plural_index is not None:
+                if isinstance(string, (list, tuple)):
+                    string = string[plural_index] if plural_index < len(string) else ''
+                elif plural_index:
+                    # A singular translation only contributes to the singular form.
+                    string = ''
+            parts.append(string)
+        return '\n'.join(parts)
 
-        if isinstance(key, (list, tuple)):
-            yield f"{prefix}msgid {normalize(key[0], prefix=prefix, width=width)}\n"
-            yield f"{prefix}msgid_plural {normalize(key[1], prefix=prefix, width=width)}\n"
-        else:
-            yield f"{prefix}msgid {normalize(key, prefix=prefix, width=width)}\n"
-        yield f"{prefix}msgstr {normalize('', prefix=prefix, width=width)}\n"
+    def _format_conflict(message, conflicts: list[ConflictInfo], prefix=''):
+        if message.context:
+            yield f"{prefix}msgctxt {normalize(message.context, prefix=prefix, width=width)}\n"
 
-        for conflict in conflicts:
-            message = conflict['message']
-            yield from _format_conflict_comment(conflict['filename'], conflict['project'], conflict['version'], prefix=prefix)
-            if isinstance(key, (list, tuple)):
-                for idx in range(catalog.num_plurals):
-                    try:
-                        string = message.string[idx]
-                    except IndexError:
-                        string = ''
-                    yield f"{prefix}msgstr[{idx:d}] {normalize(string, prefix=prefix, width=width)}\n"
-            else:
-                yield f"{normalize(message.string, prefix=prefix, width=width)}\n"
+        if isinstance(message.id, (list, tuple)):
+            yield f"{prefix}msgid {normalize(message.id[0], prefix=prefix, width=width)}\n"
+            yield f"{prefix}msgid_plural {normalize(message.id[1], prefix=prefix, width=width)}\n"
+            for idx in range(catalog.num_plurals):
+                string = _get_conflict_string(conflicts, plural_index=idx)
+                yield f"{prefix}msgstr[{idx:d}] {normalize(string, prefix=prefix, width=width)}\n"
+        else:
+            yield f"{prefix}msgid {normalize(message.id, prefix=prefix, width=width)}\n"
+            string = _get_conflict_string(conflicts)
+            yield f"{prefix}msgstr {normalize(string, prefix=prefix, width=width)}\n"
 
     def _format_message(message, prefix=''):
         if isinstance(message.id, (list, tuple)):
@@ -745,8 +749,8 @@ def _format_message(message, prefix=''):
                 norm_previous_id = normalize(message.previous_id[1], width=width)
                 yield from _format_comment(f'msgid_plural {norm_previous_id}', prefix='|')
 
-        if len(conflicts := catalog.get_conflicts(message.id)) > 0:
-            yield from _format_conflict(message.id, conflicts)
+        if conflicts := catalog.get_conflicts(message.id, message.context):
+            yield from _format_conflict(message, conflicts)
         else:
             yield from _format_message(message)
         yield '\n'