Ported: Refactor editor extract.py - timothyqiu

545e2ceb8a

editor/translations/extract.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import enum
 import fnmatch
 import os
 import re
@@ -8,6 +9,32 @@ import subprocess
 import sys
 
 
+class Message:
+    __slots__ = ("msgid", "msgctxt", "comments", "locations")
+
+    def format(self):
+        lines = []
+
+        if self.comments:
+            for i, content in enumerate(self.comments):
+                prefix = "#. TRANSLATORS:" if i == 0 else "#."
+                lines.append(prefix + content)
+
+        lines.append("#: " + " ".join(self.locations))
+
+        if self.msgctxt:
+            lines.append('msgctxt "{}"'.format(self.msgctxt))
+
+        lines += [
+            'msgid "{}"'.format(self.msgid),
+            'msgstr ""',
+        ]
+
+        return "\n".join(lines)
+
+
+messages_map = {}  # (id, context) -> Message.
+
 line_nb = False
 
 for arg in sys.argv[1:]:
@@ -33,16 +60,14 @@ matches.sort()
 
 
 remaps = {}
-remap_re = re.compile(r'capitalize_string_remaps\["(.+)"\] = "(.+)";')
+remap_re = re.compile(r'^\t*capitalize_string_remaps\["(?P<from>.+)"\] = (String::utf8\()?"(?P<to>.+)"')
 with open("editor/editor_property_name_processor.cpp") as f:
     for line in f:
         m = remap_re.search(line)
         if m:
-            remaps[m.group(1)] = m.group(2)
+            remaps[m.group("from")] = m.group("to")
 
 
-unique_str = []
-unique_loc = {}
 main_po = """
 # LANGUAGE translation of the Pandemonium Engine editor.
 # Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.
@@ -62,15 +87,27 @@ msgstr ""
 """
 
 
+class ExtractType(enum.IntEnum):
+    TEXT = 1
+    PROPERTY_PATH = 2
+    GROUP = 3
+
+
+# Regex "(?P<name>([^"\\]|\\.)*)" creates a group named `name` that matches a string.
 message_patterns = {
-    re.compile(r'RTR\("(([^"\\]|\\.)*)"\)'): False,
+    re.compile(r'RTR\("(?P<message>([^"\\]|\\.)*)"\)'): ExtractType.TEXT,
-    re.compile(r'TTR\("(([^"\\]|\\.)*)"\)'): False,
+    re.compile(r'TTR\("(?P<message>([^"\\]|\\.)*)"(, "(?P<context>([^"\\]|\\.)*)")?\)'): ExtractType.TEXT,
-    re.compile(r'TTRC\("(([^"\\]|\\.)*)"\)'): False,
+    re.compile(r'TTRC\("(?P<message>([^"\\]|\\.)*)"\)'): ExtractType.TEXT,
-    re.compile(r'_initial_set\("([^"]+?)",'): True,
+    re.compile(r'_initial_set\("(?P<message>[^"]+?)",'): ExtractType.PROPERTY_PATH,
-    re.compile(r'GLOBAL_DEF(?:_RST)?\("([^".]+?)",'): True,
+    re.compile(r'GLOBAL_DEF(_RST)?(_NOVAL)?\("(?P<message>[^".]+?)",'): ExtractType.PROPERTY_PATH,
-    re.compile(r'EDITOR_DEF(?:_RST)?\("([^"]+?)",'): True,
+    re.compile(r'EDITOR_DEF(_RST)?\("(?P<message>[^"]+?)",'): ExtractType.PROPERTY_PATH,
-    re.compile(r'ADD_PROPERTY\(PropertyInfo\(Variant::[A-Z]+,\s*"([^"]+?)",'): True,
+    re.compile(
-    re.compile(r'ADD_GROUP\("([^"]+?)",'): False,
+        r'(ADD_PROPERTYI?|ImportOption|ExportOption)\(PropertyInfo\(Variant::[_A-Z0-9]+, "(?P<message>[^"]+?)"[,)]'
+    ): ExtractType.PROPERTY_PATH,
+    re.compile(
+        r"(?!#define )LIMPL_PROPERTY(_RANGE)?\(Variant::[_A-Z0-9]+, (?P<message>[^,]+?),"
+    ): ExtractType.PROPERTY_PATH,
+    re.compile(r'ADD_GROUP\("(?P<message>[^"]+?)", "(?P<prefix>[^"]*?)"\)'): ExtractType.GROUP,
 }
 
 
@@ -79,76 +116,24 @@ capitalize_re = re.compile(r"(?<=\D)(?=\d)|(?<=\d)(?=\D([a-z]|\d))")
 
 
 def _process_editor_string(name):
-    # See String::capitalize().
+    # See EditorPropertyNameProcessor::process_string().
-    # fmt: off
+    capitalized_parts = []
-    capitalized = " ".join(
+    for segment in name.split("_"):
-        part.title()
+        if not segment:
-        for part in capitalize_re.sub("_", name).replace("_", " ").split()
+            continue
-    )
+        remapped = remaps.get(segment)
-    # fmt: on
+        if remapped:
-    # See EditorStringProcessor::process_string().
+            capitalized_parts.append(remapped)
-    for key, value in remaps.items():
-        capitalized = capitalized.replace(key, value)
-    return capitalized
-
-
-def _write_translator_comment(msg, translator_comment):
-    if translator_comment == "":
-        return
-
-    global main_po
-    msg_pos = main_po.find('\nmsgid "' + msg + '"')
-
-    # If it's a new message, just append comment to the end of PO file.
-    if msg_pos == -1:
-        main_po += _format_translator_comment(translator_comment, True)
-        return
-
-    # Find position just before location. Translator comment will be added there.
-    translator_comment_pos = main_po.rfind("\n\n#", 0, msg_pos) + 2
-    if translator_comment_pos - 2 == -1:
-        print("translator_comment_pos not found")
-        return
-
-    # Check if a previous translator comment already exists. If so, merge them together.
-    if main_po.find("TRANSLATORS:", translator_comment_pos, msg_pos) != -1:
-        translator_comment_pos = main_po.find("\n#:", translator_comment_pos, msg_pos) + 1
-        if translator_comment_pos == 0:
-            print('translator_comment_pos after "TRANSLATORS:" not found')
-            return
-        main_po = (
-            main_po[:translator_comment_pos]
-            + _format_translator_comment(translator_comment, False)
-            + main_po[translator_comment_pos:]
-        )
-        return
-
-    main_po = (
-        main_po[:translator_comment_pos]
-        + _format_translator_comment(translator_comment, True)
-        + main_po[translator_comment_pos:]
-    )
-
-
-def _format_translator_comment(comment, new):
-    if not comment:
-        return ""
-
-    comment_lines = comment.split("\n")
-
-    formatted_comment = ""
-    if not new:
-        for comment in comment_lines:
-            formatted_comment += "#. " + comment.strip() + "\n"
-        return formatted_comment
-
-    formatted_comment = "#. TRANSLATORS: "
-    for i in range(len(comment_lines)):
-        if i == 0:
-            formatted_comment += comment_lines[i].strip() + "\n"
         else:
-            formatted_comment += "#. " + comment_lines[i].strip() + "\n"
+            # See String::capitalize().
-    return formatted_comment
+            # fmt: off
+            capitalized_parts.append(" ".join(
+                part.title()
+                for part in capitalize_re.sub("_", segment).replace("_", " ").split()
+            ))
+            # fmt: on
+
+    return " ".join(capitalized_parts)
 
 
 def _is_block_translator_comment(translator_line):
@@ -194,6 +179,7 @@ def process_file(f, fname):
     reading_translator_comment = False
     is_block_translator_comment = False
     translator_comment = ""
+    current_group = ""
 
     while l:
 
@@ -212,45 +198,49 @@ def process_file(f, fname):
                 translator_comment = translator_comment[:-1]  # Remove extra \n at the end.
 
         if not reading_translator_comment:
-            for pattern, is_property_path in message_patterns.items():
+            for pattern, extract_type in message_patterns.items():
                 for m in pattern.finditer(l):
                     location = os.path.relpath(fname).replace("\\", "/")
                     if line_nb:
                         location += ":" + str(lc)
 
-                    msg = m.group(1)
+                    captures = m.groupdict("")
+                    msg = captures.get("message", "")
+                    msgctx = captures.get("context", "")
 
-                    if is_property_path:
+                    if extract_type == ExtractType.TEXT:
+                        _add_message(msg, msgctx, location, translator_comment)
+                    elif extract_type == ExtractType.PROPERTY_PATH:
+                        if current_group:
+                            if msg.startswith(current_group):
+                                msg = msg[len(current_group) :]
+                            else:
+                                current_group = ""
                         for part in msg.split("/"):
-                            _add_message(_process_editor_string(part), location, translator_comment)
+                            _add_message(_process_editor_string(part), msgctx, location, translator_comment)
-                    else:
+                    elif extract_type == ExtractType.GROUP:
-                        _add_message(msg, location, translator_comment)
+                        _add_message(msg, msgctx, location, translator_comment)
-
+                        current_group = captures["prefix"]
             translator_comment = ""
 
         l = f.readline()
         lc += 1
 
 
-def _add_message(msg, location, translator_comment):
+def _add_message(msg, msgctx, location, translator_comment):
-    global main_po, unique_str, unique_loc
+    key = (msg, msgctx)
-
+    message = messages_map.get(key)
-    # Write translator comment.
+    if not message:
-    _write_translator_comment(msg, translator_comment)
+        message = Message()
-
+        message.msgid = msg
-    if not msg in unique_str:
+        message.msgctxt = msgctx
-        main_po += "#: " + location + "\n"
+        message.locations = []
-        main_po += 'msgid "' + msg + '"\n'
+        message.comments = []
-        main_po += 'msgstr ""\n\n'
+        messages_map[key] = message
-        unique_str.append(msg)
+    if location not in message.locations:
-        unique_loc[msg] = [location]
+        message.locations.append(location)
-    elif not location in unique_loc[msg]:
+    if translator_comment and translator_comment not in message.comments:
-        # Add additional location to previous occurrence too
+        message.comments.append(translator_comment)
-        msg_pos = main_po.find('\nmsgid "' + msg + '"')
-        if msg_pos == -1:
-            print("Someone apparently thought writing Python was as easy as GDScript. Ping Akien.")
-        main_po = main_po[:msg_pos] + " " + location + main_po[msg_pos:]
-        unique_loc[msg].append(location)
 
 
 print("Updating the editor.pot template...")
@@ -259,6 +249,8 @@ for fname in matches:
     with open(fname, "r", encoding="utf8") as f:
         process_file(f, fname)
 
+main_po += "\n\n".join(message.format() for message in messages_map.values())
+
 with open("editor.pot", "w") as f:
     f.write(main_po)