diff --git a/editor/translations/extract.py b/editor/translations/extract.py index b4c2ccc61..0862954e5 100755 --- a/editor/translations/extract.py +++ b/editor/translations/extract.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import enum import fnmatch import os import re @@ -8,6 +9,32 @@ import subprocess import sys +class Message: + __slots__ = ("msgid", "msgctxt", "comments", "locations") + + def format(self): + lines = [] + + if self.comments: + for i, content in enumerate(self.comments): + prefix = "#. TRANSLATORS:" if i == 0 else "#." + lines.append(prefix + content) + + lines.append("#: " + " ".join(self.locations)) + + if self.msgctxt: + lines.append('msgctxt "{}"'.format(self.msgctxt)) + + lines += [ + 'msgid "{}"'.format(self.msgid), + 'msgstr ""', + ] + + return "\n".join(lines) + + +messages_map = {} # (id, context) -> Message. + line_nb = False for arg in sys.argv[1:]: @@ -33,16 +60,14 @@ matches.sort() remaps = {} -remap_re = re.compile(r'capitalize_string_remaps\["(.+)"\] = "(.+)";') +remap_re = re.compile(r'^\t*capitalize_string_remaps\["(?P.+)"\] = (String::utf8\()?"(?P.+)"') with open("editor/editor_property_name_processor.cpp") as f: for line in f: m = remap_re.search(line) if m: - remaps[m.group(1)] = m.group(2) + remaps[m.group("from")] = m.group("to") -unique_str = [] -unique_loc = {} main_po = """ # LANGUAGE translation of the Pandemonium Engine editor. # Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur. @@ -62,15 +87,27 @@ msgstr "" """ +class ExtractType(enum.IntEnum): + TEXT = 1 + PROPERTY_PATH = 2 + GROUP = 3 + + +# Regex "(?P([^"\\]|\\.)*)" creates a group named `name` that matches a string. message_patterns = { - re.compile(r'RTR\("(([^"\\]|\\.)*)"\)'): False, - re.compile(r'TTR\("(([^"\\]|\\.)*)"\)'): False, - re.compile(r'TTRC\("(([^"\\]|\\.)*)"\)'): False, - re.compile(r'_initial_set\("([^"]+?)",'): True, - re.compile(r'GLOBAL_DEF(?:_RST)?\("([^".]+?)",'): True, - re.compile(r'EDITOR_DEF(?:_RST)?\("([^"]+?)",'): True, - re.compile(r'ADD_PROPERTY\(PropertyInfo\(Variant::[A-Z]+,\s*"([^"]+?)",'): True, - re.compile(r'ADD_GROUP\("([^"]+?)",'): False, + re.compile(r'RTR\("(?P([^"\\]|\\.)*)"\)'): ExtractType.TEXT, + re.compile(r'TTR\("(?P([^"\\]|\\.)*)"(, "(?P([^"\\]|\\.)*)")?\)'): ExtractType.TEXT, + re.compile(r'TTRC\("(?P([^"\\]|\\.)*)"\)'): ExtractType.TEXT, + re.compile(r'_initial_set\("(?P[^"]+?)",'): ExtractType.PROPERTY_PATH, + re.compile(r'GLOBAL_DEF(_RST)?(_NOVAL)?\("(?P[^".]+?)",'): ExtractType.PROPERTY_PATH, + re.compile(r'EDITOR_DEF(_RST)?\("(?P[^"]+?)",'): ExtractType.PROPERTY_PATH, + re.compile( + r'(ADD_PROPERTYI?|ImportOption|ExportOption)\(PropertyInfo\(Variant::[_A-Z0-9]+, "(?P[^"]+?)"[,)]' + ): ExtractType.PROPERTY_PATH, + re.compile( + r"(?!#define )LIMPL_PROPERTY(_RANGE)?\(Variant::[_A-Z0-9]+, (?P[^,]+?)," + ): ExtractType.PROPERTY_PATH, + re.compile(r'ADD_GROUP\("(?P[^"]+?)", "(?P[^"]*?)"\)'): ExtractType.GROUP, } @@ -79,76 +116,24 @@ capitalize_re = re.compile(r"(?<=\D)(?=\d)|(?<=\d)(?=\D([a-z]|\d))") def _process_editor_string(name): - # See String::capitalize(). - # fmt: off - capitalized = " ".join( - part.title() - for part in capitalize_re.sub("_", name).replace("_", " ").split() - ) - # fmt: on - # See EditorStringProcessor::process_string(). - for key, value in remaps.items(): - capitalized = capitalized.replace(key, value) - return capitalized - - -def _write_translator_comment(msg, translator_comment): - if translator_comment == "": - return - - global main_po - msg_pos = main_po.find('\nmsgid "' + msg + '"') - - # If it's a new message, just append comment to the end of PO file. - if msg_pos == -1: - main_po += _format_translator_comment(translator_comment, True) - return - - # Find position just before location. Translator comment will be added there. - translator_comment_pos = main_po.rfind("\n\n#", 0, msg_pos) + 2 - if translator_comment_pos - 2 == -1: - print("translator_comment_pos not found") - return - - # Check if a previous translator comment already exists. If so, merge them together. - if main_po.find("TRANSLATORS:", translator_comment_pos, msg_pos) != -1: - translator_comment_pos = main_po.find("\n#:", translator_comment_pos, msg_pos) + 1 - if translator_comment_pos == 0: - print('translator_comment_pos after "TRANSLATORS:" not found') - return - main_po = ( - main_po[:translator_comment_pos] - + _format_translator_comment(translator_comment, False) - + main_po[translator_comment_pos:] - ) - return - - main_po = ( - main_po[:translator_comment_pos] - + _format_translator_comment(translator_comment, True) - + main_po[translator_comment_pos:] - ) - - -def _format_translator_comment(comment, new): - if not comment: - return "" - - comment_lines = comment.split("\n") - - formatted_comment = "" - if not new: - for comment in comment_lines: - formatted_comment += "#. " + comment.strip() + "\n" - return formatted_comment - - formatted_comment = "#. TRANSLATORS: " - for i in range(len(comment_lines)): - if i == 0: - formatted_comment += comment_lines[i].strip() + "\n" + # See EditorPropertyNameProcessor::process_string(). + capitalized_parts = [] + for segment in name.split("_"): + if not segment: + continue + remapped = remaps.get(segment) + if remapped: + capitalized_parts.append(remapped) else: - formatted_comment += "#. " + comment_lines[i].strip() + "\n" - return formatted_comment + # See String::capitalize(). + # fmt: off + capitalized_parts.append(" ".join( + part.title() + for part in capitalize_re.sub("_", segment).replace("_", " ").split() + )) + # fmt: on + + return " ".join(capitalized_parts) def _is_block_translator_comment(translator_line): @@ -194,6 +179,7 @@ def process_file(f, fname): reading_translator_comment = False is_block_translator_comment = False translator_comment = "" + current_group = "" while l: @@ -212,45 +198,49 @@ def process_file(f, fname): translator_comment = translator_comment[:-1] # Remove extra \n at the end. if not reading_translator_comment: - for pattern, is_property_path in message_patterns.items(): + for pattern, extract_type in message_patterns.items(): for m in pattern.finditer(l): location = os.path.relpath(fname).replace("\\", "/") if line_nb: location += ":" + str(lc) - msg = m.group(1) + captures = m.groupdict("") + msg = captures.get("message", "") + msgctx = captures.get("context", "") - if is_property_path: + if extract_type == ExtractType.TEXT: + _add_message(msg, msgctx, location, translator_comment) + elif extract_type == ExtractType.PROPERTY_PATH: + if current_group: + if msg.startswith(current_group): + msg = msg[len(current_group) :] + else: + current_group = "" for part in msg.split("/"): - _add_message(_process_editor_string(part), location, translator_comment) - else: - _add_message(msg, location, translator_comment) - + _add_message(_process_editor_string(part), msgctx, location, translator_comment) + elif extract_type == ExtractType.GROUP: + _add_message(msg, msgctx, location, translator_comment) + current_group = captures["prefix"] translator_comment = "" l = f.readline() lc += 1 -def _add_message(msg, location, translator_comment): - global main_po, unique_str, unique_loc - - # Write translator comment. - _write_translator_comment(msg, translator_comment) - - if not msg in unique_str: - main_po += "#: " + location + "\n" - main_po += 'msgid "' + msg + '"\n' - main_po += 'msgstr ""\n\n' - unique_str.append(msg) - unique_loc[msg] = [location] - elif not location in unique_loc[msg]: - # Add additional location to previous occurrence too - msg_pos = main_po.find('\nmsgid "' + msg + '"') - if msg_pos == -1: - print("Someone apparently thought writing Python was as easy as GDScript. Ping Akien.") - main_po = main_po[:msg_pos] + " " + location + main_po[msg_pos:] - unique_loc[msg].append(location) +def _add_message(msg, msgctx, location, translator_comment): + key = (msg, msgctx) + message = messages_map.get(key) + if not message: + message = Message() + message.msgid = msg + message.msgctxt = msgctx + message.locations = [] + message.comments = [] + messages_map[key] = message + if location not in message.locations: + message.locations.append(location) + if translator_comment and translator_comment not in message.comments: + message.comments.append(translator_comment) print("Updating the editor.pot template...") @@ -259,6 +249,8 @@ for fname in matches: with open(fname, "r", encoding="utf8") as f: process_file(f, fname) +main_po += "\n\n".join(message.format() for message in messages_map.values()) + with open("editor.pot", "w") as f: f.write(main_po)