Relintai/pandemonium_engine
1
0
Fork 0
mirror of https://github.com/Relintai/pandemonium_engine.git synced 2024-11-22 08:57:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

Ported: [3.x] Improve embedded PCK loading and exporting. - bruvzg

Browse Source 
bea7c9be5d
This commit is contained in:
Relintai 2022-07-27 09:33:39 +02:00
parent 0768f95415
commit b26a1cd3b4
11 changed files with 320 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
core/io/file_access_pack.cpp
View File

@ -30,6 +30,7 @@
#include "file_access_pack.h"
#include "core/os/os.h"
#include "core/version.h"
#include <stdio.h>
@ -130,40 +131,77 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
return false;
}
bool pck_header_found = false;
// Search for the header at the start offset - standalone PCK file.
f->seek(p_offset);
uint32_t magic = f->get_32();
if (magic == PACK_HEADER_MAGIC) {
pck_header_found = true;
}
if (magic != PACK_HEADER_MAGIC) {
// loading with offset feature not supported for self contained exe files
// Search for the header in the executable "pck" section - self contained executable.
if (!pck_header_found) {
// Loading with offset feature not supported for self contained exe files.
if (p_offset != 0) {
f->close();
memdelete(f);
ERR_FAIL_V_MSG(false, "Loading self-contained executable with offset not supported.");
}
int64_t pck_off = OS::get_singleton()->get_embedded_pck_offset();
if (pck_off != 0) {
// Search for the header, in case PCK start and section have different alignment.
for (int i = 0; i < 8; i++) {
f->seek(pck_off);
magic = f->get_32();
if (magic == PACK_HEADER_MAGIC) {
#ifdef DEBUG_ENABLED
print_verbose("PCK header found in executable pck section, loading from offset 0x" + String::num_int64(pck_off - 4, 16));
#endif
pck_header_found = true;
break;
}
pck_off++;
}
}
}
// Search for the header at the end of file - self contained executable.
if (!pck_header_found) {
// Loading with offset feature not supported for self contained exe files.
if (p_offset != 0) {
f->close();
memdelete(f);
ERR_FAIL_V_MSG(false, "Loading self-contained executable with offset not supported.");
}
//maybe at the end.... self contained exe
f->seek_end();
f->seek(f->get_position() - 4);
magic = f->get_32();
if (magic != PACK_HEADER_MAGIC) {
f->close();
memdelete(f);
return false;
}
f->seek(f->get_position() - 12);
uint64_t ds = f->get_64();
f->seek(f->get_position() - ds - 8);
magic = f->get_32();
if (magic != PACK_HEADER_MAGIC) {
f->close();
memdelete(f);
return false;
if (magic == PACK_HEADER_MAGIC) {
f->seek(f->get_position() - 12);
uint64_t ds = f->get_64();
f->seek(f->get_position() - ds - 8);
magic = f->get_32();
if (magic == PACK_HEADER_MAGIC) {
#ifdef DEBUG_ENABLED
print_verbose("PCK header found at the end of executable, loading from offset 0x" + String::num_int64(f->get_position() - 4, 16));
#endif
pck_header_found = true;
}
}
}
if (!pck_header_found) {
f->close();
memdelete(f);
return false;
}
uint32_t version = f->get_32();
uint32_t ver_major = f->get_32();
uint32_t ver_minor = f->get_32();

5
core/os/os.cpp
View File

@ -416,6 +416,11 @@ String OS::get_locale_language() const {
return get_locale().left(3).replace("_", "");
}
// Embedded PCK offset.
uint64_t OS::get_embedded_pck_offset() const {
return 0;
}
// Helper function to ensure that a dir name/path will be valid on the OS
String OS::get_safe_dir_name(const String &p_dir_name, bool p_allow_dir_separator) const {
Vector<String> invalid_chars = String(": * ? \" < > |").split(" ");

2
core/os/os.h
View File

@ -480,6 +480,8 @@ public:
virtual String get_locale() const;
String get_locale_language() const;
virtual uint64_t get_embedded_pck_offset() const;
String get_safe_dir_name(const String &p_dir_name, bool p_allow_dir_separator = false) const;
virtual String get_pandemonium_dir_name() const;

84
editor/editor_export.cpp
View File

@ -30,27 +30,27 @@
#include "editor_export.h"
#include "core/class_db.h"
#include "core/crypto/crypto_core.h"
#include "core/error_macros.h"
#include "core/io/config_file.h"
#include "core/io/file_access_pack.h" // PACK_HEADER_MAGIC, PACK_FORMAT_VERSION
#include "core/io/resource_loader.h"
#include "core/io/zip_io.h"
#include "core/os/dir_access.h"
#include "core/os/file_access.h"
#include "core/os/memory.h"
#include "core/os/os.h"
#include "core/project_settings.h"
#include "core/script_language.h"
#include "core/version.h"
#include "core/version_generated.gen.h"
#include "editor/editor_file_system.h"
#include "editor/plugins/script_editor_plugin.h"
#include "editor_node.h"
#include "editor_settings.h"
#include "scene/resources/resource_format_text.h"
#include "core/class_db.h"
#include "core/error_macros.h"
#include "core/os/memory.h"
#include "core/os/os.h"
#include "core/version_generated.gen.h"
#include "scene/main/timer.h"
#include "scene/resources/resource_format_text.h"
#include "scene/resources/theme.h"
#include "thirdparty/minizip/ioapi.h"
#include "thirdparty/minizip/zip.h"
@ -1634,6 +1634,19 @@ List<String> EditorExportPlatformPC::get_binary_extensions(const Ref<EditorExpor
Error EditorExportPlatformPC::export_project(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int p_flags) {
ExportNotifier notifier(*this, p_preset, p_debug, p_path, p_flags);
Error err = prepare_template(p_preset, p_debug, p_path, p_flags);
if (err == OK) {
err = export_project_data(p_preset, p_debug, p_path, p_flags);
}
return err;
}
Error EditorExportPlatformPC::prepare_template(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int p_flags) {
if (!DirAccess::exists(p_path.get_base_dir())) {
return ERR_FILE_BAD_PATH;
}
String custom_debug = p_preset->get("custom_template/debug");
String custom_release = p_preset->get("custom_template/release");
@ -1666,38 +1679,41 @@ Error EditorExportPlatformPC::export_project(const Ref<EditorExportPreset> &p_pr
da->make_dir_recursive(p_path.get_base_dir());
Error err = da->copy(template_path, p_path, get_chmod_flags());
if (err == OK) {
String pck_path;
if (p_preset->get("binary_format/embed_pck")) {
pck_path = p_path;
} else {
pck_path = p_path.get_basename() + ".pck";
return err;
}
Error EditorExportPlatformPC::export_project_data(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int p_flags) {
String pck_path;
if (p_preset->get("binary_format/embed_pck")) {
pck_path = p_path;
} else {
pck_path = p_path.get_basename() + ".pck";
}
Vector<SharedObject> so_files;
int64_t embedded_pos;
int64_t embedded_size;
Error err = save_pack(p_preset, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size);
if (err == OK && p_preset->get("binary_format/embed_pck")) {
if (embedded_size >= 0x100000000 && !p_preset->get("binary_format/64_bits")) {
EditorNode::get_singleton()->show_warning(TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB."));
return ERR_INVALID_PARAMETER;
}
Vector<SharedObject> so_files;
int64_t embedded_pos;
int64_t embedded_size;
err = save_pack(p_preset, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size);
if (err == OK && p_preset->get("binary_format/embed_pck")) {
if (embedded_size >= 0x100000000 && !p_preset->get("binary_format/64_bits")) {
EditorNode::get_singleton()->show_warning(TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB."));
return ERR_INVALID_PARAMETER;
}
FixUpEmbeddedPckFunc fixup_func = get_fixup_embedded_pck_func();
if (fixup_func) {
err = fixup_func(p_path, embedded_pos, embedded_size);
}
FixUpEmbeddedPckFunc fixup_func = get_fixup_embedded_pck_func();
if (fixup_func) {
err = fixup_func(p_path, embedded_pos, embedded_size);
}
}
if (err == OK && !so_files.empty()) {
// If shared object files, copy them.
for (int i = 0; i < so_files.size() && err == OK; i++) {
err = da->copy(so_files[i].path, p_path.get_base_dir().plus_file(so_files[i].path.get_file()));
if (err == OK) {
err = sign_shared_object(p_preset, p_debug, p_path.get_base_dir().plus_file(so_files[i].path.get_file()));
}
if (err == OK && !so_files.empty()) {
// If shared object files, copy them.
DirAccessRef da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
for (int i = 0; i < so_files.size() && err == OK; i++) {
err = da->copy(so_files[i].path, p_path.get_base_dir().plus_file(so_files[i].path.get_file()));
if (err == OK) {
err = sign_shared_object(p_preset, p_debug, p_path.get_base_dir().plus_file(so_files[i].path.get_file()));
}
}
}

5
editor/editor_export.h
View File

@ -37,11 +37,11 @@
#include "scene/main/timer.h"
#include "scene/resources/texture.h"
#include "core/os/dir_access.h"
#include "core/error_list.h"
#include "core/list.h"
#include "core/map.h"
#include "core/object.h"
#include "core/os/dir_access.h"
#include "core/pool_vector.h"
#include "core/set.h"
#include "core/string_name.h"
@ -449,6 +449,9 @@ public:
virtual Error export_project(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int p_flags = 0);
virtual Error sign_shared_object(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path);
Error prepare_template(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int p_flags);
Error export_project_data(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int p_flags);
void set_extension(const String &p_extension, const String &p_feature_key = "default");
void set_name(const String &p_name);
void set_os_name(const String &p_name);

68
platform/windows/export/export.cpp
View File

@ -42,7 +42,7 @@
static Error fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size);
class EditorExportPlatformWindows : public EditorExportPlatformPC {
void _rcedit_add_data(const Ref<EditorExportPreset> &p_preset, const String &p_path);
Error _rcedit_add_data(const Ref<EditorExportPreset> &p_preset, const String &p_path);
Error _code_sign(const Ref<EditorExportPreset> &p_preset, const String &p_path);
public:
@ -62,16 +62,27 @@ Error EditorExportPlatformWindows::sign_shared_object(const Ref<EditorExportPres
}
Error EditorExportPlatformWindows::export_project(const Ref<EditorExportPreset> &p_preset, bool p_debug, const String &p_path, int p_flags) {
Error err = EditorExportPlatformPC::export_project(p_preset, p_debug, p_path, p_flags);
if (err != OK) {
return err;
String pck_path = p_path;
if (p_preset->get("binary_format/embed_pck")) {
pck_path = p_path.get_basename() + ".tmp";
}
_rcedit_add_data(p_preset, p_path);
Error err = EditorExportPlatformPC::prepare_template(p_preset, p_debug, pck_path, p_flags);
if (p_preset->get("application/modify_resources") && err == OK) {
err = _rcedit_add_data(p_preset, pck_path);
}
if (err == OK) {
err = EditorExportPlatformPC::export_project_data(p_preset, p_debug, pck_path, p_flags);
}
if (p_preset->get("codesign/enable") && err == OK) {
err = _code_sign(p_preset, p_path);
err = _code_sign(p_preset, pck_path);
}
if (p_preset->get("binary_format/embed_pck") && err == OK) {
DirAccessRef tmp_dir = DirAccess::create_for_path(p_path.get_base_dir());
err = tmp_dir->rename(pck_path, p_path);
}
return err;
@ -98,6 +109,7 @@ void EditorExportPlatformWindows::get_export_options(List<ExportOption> *r_optio
r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "codesign/description"), ""));
r_options->push_back(ExportOption(PropertyInfo(Variant::POOL_STRING_ARRAY, "codesign/custom_options"), PoolStringArray()));
r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "application/modify_resources"), true));
r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/icon", PROPERTY_HINT_FILE, "*.ico"), ""));
r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/file_version", PROPERTY_HINT_PLACEHOLDER_TEXT, "1.0.0.0"), ""));
r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/product_version", PROPERTY_HINT_PLACEHOLDER_TEXT, "1.0.0.0"), ""));
@ -108,17 +120,16 @@ void EditorExportPlatformWindows::get_export_options(List<ExportOption> *r_optio
r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/trademarks"), ""));
}
void EditorExportPlatformWindows::_rcedit_add_data(const Ref<EditorExportPreset> &p_preset, const String &p_path) {
Error EditorExportPlatformWindows::_rcedit_add_data(const Ref<EditorExportPreset> &p_preset, const String &p_path) {
String rcedit_path = EditorSettings::get_singleton()->get("export/windows/rcedit");
if (rcedit_path.empty()) {
WARN_PRINT("The rcedit tool is not configured in the Editor Settings (Export > Windows > Rcedit). No custom icon or app information data will be embedded in the exported executable.");
return;
if (rcedit_path != String() && !FileAccess::exists(rcedit_path)) {
ERR_PRINT("Could not find rcedit executable at " + rcedit_path + ", aborting.");
return ERR_FILE_NOT_FOUND;
}
if (!FileAccess::exists(rcedit_path)) {
ERR_PRINT("Could not find rcedit executable at " + rcedit_path + ", no icon or app information data will be included.");
return;
if (rcedit_path == String()) {
rcedit_path = "rcedit"; // try to run rcedit from PATH
}
#ifndef WINDOWS_ENABLED
@ -126,8 +137,8 @@ void EditorExportPlatformWindows::_rcedit_add_data(const Ref<EditorExportPreset>
String wine_path = EditorSettings::get_singleton()->get("export/windows/wine");
if (wine_path != String() && !FileAccess::exists(wine_path)) {
ERR_PRINT("Could not find wine executable at " + wine_path + ", no icon or app information data will be included.");
return;
ERR_PRINT("Could not find wine executable at " + wine_path + ", aborting.");
return ERR_FILE_NOT_FOUND;
}
if (wine_path == String()) {
@ -185,13 +196,22 @@ void EditorExportPlatformWindows::_rcedit_add_data(const Ref<EditorExportPreset>
args.push_back(trademarks);
}
#ifdef WINDOWS_ENABLED
OS::get_singleton()->execute(rcedit_path, args, true);
#else
#ifndef WINDOWS_ENABLED
// On non-Windows we need WINE to run rcedit
args.push_front(rcedit_path);
OS::get_singleton()->execute(wine_path, args, true);
rcedit_path = wine_path;
#endif
String str;
Error err = OS::get_singleton()->execute(rcedit_path, args, true, nullptr, &str, nullptr, true);
ERR_FAIL_COND_V_MSG(err != OK, err, "Could not start rcedit executable, configure rcedit path in the Editor Settings (Export > Windows > Rcedit).");
print_line("rcedit (" + p_path + "): " + str);
if (str.find("Fatal error") != -1) {
return FAILED;
}
return OK;
}
Error EditorExportPlatformWindows::_code_sign(const Ref<EditorExportPreset> &p_preset, const String &p_path) {
@ -328,7 +348,7 @@ Error EditorExportPlatformWindows::_code_sign(const Ref<EditorExportPreset> &p_p
String str;
Error err = OS::get_singleton()->execute(signtool_path, args, true, nullptr, &str, nullptr, true);
ERR_FAIL_COND_V(err != OK, err);
ERR_FAIL_COND_V_MSG(err != OK, err, "Could not start signtool executable, configure signtool path in the Editor Settings (Export > Windows > Signtool).");
print_line("codesign (" + p_path + "): " + str);
#ifndef WINDOWS_ENABLED
@ -357,7 +377,7 @@ bool EditorExportPlatformWindows::can_export(const Ref<EditorExportPreset> &p_pr
bool valid = EditorExportPlatformPC::can_export(p_preset, err, r_missing_templates);
String rcedit_path = EditorSettings::get_singleton()->get("export/windows/rcedit");
if (rcedit_path.empty()) {
if (p_preset->get("application/modify_resources") && rcedit_path.empty()) {
err += TTR("The rcedit tool must be configured in the Editor Settings (Export > Windows > Rcedit) to change the icon or app information data.") + "\n";
}
@ -432,6 +452,10 @@ void register_windows_exporter() {
static Error fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size) {
// Patch the header of the "pck" section in the PE file so that it corresponds to the embedded data
if (p_embedded_size + p_embedded_start >= 0x100000000) { // Check for total executable size
ERR_FAIL_V_MSG(ERR_INVALID_DATA, "Windows executables cannot be >= 4 GiB.");
}
FileAccess *f = FileAccess::open(p_path, FileAccess::READ_WRITE);
if (!f) {
return ERR_CANT_OPEN;

52
platform/windows/os_windows.cpp
View File

@ -3416,6 +3416,58 @@ MainLoop *OS_Windows::get_main_loop() const {
return main_loop;
}
uint64_t OS_Windows::get_embedded_pck_offset() const {
FileAccessRef f = FileAccess::open(get_executable_path(), FileAccess::READ);
if (!f) {
return 0;
}
// Process header.
{
f->seek(0x3c);
uint32_t pe_pos = f->get_32();
f->seek(pe_pos);
uint32_t magic = f->get_32();
if (magic != 0x00004550) {
return 0;
}
}
int num_sections;
{
int64_t header_pos = f->get_position();
f->seek(header_pos + 2);
num_sections = f->get_16();
f->seek(header_pos + 16);
uint16_t opt_header_size = f->get_16();
// Skip rest of header + optional header to go to the section headers.
f->seek(f->get_position() + 2 + opt_header_size);
}
int64_t section_table_pos = f->get_position();
// Search for the "pck" section.
int64_t off = 0;
for (int i = 0; i < num_sections; ++i) {
int64_t section_header_pos = section_table_pos + i * 40;
f->seek(section_header_pos);
uint8_t section_name[9];
f->get_buffer(section_name, 8);
section_name[8] = '\0';
if (strcmp((char *)section_name, "pck") == 0) {
f->seek(section_header_pos + 20);
off = f->get_32();
break;
}
}
return off;
}
String OS_Windows::get_config_path() const {
// The XDG Base Directory specification technically only applies on Linux/*BSD, but it doesn't hurt to support it on Windows as well.
if (has_environment("XDG_CONFIG_HOME")) {

6
platform/windows/os_windows.h
View File

@ -481,6 +481,8 @@ public:
virtual MainLoop *get_main_loop() const;
virtual uint64_t get_embedded_pck_offset() const;
virtual String get_name() const;
virtual Date get_date(bool utc) const;
@ -554,7 +556,9 @@ public:
void run();
virtual bool get_swap_ok_cancel() { return true; }
virtual bool get_swap_ok_cancel() {
return true;
}
virtual bool is_joy_known(int p_device);
virtual String get_joy_guid(int p_device) const;

24
platform/windows/pandemonium_windows.cpp
View File

@ -39,16 +39,25 @@
#ifndef TOOLS_ENABLED
#if defined _MSC_VER
#pragma section("pck", read)
__declspec(allocate("pck")) static const char dummy[8] = { 0 };
__declspec(allocate("pck")) static char dummy[8] = { 0 };
// Dummy function to prevent LTO from discarding "pck" section.
extern "C" char *__cdecl pck_section_dummy_call() {
return &dummy[0];
};
#if defined _AMD64_
#pragma comment(linker, "/include:pck_section_dummy_call")
#elif defined _X86_
#pragma comment(linker, "/include:_pck_section_dummy_call")
#endif
#elif defined __GNUC__
static const char dummy[8] __attribute__((section("pck"), used)) = { 0 };
#endif
#endif
PCHAR *
CommandLineToArgvA(
PCHAR CmdLine,
int *_argc) {
PCHAR *CommandLineToArgvA(PCHAR CmdLine, int *_argc) {
PCHAR *argv;
PCHAR _argv;
ULONG len;
@ -140,11 +149,6 @@ __declspec(dllexport) int widechar_main(int argc, wchar_t **argv) {
setlocale(LC_CTYPE, "");
#ifndef TOOLS_ENABLED
// Workaround to prevent LTCG (MSVC LTO) from removing "pck" section
const char *dummy_guard = dummy;
#endif
char **argv_utf8 = new char *[argc];
for (int i = 0; i < argc; ++i) {

85
platform/x11/os_x11.cpp
View File

@ -3047,6 +3047,91 @@ MainLoop *OS_X11::get_main_loop() const {
return main_loop;
}
uint64_t OS_X11::get_embedded_pck_offset() const {
FileAccessRef f = FileAccess::open(get_executable_path(), FileAccess::READ);
if (!f) {
return 0;
}
// Read and check ELF magic number.
{
uint32_t magic = f->get_32();
if (magic != 0x464c457f) { // 0x7F + "ELF"
return 0;
}
}
// Read program architecture bits from class field.
int bits = f->get_8() * 32;
// Get info about the section header table.
int64_t section_table_pos;
int64_t section_header_size;
if (bits == 32) {
section_header_size = 40;
f->seek(0x20);
section_table_pos = f->get_32();
f->seek(0x30);
} else { // 64
section_header_size = 64;
f->seek(0x28);
section_table_pos = f->get_64();
f->seek(0x3c);
}
int num_sections = f->get_16();
int string_section_idx = f->get_16();
// Load the strings table.
uint8_t *strings;
{
// Jump to the strings section header.
f->seek(section_table_pos + string_section_idx * section_header_size);
// Read strings data size and offset.
int64_t string_data_pos;
int64_t string_data_size;
if (bits == 32) {
f->seek(f->get_position() + 0x10);
string_data_pos = f->get_32();
string_data_size = f->get_32();
} else { // 64
f->seek(f->get_position() + 0x18);
string_data_pos = f->get_64();
string_data_size = f->get_64();
}
// Read strings data.
f->seek(string_data_pos);
strings = (uint8_t *)memalloc(string_data_size);
if (!strings) {
return 0;
}
f->get_buffer(strings, string_data_size);
}
// Search for the "pck" section.
int64_t off = 0;
for (int i = 0; i < num_sections; ++i) {
int64_t section_header_pos = section_table_pos + i * section_header_size;
f->seek(section_header_pos);
uint32_t name_offset = f->get_32();
if (strcmp((char *)strings + name_offset, "pck") == 0) {
if (bits == 32) {
f->seek(section_header_pos + 0x10);
off = f->get_32();
} else { // 64
f->seek(section_header_pos + 0x18);
off = f->get_64();
}
break;
}
}
memfree(strings);
return off;
}
void OS_X11::delete_main_loop() {
// Send owned clipboard data to clipboard manager before exit.
// This has to be done here because the clipboard data is cleared before finalize().

2
platform/x11/os_x11.h
View File

@ -268,6 +268,8 @@ public:
virtual MainLoop *get_main_loop() const;
virtual uint64_t get_embedded_pck_offset() const;
virtual bool can_draw() const;
virtual void set_clipboard(const String &p_text);