Relintai/pandemonium_engine
1
0
Fork 0
mirror of https://github.com/Relintai/pandemonium_engine.git synced 2024-11-21 16:37:20 +01:00
Code Issues Packages Projects Releases Wiki Activity
6adc5ec212
pandemonium_engine/scu_builders.py

596 lines
17 KiB
Python
Raw Blame History

"""Functions used to generate scu build source files during build time
"""
import glob, os
import math
from pathlib import Path
from os.path import normpath, basename
base_folder_path = str(Path(__file__).parent) + "/"
base_folder_only = os.path.basename(os.path.normpath(base_folder_path))
_verbose = False
_is_release_build = False
_scu_folders = set()
def clear_out_existing_files(output_folder, extension):
output_folder = os.path.abspath(output_folder)
# print("clear_out_existing_files from folder: " + output_folder)
if not os.path.isdir(output_folder):
# folder does not exist or has not been created yet,
# no files to clearout. (this is not an error)
return
os.chdir(output_folder)
for file in glob.glob("*." + extension):
# print("removed pre-existing file: " + file)
os.remove(file)
def folder_not_found(folder):
abs_folder = base_folder_path + folder + "/"
return not os.path.isdir(abs_folder)
def find_files_in_folder(folder, sub_folder, include_list, extension, sought_exceptions, found_exceptions):
abs_folder = base_folder_path + folder + "/" + sub_folder
if not os.path.isdir(abs_folder):
print("ERROR " + abs_folder + " not found.")
return include_list, found_exceptions
os.chdir(abs_folder)
sub_folder_slashed = ""
if sub_folder != "":
sub_folder_slashed = sub_folder + "/"
for file in glob.glob("*." + extension):
simple_name = Path(file).stem
if file.endswith(".gen.cpp"):
continue
li = '#include "' + folder + "/" + sub_folder_slashed + file + '"'
if not simple_name in sought_exceptions:
include_list.append(li)
else:
found_exceptions.append(li)
return include_list, found_exceptions
def write_output_file(file_count, include_list, start_line, end_line, output_folder, output_filename_prefix, extension):
output_folder = os.path.abspath(output_folder)
if not os.path.isdir(output_folder):
# create
os.mkdir(output_folder)
if not os.path.isdir(output_folder):
print("ERROR " + output_folder + " could not be created.")
return
print("CREATING folder " + output_folder)
file_text = ""
for l in range(start_line, end_line):
if l < len(include_list):
line = include_list[l]
li = line + "\n"
file_text += li
num_string = ""
if file_count > 0:
num_string = "_" + str(file_count)
short_filename = output_filename_prefix + num_string + ".gen." + extension
output_filename = output_folder + "/" + short_filename
if _verbose:
print("generating: " + short_filename)
output_path = Path(output_filename)
output_path.write_text(file_text, encoding="utf8")
def write_exception_output_file(file_count, exception_string, output_folder, output_filename_prefix, extension):
output_folder = os.path.abspath(output_folder)
if not os.path.isdir(output_folder):
print("ERROR " + output_folder + " does not exist.")
return
file_text = exception_string + "\n"
num_string = ""
if file_count > 0:
num_string = "_" + str(file_count)
short_filename = output_filename_prefix + "_exception" + num_string + ".gen." + extension
output_filename = output_folder + "/" + short_filename
if _verbose:
print("generating: " + short_filename)
output_path = Path(output_filename)
output_path.write_text(file_text, encoding="utf8")
def find_section_name(sub_folder):
# Construct a useful name for the section from the path for debug logging
section_path = os.path.abspath(base_folder_path + sub_folder) + "/"
folders = []
folder = ""
for i in range(8):
folder = os.path.dirname(section_path)
folder = os.path.basename(folder)
if folder == base_folder_only:
break
folders.append(folder)
section_path += "../"
section_path = os.path.abspath(section_path) + "/"
section_name = ""
for n in range(len(folders)):
section_name += folders[len(folders) - n - 1]
if n != (len(folders) - 1):
section_name += "_"
return section_name
# "folders" is a list of folders to add all the files from to add to the SCU
# "section (like a module)". The name of the scu file will be derived from the first folder
# (thus e.g. scene/3d becomes scu_scene_3d.gen.cpp)
# "includes_per_scu" limits the number of includes in a single scu file.
# This allows the module to be built in several translation units instead of just 1.
# This will usually be slower to compile but will use less memory per compiler instance, which
# is most relevant in release builds.
# "sought_exceptions" are a list of files (without extension) that contain
# e.g. naming conflicts, and are therefore not suitable for the scu build.
# These will automatically be placed in their own separate scu file,
# which is slow like a normal build, but prevents the naming conflicts.
# Ideally in these situations, the source code should be changed to prevent naming conflicts.
# "extension" will usually be cpp, but can also be set to c (for e.g. third party libraries that use c)
def process_folder(folders, sought_exceptions=[], includes_per_scu=0, extension="cpp"):
if len(folders) == 0:
return
# Construct the filename prefix from the FIRST folder name
# e.g. "scene_3d"
out_filename = find_section_name(folders[0])
found_includes = []
found_exceptions = []
main_folder = folders[0]
abs_main_folder = base_folder_path + main_folder
# Keep a record of all folders that have been processed for SCU,
# this enables deciding what to do when we call "add_source_files()"
global _scu_folders
_scu_folders.add(main_folder)
# main folder (first)
found_includes, found_exceptions = find_files_in_folder(
main_folder, "", found_includes, extension, sought_exceptions, found_exceptions
)
# sub folders
for d in range(1, len(folders)):
found_includes, found_exceptions = find_files_in_folder(
main_folder, folders[d], found_includes, extension, sought_exceptions, found_exceptions
)
found_includes = sorted(found_includes)
# calculate how many lines to write in each file
total_lines = len(found_includes)
# adjust number of output files according to whether DEV or release
num_output_files = 1
if _is_release_build:
# always have a maximum in release
includes_per_scu = 8
num_output_files = max(math.ceil(total_lines / includes_per_scu), 1)
else:
if includes_per_scu > 0:
num_output_files = max(math.ceil(total_lines / includes_per_scu), 1)
# error condition
if total_lines == 0:
return
lines_per_file = math.ceil(total_lines / num_output_files)
lines_per_file = max(lines_per_file, 1)
start_line = 0
file_number = 0
# These do not vary throughout the loop
output_folder = abs_main_folder + "/.scu/"
output_filename_prefix = "scu_" + out_filename
# Clear out any existing files (usually we will be overwriting,
# but we want to remove any that are pre-existing that will not be
# overwritten, so as to not compile anything stale)
clear_out_existing_files(output_folder, extension)
for file_count in range(0, num_output_files):
end_line = start_line + lines_per_file
# special case to cover rounding error in final file
if file_count == (num_output_files - 1):
end_line = len(found_includes)
write_output_file(
file_count, found_includes, start_line, end_line, output_folder, output_filename_prefix, extension
)
start_line = end_line
# Write the exceptions each in their own scu gen file,
# so they can effectively compile in "old style / normal build".
for exception_count in range(len(found_exceptions)):
write_exception_output_file(
exception_count, found_exceptions[exception_count], output_folder, output_filename_prefix, extension
)
def generate_scu_files(verbose, is_release_build):
print("=============================")
print("Single Compilation Unit Build")
print("=============================")
print("Generating SCU build files")
global _verbose
_verbose = verbose
global _is_release_build
_is_release_build = is_release_build
curr_folder = os.path.abspath("./")
# check we are running from the correct folder
if folder_not_found("core") or folder_not_found("platform") or folder_not_found("scene"):
raise RuntimeError("scu_builders.py must be run from the godot folder.")
return
process_folder(["core"])
process_folder(["core/bind"])
process_folder(["core/config"])
process_folder(["core/containers"])
process_folder(["core/error"])
process_folder(["core/input"])
process_folder(["core/log"])
process_folder(["core/math"])
process_folder(["core/object"])
process_folder(["core/os"])
process_folder(["core/string"])
process_folder(["core/variant"])
process_folder(["core/io"])
process_folder(["core/crypto"])
process_folder(["drivers/gles2"])
process_folder(["drivers/unix"])
process_folder(["drivers/png"])
process_folder(["editor"])
process_folder(["editor/import"])
process_folder(["editor/plugins"])
process_folder(["main"])
process_folder(["main/tests"])
process_folder(
[
"platform",
"android/export",
"iphone/export",
"javascript/export",
"osx/export",
#"uwp/export",
"windows/export",
"x11/export",
]
)
#TODO These should be moved to module conpig.py s
process_folder(["modules/broken_seals_module"])
process_folder(["modules/cscript"])
process_folder(["modules/csg"])
process_folder(["modules/database"])
process_folder(["modules/database_sqlite"])
process_folder(["modules/entity_spell_system",
"data/atlases",
"data/auras",
"data/items",
"data/loot",
"data/species",
"data/spells",
"database",
"drag_and_drop",
"editor",
"entities",
"entities/ai",
"entities/auras",
"entities/data",
"entities/resources",
"entities/skills",
"entities/stats",
"formations",
"infos",
"inventory",
"material_cache",
"pipelines",
"profiles",
"profiles/actionbar",
"profiles/input",
"projectiles/3d",
"props",
"singletons",
"skeleton",
"spawners",
"utility",
])
process_folder(["modules/fastnoise"])
process_folder(["modules/gdnative",
"gdnative",
"nativescript",
#"arvr",
"pluginscript",
"net",
])
process_folder(["modules/gdscript"])
process_folder(["modules/gdscript/language_server"])
process_folder(["modules/gridmap"])
process_folder(["modules/gridmap/geometry_parser"])
process_folder(["modules/http_server_simple"])
process_folder(["modules/lz4"])
process_folder(["modules/material_maker",
"algos",
"editor",
"editor/widgets/color_picker_popup",
"editor/widgets/curve_edit",
"editor/widgets/file_dialog",
"editor/widgets/float_edit",
"editor/widgets/gradient_editor",
"editor/widgets/image_picker_button",
"editor/widgets/mm_dnd_color_picker_button",
"editor/widgets/polygon_edit",
"editor/widgets/tones_editor",
"nodes",
"nodes/bases",
"nodes/filter",
"nodes/gradient",
"nodes/noise",
"nodes/other",
"nodes/pattern",
"nodes/sdf2d",
"nodes/sdf3d",
"nodes/simple",
"nodes/transform",
"nodes/uniform",
])
process_folder(["modules/mbedtls"])
process_folder(["modules/mesh_data_resource",
"editor",
"editor/utilities",
"editor/uv_editor",
"nodes",
"plugin",
"plugin_gltf",
"props",
"props_2d",
])
process_folder(["modules/mesh_utils"])
process_folder(["modules/navigation"])
process_folder(["modules/navigation_dummy"])
process_folder(["modules/navigation_geometry_parsers",
"geometry_parser_2d",
"geometry_parser_3d",
])
process_folder(["modules/navigation_mesh_generator"])
process_folder(["modules/navigation_mesh_generator/editor"])
process_folder(["modules/network_synchronizer"])
process_folder(["modules/paint",
"actions",
"editor",
"nodes",
"ui",
"ui/property_inspectors",
])
process_folder(["modules/props",
"clutter",
"editor",
"jobs",
"lights",
"material_cache",
"props",
"singleton",
"tiled_wall",
])
process_folder(["modules/props_2d",
"clutter",
"editor",
"jobs",
"lights",
"material_cache",
"props",
"singleton",
"tiled_wall",
])
process_folder(["modules/skeleton_2d",
"editor",
"nodes",
"resources",
])
process_folder(["modules/skeleton_3d",
"editor",
"nodes",
"resources",
])
process_folder(["modules/steering_ai",
"agents",
"behaviors",
"proximities",
])
process_folder(["modules/terraman",
"areas",
"data",
"level_generator",
"library",
"meshers",
"meshers/blocky",
"meshers/default",
"nodes",
"world",
"world/blocky",
"world/default",
"world/jobs",
])
process_folder(["modules/terraman_2d",
"areas",
"data",
"level_generator",
"library",
"meshers",
"meshers/default",
"meshers/isometric",
"meshers/simple",
"nodes",
"world",
"world/default",
"world/isometric",
"world/jobs",
"world/simple",
])
process_folder(["modules/texture_packer",
"layers",
"rectpack2D",
"texture_resource",
])
process_folder(["modules/tile_map",
"geometry_parser",
])
process_folder(["modules/ui_extensions"])
process_folder(["modules/unit_test"])
process_folder(["modules/users",
"managers",
"singleton",
"users",
"web/middleware",
"web/web_nodes",
])
process_folder(["modules/voxelman",
"areas",
"data",
"level_generator",
"library",
"meshers",
"meshers/blocky",
"meshers/cubic",
"meshers/default",
"meshers/marching_cubes",
"nodes",
"world",
"world/blocky",
"world/cubic",
"world/default",
"world/jobs",
"world/marching_cubes",
])
process_folder(["modules/web",
"database",
"editor",
"html",
"http",
"nodes/admin_panel",
"nodes/folder_serve_nodes",
"nodes/list_page",
"nodes/message_page",
"nodes/paged_article",
"nodes/redirect",
"nodes/static_pages",
])
process_folder(["modules/websocket"])
process_folder(["modules/wfc"])
#Editor Modules
process_folder(["editor_modules/editor_code_editor"])
process_folder(["editor_modules/gltf"])
process_folder(["editor_modules/gltf/structures"])
process_folder(["editor_modules/gltf/extensions"])
process_folder(["editor_modules/gltf/extensions/physics"])
process_folder(["editor_modules/plugin_refresher"])
process_folder(["editor_modules/shader_editor"])
process_folder(["editor_modules/text_editor"])
#process_folder(["modules/fbx"])
#process_folder(["modules/fbx/tools"])
#process_folder(["modules/fbx/data"])
#process_folder(["modules/fbx/fbx_parser"])
process_folder(["scene"])
process_folder(["scene/audio"])
process_folder(["scene/debugger"])
process_folder(["scene/2d"])
process_folder(["scene/3d"])
process_folder(["scene/animation"])
process_folder(["scene/gui"])
process_folder(["scene/main"])
process_folder(["scene/resources"])
process_folder(["servers"])
process_folder(["servers/rendering"])
process_folder(["servers/rendering/portals"])
process_folder(["servers/physics_2d"])
process_folder(["servers/physics"])
process_folder(["servers/physics/joints"])
process_folder(["servers/audio"])
process_folder(["servers/audio/effects"])
# Finally change back the path to the calling folder
os.chdir(curr_folder)
return _scu_folders
Reference in New Issue View Git Blame Copy Permalink