Skip to content

Genericize the CMake source generator for embedding files #1849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open

Genericize the CMake source generator for embedding files #1849

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
d62c2c4
cmake: rename DaemonBuildInfo as DaemonSourceGenerator
illwieckz Oct 6, 2025
7c106ed
DaemonSourceGenerator: rework it a bit
illwieckz Oct 6, 2025
10f1524
DaemonSourceGenerator: make the embedded file generator reusable
illwieckz Oct 3, 2025
fb9e39f
EmbedText make possible to chose between TEXT and BINARY formats when…
illwieckz Oct 6, 2025
9af0bec
DaemonSourceGenerator: provide embedded files as part as the namespace
illwieckz Oct 6, 2025
59386eb
EmbedText: split embedded source lines just in case if an editor or d…
illwieckz Oct 6, 2025
14a748e
DaemonSourceGenerator: also add the generated source entry-point to t…
illwieckz Oct 6, 2025
b96492c
DaemonSourceGenerator: lowercase local variables (even if the leak ou…
illwieckz Oct 6, 2025
f174fb6
DaemonSourceGenerator: also make generated source depend on generator…
illwieckz Oct 6, 2025
f87d034
DaemonSourceGenerator: explicit mkdir
illwieckz Oct 7, 2025
ac27caa
DaemonSourceGenerator: do not copy the embedded files in the map
illwieckz Oct 6, 2025
2acdc5c
cmake: rename EmbedText as DaemonFileEmbedder
illwieckz Oct 8, 2025
f41f5db
DaemonSourceGenerator: add license
illwieckz Oct 9, 2025
6ca22b6
DaemonSourceGenerator: use file(GENERATE …)
illwieckz Oct 16, 2025
2bdccd1
DaemonSourceGenerator: use explicit variable names
illwieckz Oct 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 3 additions & 34 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ if (Daemon_OUT)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${Daemon_OUT})
endif()

include(DaemonBuildInfo)
include(DaemonSourceGenerator)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Name seems inapt, how about DaemonFileEmbedding? Or alternatively something about "resources" since that is a commonly used term for embedding files in this way.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just DaemonEmbed? That's what I did in #1845 until I rebased on this pr.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DaemonEmbed or a variant of it (DamonEmbeddedFile?) is the naming I'm planing to use for EmbedText that doesn't only produce embedded text files anymore.

Also this cmake does more code generation than just embedding files (like the buildinfo thing).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Details here:

include(DaemonPlatform)

################################################################################
Expand Down Expand Up @@ -927,39 +927,8 @@ if (BUILD_CLIENT)
Tests ${CLIENTTESTLIST}
)

# generate glsl include files
set(GLSL_SOURCE_DIR ${ENGINE_DIR}/renderer/glsl_source)
set(EMBED_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/embed_data)
file(MAKE_DIRECTORY ${EMBED_INCLUDE_DIR})

set(SHADERS_CPP_TEXT "// This file is auto-generated by CMakeLists.txt.\n")
string(APPEND SHADERS_CPP_TEXT "#include \"common/Common.h\"\n\n")
set(SHADERMAP_TEXT "")

foreach(res ${GLSLSOURCELIST})
get_filename_component(filename_no_ext ${res} NAME_WE)
set(outpath ${EMBED_INCLUDE_DIR}/${filename_no_ext}.glsl.h)

add_custom_command(
OUTPUT ${outpath}
COMMAND ${CMAKE_COMMAND} "-DINPUT_FILE=${res}" "-DOUTPUT_FILE=${outpath}"
"-DVARIABLE_NAME=${filename_no_ext}_glsl" -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/EmbedText.cmake
MAIN_DEPENDENCY ${res}
)

set_property(TARGET client-objects APPEND PROPERTY SOURCES ${outpath})

string(APPEND SHADERS_CPP_TEXT "#include \"../embed_data/${filename_no_ext}.glsl.h\"\n")
string(APPEND SHADERMAP_TEXT "\t{ \"${filename_no_ext}.glsl\", ")
string(APPEND SHADERMAP_TEXT "std::string(reinterpret_cast<const char *>( ${filename_no_ext}_glsl ), ")
string(APPEND SHADERMAP_TEXT "sizeof( ${filename_no_ext}_glsl )) },\n")
endforeach()

string(APPEND SHADERS_CPP_TEXT "\nextern const std::unordered_map<std::string, std::string> shadermap\n{\n")
string(APPEND SHADERS_CPP_TEXT "${SHADERMAP_TEXT}")
string(APPEND SHADERS_CPP_TEXT "};\n")

daemon_write_generated("shaders.cpp" "${SHADERS_CPP_TEXT}")
# Generate GLSL include files.
daemon_embed_files("EngineShaders" "${GLSL_EMBED_DIR}" "${GLSL_EMBED_LIST}" "TEXT" "client-objects")
endif()

if (BUILD_SERVER)
Expand Down
44 changes: 0 additions & 44 deletions cmake/DaemonBuildInfo.cmake
View file
Open in desktop

This file was deleted.

38 changes: 38 additions & 0 deletions cmake/DaemonFileEmbedder.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Converts a text file into a C-language char array definition.
# For use in CMake script mode (cmake -P).
# Required definitions on command line:
# INPUT_FILE, OUTPUT_FILE, FILE_FORMAT, VARIABLE_NAME

# Inspired by:
# https://stackoverflow.com/questions/11813271/embed-resources-eg-shader-code-images-into-executable-library-with-cmake/27206982#27206982

file(READ ${INPUT_FILE} contents HEX)

# Translate the file content.
if ("${FILE_FORMAT}" STREQUAL "TEXT")
# Strip \r for consistency.
string(REGEX REPLACE "(0d)?(..)" "0x\\2," contents "${contents}")
elseif("${FILE_FORMAT}" STREQUAL "BINARY")
string(REGEX REPLACE "(..)" "0x\\1," contents "${contents}")
else()
message(FATAL_ERROR "Unknown file format: ${FILE_FORMAT}")
endif()

# Add null terminator.
set(contents "${contents}0x00,")

# Split long lines.
string(REGEX REPLACE
"(0x..,0x..,0x..,0x..,0x..,0x..,0x..,0x..,0x..,0x..,0x..,0x..,0x..,0x..,0x..,)" "\\1\n"
contents "${contents}"
)

# A bit more of beautification.
string(REGEX REPLACE ",$" ",\n" contents "${contents}")

file(WRITE ${OUTPUT_FILE}
"constexpr unsigned char ${VARIABLE_NAME}[] =\n"
"{\n"
"${contents}"
"};\n"
)
2 changes: 1 addition & 1 deletion cmake/DaemonGame.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ option(BUILD_GAME_NATIVE_DLL "Build the shared library files, mostly useful for
option(BUILD_GAME_NATIVE_EXE "Build native executable, which might be used for better performances by server owners" OFF)

include(ExternalProject)
include(DaemonBuildInfo)
include(DaemonSourceGenerator)
include(DaemonPlatform)

# Do not report unused native compiler if native vms are not built.
Expand Down
151 changes: 151 additions & 0 deletions cmake/DaemonSourceGenerator.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
# Daemon BSD Source Code
# Copyright (c) 2025, Daemon Developers
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the <organization> nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

set(DAEMON_SOURCE_GENERATOR "${CMAKE_CURRENT_LIST_FILE}")
get_filename_component(current_list_dir "${CMAKE_CURRENT_LIST_FILE}" DIRECTORY)
set(DAEMON_FILE_EMBEDDER "${current_list_dir}/DaemonFileEmbedder.cmake")

set(DAEMON_GENERATED_SUBDIR "GeneratedSource")
set(DAEMON_GENERATED_DIR "${CMAKE_CURRENT_BINARY_DIR}/${DAEMON_GENERATED_SUBDIR}")

set(DAEMON_BUILDINFO_SUBDIR "DaemonBuildInfo")
set(DAEMON_EMBEDDED_SUBDIR "DaemonEmbeddedFiles")

set(DAEMON_BUILDINFO_DIR "${DAEMON_GENERATED_DIR}/${DAEMON_BUILDINFO_SUBDIR}")
set(DAEMON_EMBEDDED_DIR "${DAEMON_GENERATED_DIR}/${DAEMON_EMBEDDED_SUBDIR}")

file(MAKE_DIRECTORY "${DAEMON_GENERATED_DIR}")
include_directories("${DAEMON_GENERATED_DIR}")

file(MAKE_DIRECTORY "${DAEMON_BUILDINFO_DIR}")
file(MAKE_DIRECTORY "${DAEMON_EMBEDDED_DIR}")

set(DAEMON_GENERATED_HEADER "// Automatically generated, do not modify!\n")
set(DAEMON_GENERATED_CPP_EXT ".cpp")
set(DAEMON_GENERATED_H_EXT ".h")

set(BUILDINFOLIST)

foreach(kind CPP H)
set(DAEMON_BUILDINFO_${kind}_TEXT "${DAEMON_GENERATED_HEADER}")
endforeach()

macro(daemon_add_buildinfo type name value)
string(APPEND DAEMON_BUILDINFO_CPP_TEXT "const ${type} ${name}=${value};\n")
string(APPEND DAEMON_BUILDINFO_H_TEXT "extern const ${type} ${name};\n")
endmacro()

macro(daemon_write_buildinfo name)
foreach(kind CPP H)
set(buildinfo_file_path "${DAEMON_BUILDINFO_DIR}/${name}${DAEMON_GENERATED_${kind}_EXT}")

file(GENERATE OUTPUT "${buildinfo_file_path}" CONTENT "${DAEMON_BUILDINFO_${kind}_TEXT}")
list(APPEND BUILDINFOLIST "${buildinfo_file_path}")
endforeach()
endmacro()

macro(daemon_embed_files basename dir list format targetname)
set(embed_subdir "${DAEMON_EMBEDDED_SUBDIR}/${basename}")
set(embed_dir "${DAEMON_GENERATED_DIR}/${embed_subdir}")

file(MAKE_DIRECTORY "${embed_dir}")

foreach(kind CPP H)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The code in this loop is too confusing. I shouldn't have to sit here half an hour trying to decipher this code using multiple variable indirection to do something that probably could have been done by just setting a variable to a hard-coded string.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This kind of indirection saves me a lot of time when rewriting and re-rewriting across my various iterations , and it already saved me a lot of time. It only looks better to hardcode those strings when all the work was done by someone else, because then you don't know all the hundreds of different strings that have been used before being presented to the final state. Those indirections aren't only written to reduce the amount of lines to edit when redesigning things, but also to avoid the introduction of mistakes when editing the alternative expanded boiler plate.

And we know well who is doing all those iterations and who will suffer from editing the duplicated boiler plate once it is expanded:

git ls-files | grep -i cmake | while read f; do git blame -w -M -C -C --line-porcelain "$f" | grep -I '^author '; done | sort -f | uniq -ic | sort -n --reverse
   2135 author Thomas Debesse
   1286 author TimePath
    612 author slipher
    334 author Daniel Maloney
    243 author Corentin Wallez
    175 author Darren Salt
    161 author dolcetriade
    135 author VReaperV
    108 author Amanieu d'Antras
    107 author perturbed
     43 author Tsarevich Dmitry
     29 author Morel Bérenger
     12 author Mattia Basaglia
      7 author Tim
      3 author cmf028
      2 author Jesper B. Christensen
      1 author maek
      1 author Keziolio

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It only looks better to hardcode those strings when all the work was done by someone else,

Yes that's the point. Other people besides you should be able to read the code.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I meant: You don't know yet what's the cost of maintaining it is.

I can tell you it's easier this way, because I already maintained it.

Said other way: this is the solution I have chosen because I first tried the hardcoded way and it was painful and doing it this way made it easier. Lessons got learned, I share you the result, I save you time.

set(embed_${kind}_basename "${basename}${DAEMON_GENERATED_${kind}_EXT}")
set(embed_${kind}_src_file "${DAEMON_EMBEDDED_DIR}/${embed_${kind}_basename}")
set(embed_${kind}_file "${DAEMON_EMBEDDED_SUBDIR}/${embed_${kind}_basename}")
set(embed_${kind}_text "${DAEMON_GENERATED_HEADER}")
set_property(SOURCE "${embed_${kind}_src_file}" APPEND PROPERTY OBJECT_DEPENDS "${DAEMON_SOURCE_GENERATOR}")
set_property(TARGET "${targetname}" APPEND PROPERTY SOURCES "${embed_${kind}_src_file}")
endforeach()

string(APPEND embed_CPP_text
"#include \"${embed_H_file}\"\n"
"\n"
"namespace ${basename} {\n"
)

string(APPEND embed_H_text
"#include \"common/Common.h\"\n"
"\n"
"namespace ${basename} {\n"
)

set(embed_map_text "")

foreach(filename ${list})
string(REGEX REPLACE "[^A-Za-z0-9]" "_" filename_symbol "${filename}")

set(inpath "${dir}/${filename}")
set(outpath "${embed_dir}/${filename_symbol}${DAEMON_GENERATED_H_EXT}")

add_custom_command(
OUTPUT "${outpath}"
COMMAND ${CMAKE_COMMAND}
"-DINPUT_FILE=${inpath}"
"-DOUTPUT_FILE=${outpath}"
"-DFILE_FORMAT=${format}"
"-DVARIABLE_NAME=${filename_symbol}"
-P "${DAEMON_FILE_EMBEDDER}"
MAIN_DEPENDENCY ${inpath}
DEPENDS
"${DAEMON_FILE_EMBEDDER}"
"${DAEMON_SOURCE_GENERATOR}"
)

set_property(TARGET "${targetname}" APPEND PROPERTY SOURCES "${outpath}")

string(APPEND embed_CPP_text
"#include \"${basename}/${filename_symbol}.h\"\n"
)

string(APPEND embed_H_text
"extern const unsigned char ${filename_symbol}[];\n"
)

string(APPEND embed_map_text
"\t{ \"${filename}\", { ${filename_symbol}, sizeof( ${filename_symbol}) - 1 } },\n"
)
endforeach()

string(APPEND embed_CPP_text
"\n"
"const embeddedFileMap_t FileMap\n{\n"
"${embed_map_text}"
"};\n"
"}"
)

string(APPEND embed_H_text
"extern const embeddedFileMap_t FileMap;\n"
"};\n"
)

foreach(kind CPP H)
set(embed_file "${DAEMON_GENERATED_DIR}/${embed_${kind}_file}")
file(GENERATE OUTPUT "${embed_file}" CONTENT "${embed_${kind}_text}")
endforeach()
endmacro()
8 changes: 0 additions & 8 deletions cmake/EmbedText.cmake
View file
Open in desktop

This file was deleted.

Loading