From adf138482e849e7ed2bed8ec1489587751d5e694 Mon Sep 17 00:00:00 2001 From: Lee Kerley Date: Fri, 13 Dec 2024 14:22:27 -0800 Subject: [PATCH 01/22] Named Value and template support added to standard library. Allowing for signficant reduction in the size of the source for the standard library. Build time options to exapnd these features to retain built standard library partity enabled by default. Runtime implementation is available, but this requires that the installed standard library format be non-compliant with the spec. --- .github/workflows/main.yml | 11 +- CMakeLists.txt | 164 +- cmake/macros/Public.cmake | 119 + javascript/MaterialXTest/xmlIo.spec.js | 2 +- libraries/CMakeLists.txt | 60 +- libraries/stdlib/stdlib_defs.mtlx | 406 +--- libraries/stdlib/stdlib_ng.mtlx | 1968 +++-------------- python/MaterialXTest/main.py | 3 +- source/JsMaterialX/CMakeLists.txt | 4 +- source/MaterialXBuildTools/CMakeLists.txt | 19 + .../buildLibrary/CMakeLists.txt | 10 + .../MaterialXBuildTools/buildLibrary/Main.cpp | 201 ++ source/MaterialXCore/CMakeLists.txt | 6 + source/MaterialXCore/Element.cpp | 30 + source/MaterialXCore/Element.h | 5 +- source/MaterialXFormat/XmlIo.cpp | 82 +- source/MaterialXFormat/XmlIo.h | 15 +- 17 files changed, 957 insertions(+), 2148 deletions(-) create mode 100644 cmake/macros/Public.cmake create mode 100644 source/MaterialXBuildTools/CMakeLists.txt create mode 100644 source/MaterialXBuildTools/buildLibrary/CMakeLists.txt create mode 100644 source/MaterialXBuildTools/buildLibrary/Main.cpp diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 16f3d5919b..73281b6035 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,12 +31,21 @@ jobs: compiler_version: "14" python: 3.9 - - name: Linux_GCC_14_Python311 + - name: Linux_GCC_14_Python311_expand_lib os: ubuntu-24.04 compiler: gcc compiler_version: "14" python: 3.11 test_render: ON + cmake_config: -DMATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS=ON -DMATERIALX_BUILD_BAKE_NAMED_VALUES=ON + + - name: Linux_GCC_14_Python311_unexpanded_lib + os: ubuntu-24.04 + compiler: gcc + compiler_version: "14" + python: 3.11 + test_render: ON + cmake_config: -DMATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS=OFF -DMATERIALX_BUILD_BAKE_NAMED_VALUES=OFF - name: Linux_GCC_CoverageAnalysis os: ubuntu-24.04 diff --git a/CMakeLists.txt b/CMakeLists.txt index c9836e1225..de1131f0eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,8 @@ option(MATERIALX_BUILD_BENCHMARK_TESTS "Build benchmark tests." OFF) option(MATERIALX_BUILD_SHARED_LIBS "Build MaterialX libraries as shared rather than static." OFF) option(MATERIALX_BUILD_DATA_LIBRARY "Build generated products from the MaterialX data library." OFF) +option(MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS "Process the data library files at build time to expand any template elements." ON) +option(MATERIALX_BUILD_BAKE_NAMED_VALUES "Process the data library files at build time to bake out the named values." ON) option(MATERIALX_BUILD_MONOLITHIC "Build a single monolithic MaterialX library." OFF) option(MATERIALX_BUILD_USE_CCACHE "Enable the use of ccache to speed up build time, if present." ON) option(MATERIALX_PYTHON_LTO "Enable link-time optimizations for MaterialX Python." ON) @@ -68,6 +70,10 @@ if (MATERIALX_BUILD_IOS) set(CMAKE_SYSTEM_NAME iOS) endif() +list(APPEND CMAKE_MODULE_PATH + ${PROJECT_SOURCE_DIR}/cmake/macros) +include(Public) + # Apple ecosystem cross-compilation # https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#cross-compiling-for-ios-tvos-visionos-or-watchos set(MATERIALX_BUILD_APPLE_EMBEDDED OFF) @@ -306,124 +312,6 @@ else() endif() endif() -# Shared functions -function(assign_source_group prefix) - foreach(_source IN ITEMS ${ARGN}) - if(IS_ABSOLUTE "${_source}") - file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}") - else() - set(_source_rel "${_source}") - endif() - get_filename_component(_source_path "${_source_rel}" PATH) - string(REPLACE "/" "\\" _source_path_msvc "${_source_path}") - source_group("${prefix}\\${_source_path_msvc}" FILES "${_source}") - endforeach() -endfunction(assign_source_group) - -function(mx_add_library MATERIALX_MODULE_NAME) - set(options ADD_OBJECTIVE_C_CODE) - set(oneValueArgs EXPORT_DEFINE) - set(multiValueArgs - SOURCE_FILES - HEADER_FILES - INLINED_FILES - MTLX_MODULES) - cmake_parse_arguments(args - "${options}" - "${oneValueArgs}" - "${multiValueArgs}" - ${ARGN}) - - if (APPLE AND args_ADD_OBJECTIVE_C_CODE) - file(GLOB_RECURSE materialx_source_oc "${CMAKE_CURRENT_SOURCE_DIR}/*.m*") - set_source_files_properties(${materialx_source_oc} PROPERTIES - COMPILE_FLAGS "-x objective-c++") - set(args_SOURCE_FILES ${args_SOURCE_FILES} ${materialx_source_oc}) - endif() - - assign_source_group("Source Files" ${args_SOURCE_FILES}) - assign_source_group("Source Files" ${args_INLINED_FILES}) - assign_source_group("Header Files" ${args_HEADER_FILES}) - - if (NOT MATERIALX_BUILD_MONOLITHIC) - set(TARGET_NAME ${MATERIALX_MODULE_NAME}) - add_library(${TARGET_NAME}) - - # Create version resource - if(MATERIALX_BUILD_SHARED_LIBS AND MSVC) - configure_file(${PROJECT_SOURCE_DIR}/cmake/modules/MaterialXVersion.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc) - target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/version.rc) - endif() - - target_link_libraries(${TARGET_NAME} - PUBLIC - ${args_MTLX_MODULES} - ${CMAKE_DL_LIBS}) - - target_include_directories(${TARGET_NAME} - PUBLIC - $ - $ - PRIVATE - ${EXTERNAL_INCLUDE_DIRS}) - - set_target_properties( - ${TARGET_NAME} PROPERTIES - OUTPUT_NAME ${MATERIALX_MODULE_NAME}${MATERIALX_LIBNAME_SUFFIX} - COMPILE_FLAGS "${EXTERNAL_COMPILE_FLAGS}" - LINK_FLAGS "${EXTERNAL_LINK_FLAGS}" - INSTALL_RPATH "${MATERIALX_SAME_DIR_RPATH}" - VERSION "${MATERIALX_LIBRARY_VERSION}" - SOVERSION "${MATERIALX_MAJOR_VERSION}") - else() - set(TARGET_NAME ${MATERIALX_MONOLITHIC_TARGET}) - add_library(${MATERIALX_MODULE_NAME} ALIAS ${MATERIALX_MONOLITHIC_TARGET}) - - # Store the aliased MaterialX modules name to create CMake export aliases later. - set_property(GLOBAL APPEND PROPERTY MATERIALX_MODULES ${MATERIALX_MODULE_NAME}) - endif() - - set_target_properties(${TARGET_NAME} PROPERTIES CXX_VISIBILITY_PRESET hidden) - set_target_properties(${TARGET_NAME} PROPERTIES CMAKE_VISIBILITY_INLINES_HIDDEN 1) - - target_sources(${TARGET_NAME} - PRIVATE - ${args_SOURCE_FILES} - PUBLIC - FILE_SET - mxHeaders - TYPE - HEADERS - BASE_DIRS - ${CMAKE_CURRENT_SOURCE_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/.. - FILES - ${args_HEADER_FILES} - ${args_INLINED_FILES}) - - target_include_directories(${TARGET_NAME} PUBLIC - $) - - target_compile_definitions(${TARGET_NAME} PRIVATE "-D${args_EXPORT_DEFINE}") - - if(NOT SKBUILD) - if(NOT MATERIALX_BUILD_MONOLITHIC) - install(TARGETS ${MATERIALX_MODULE_NAME} - EXPORT MaterialX - ARCHIVE DESTINATION ${MATERIALX_INSTALL_LIB_PATH} - LIBRARY DESTINATION ${MATERIALX_INSTALL_LIB_PATH} - RUNTIME DESTINATION bin - FILE_SET mxHeaders DESTINATION ${MATERIALX_INSTALL_INCLUDE_PATH}) - endif() - - install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/${MATERIALX_MODULE_NAME}.pdb" - DESTINATION "${MATERIALX_INSTALL_LIB_PATH}/" OPTIONAL) - endif() - - # Pass TARGET_NAME back to call site, so the caller can modify the build target. - set(TARGET_NAME ${TARGET_NAME} PARENT_SCOPE) -endfunction() - # Propagate shared library setting to NanoGUI if(MATERIALX_BUILD_SHARED_LIBS) set(BUILD_SHARED_LIBS "ON") @@ -431,15 +319,55 @@ else() set(BUILD_SHARED_LIBS "OFF") endif() +# If we're baking the named "Value:" entries - then we have to also expand any +# template elements +if (MATERIALX_BUILD_BAKE_NAMED_VALUES) + set(MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS ON) +endif() + +if (MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS OR MATERIALX_BUILD_BAKE_NAMED_VALUES) + set(MATERIALX_BUILD_DATA_LIBRARY ON) +endif() + # Build a monolithic target - needs to be added before the other build targets that may be included. if (MATERIALX_BUILD_MONOLITHIC) set(MATERIALX_MONOLITHIC_TARGET MaterialX) add_subdirectory(source) endif() +message("XXX : MATERIALX_BUILD_BAKE_NAMED_VALUES : ${MATERIALX_BUILD_BAKE_NAMED_VALUES}") +message("XXX : MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS : ${MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS}") +message("XXX : MATERIALX_BUILD_DATA_LIBRARY : ${MATERIALX_BUILD_DATA_LIBRARY}") + # Add core subdirectories add_subdirectory(source/MaterialXCore) add_subdirectory(source/MaterialXFormat) +if (MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS OR MATERIALX_BUILD_BAKE_NAMED_VALUES) + if (CMAKE_CROSSCOMPILING) + set(_MaterialXBuildLibrary "${CMAKE_BINARY_DIR}/BUILD_TOOLS/MaterialXBuildLibrary") + + add_custom_command( + OUTPUT ${_MaterialXBuildLibrary} + COMMAND ${CMAKE_COMMAND} + -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DCMAKE_RUNTIME_OUTPUT_DIRECTORY:PATH=${CMAKE_BINARY_DIR}/BUILD_TOOLS + -DMATERIALX_MAJOR_VERSION=${MATERIALX_MAJOR_VERSION} + -DMATERIALX_MINOR_VERSION=${MATERIALX_MINOR_VERSION} + -DMATERIALX_BUILD_VERSION=${MATERIALX_BUILD_VERSION} + -DMATERIALX_NAMESPACE=${MATERIALX_NAMESPACE} + -DMATERIALX_BUILD_BAKE_NAMED_VALUES=ON + -B"BUILD_TOOLS" + -H"${CMAKE_SOURCE_DIR}/source/MaterialXBuildTools" + COMMAND ${CMAKE_COMMAND} --build BUILD_TOOLS + DEPENDS MaterialXCore MaterialXFormat + ) + add_custom_target(MaterialXBuildLibrary ALL + DEPENDS ${_MaterialXBuildLibrary}) + else() + set(_MaterialXBuildLibrary MaterialXBuildLibrary) + add_subdirectory(source/MaterialXBuildTools/buildLibrary) + endif() +endif() # Add shader generation subdirectories add_subdirectory(source/MaterialXGenShader) diff --git a/cmake/macros/Public.cmake b/cmake/macros/Public.cmake new file mode 100644 index 0000000000..e21c2f7e2a --- /dev/null +++ b/cmake/macros/Public.cmake @@ -0,0 +1,119 @@ +# Shared functions need to be extracted to allow them to be shared with the subproject +# in source/MaterialXBuildTools + +function(assign_source_group prefix) + foreach(_source IN ITEMS ${ARGN}) + if(IS_ABSOLUTE "${_source}") + file(RELATIVE_PATH _source_rel "${CMAKE_CURRENT_SOURCE_DIR}" "${_source}") + else() + set(_source_rel "${_source}") + endif() + get_filename_component(_source_path "${_source_rel}" PATH) + string(REPLACE "/" "\\" _source_path_msvc "${_source_path}") + source_group("${prefix}\\${_source_path_msvc}" FILES "${_source}") + endforeach() +endfunction(assign_source_group) + +function(mx_add_library MATERIALX_MODULE_NAME) + set(options ADD_OBJECTIVE_C_CODE) + set(oneValueArgs EXPORT_DEFINE) + set(multiValueArgs + SOURCE_FILES + HEADER_FILES + INLINED_FILES + MTLX_MODULES) + cmake_parse_arguments(args + "${options}" + "${oneValueArgs}" + "${multiValueArgs}" + ${ARGN}) + + if (APPLE AND args_ADD_OBJECTIVE_C_CODE) + file(GLOB_RECURSE materialx_source_oc "${CMAKE_CURRENT_SOURCE_DIR}/*.m*") + set_source_files_properties(${materialx_source_oc} PROPERTIES + COMPILE_FLAGS "-x objective-c++") + set(args_SOURCE_FILES ${args_SOURCE_FILES} ${materialx_source_oc}) + endif() + + assign_source_group("Source Files" ${args_SOURCE_FILES}) + assign_source_group("Source Files" ${args_INLINED_FILES}) + assign_source_group("Header Files" ${args_HEADER_FILES}) + + if (NOT MATERIALX_BUILD_MONOLITHIC) + set(TARGET_NAME ${MATERIALX_MODULE_NAME}) + add_library(${TARGET_NAME}) + + # Create version resource + if(MATERIALX_BUILD_SHARED_LIBS AND MSVC) + configure_file(${PROJECT_SOURCE_DIR}/cmake/modules/MaterialXVersion.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc) + target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/version.rc) + endif() + + target_link_libraries(${TARGET_NAME} + PUBLIC + ${args_MTLX_MODULES} + ${CMAKE_DL_LIBS}) + + target_include_directories(${TARGET_NAME} + PUBLIC + $ + $ + PRIVATE + ${EXTERNAL_INCLUDE_DIRS}) + + set_target_properties( + ${TARGET_NAME} PROPERTIES + OUTPUT_NAME ${MATERIALX_MODULE_NAME}${MATERIALX_LIBNAME_SUFFIX} + COMPILE_FLAGS "${EXTERNAL_COMPILE_FLAGS}" + LINK_FLAGS "${EXTERNAL_LINK_FLAGS}" + INSTALL_RPATH "${MATERIALX_SAME_DIR_RPATH}" + VERSION "${MATERIALX_LIBRARY_VERSION}" + SOVERSION "${MATERIALX_MAJOR_VERSION}") + else() + set(TARGET_NAME ${MATERIALX_MONOLITHIC_TARGET}) + add_library(${MATERIALX_MODULE_NAME} ALIAS ${MATERIALX_MONOLITHIC_TARGET}) + + # Store the aliased MaterialX modules name to create CMake export aliases later. + set_property(GLOBAL APPEND PROPERTY MATERIALX_MODULES ${MATERIALX_MODULE_NAME}) + endif() + + set_target_properties(${TARGET_NAME} PROPERTIES CXX_VISIBILITY_PRESET hidden) + set_target_properties(${TARGET_NAME} PROPERTIES CMAKE_VISIBILITY_INLINES_HIDDEN 1) + + target_sources(${TARGET_NAME} + PRIVATE + ${args_SOURCE_FILES} + PUBLIC + FILE_SET + mxHeaders + TYPE + HEADERS + BASE_DIRS + ${CMAKE_CURRENT_SOURCE_DIR}/.. + ${CMAKE_CURRENT_BINARY_DIR}/.. + FILES + ${args_HEADER_FILES} + ${args_INLINED_FILES}) + + target_include_directories(${TARGET_NAME} PUBLIC + $) + + target_compile_definitions(${TARGET_NAME} PRIVATE "-D${args_EXPORT_DEFINE}") + + if(NOT SKBUILD) + if(NOT MATERIALX_BUILD_MONOLITHIC) + install(TARGETS ${MATERIALX_MODULE_NAME} + EXPORT MaterialX + ARCHIVE DESTINATION ${MATERIALX_INSTALL_LIB_PATH} + LIBRARY DESTINATION ${MATERIALX_INSTALL_LIB_PATH} + RUNTIME DESTINATION bin + FILE_SET mxHeaders DESTINATION ${MATERIALX_INSTALL_INCLUDE_PATH}) + endif() + + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/${MATERIALX_MODULE_NAME}.pdb" + DESTINATION "${MATERIALX_INSTALL_LIB_PATH}/" OPTIONAL) + endif() + + # Pass TARGET_NAME back to call site, so the caller can modify the build target. + set(TARGET_NAME ${TARGET_NAME} PARENT_SCOPE) +endfunction() diff --git a/javascript/MaterialXTest/xmlIo.spec.js b/javascript/MaterialXTest/xmlIo.spec.js index 670b0df43d..591459d472 100644 --- a/javascript/MaterialXTest/xmlIo.spec.js +++ b/javascript/MaterialXTest/xmlIo.spec.js @@ -10,7 +10,7 @@ describe('XmlIo', () => // These should be relative to cwd const includeTestPath = 'data/includes'; - const libraryPath = '../../libraries/stdlib'; + const libraryPath = '../build/libraries/DataLibraryBuild/stdlib'; const examplesPath = '../../resources/Materials/Examples'; // TODO: Is there a better way to get these filenames than hardcoding them here? // The C++ tests load all files in the given directories. This would work in Node, but not in the browser. diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index cc8139aef3..af2decab62 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,3 +1,4 @@ + if(MATERIALX_BUILD_DATA_LIBRARY) # Build generated products from the MaterialX data library. # Initially, this step is a simple copy across folders, but our intent @@ -5,17 +6,54 @@ if(MATERIALX_BUILD_DATA_LIBRARY) set(DATA_LIBRARY_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/DataLibraryBuild) - file(GLOB_RECURSE MATERIALX_DATA_LIBRARY_SOURCE_FILES - RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - LIST_DIRECTORIES false - *.mtlx - *.md - *.glsl - *.osl - *.h - *.metal) - - foreach(SOURCE_FILE IN LISTS MATERIALX_DATA_LIBRARY_SOURCE_FILES) + file(GLOB_RECURSE MATERIALX_DATA_LIBRARY_MTLX_SOURCE_FILES + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + LIST_DIRECTORIES false + *.mtlx) + + file(GLOB_RECURSE MATERIALX_DATA_LIBRARY_SHADER_SOURCE_FILES + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + LIST_DIRECTORIES false + *.md + *.glsl + *.osl + *.h + *.metal) + + if (MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS OR MATERIALX_BUILD_BAKE_NAMED_VALUES) + foreach(SOURCE_FILE IN LISTS MATERIALX_DATA_LIBRARY_MTLX_SOURCE_FILES) + set(DEST_FILEPATH ${DATA_LIBRARY_BUILD_DIR}/${SOURCE_FILE}) + list(APPEND MATERIALX_DATA_LIBRARY_MTLX_BUILD_FILES ${DEST_FILEPATH}) + endforeach() + + if (MATERIALX_BUILD_EXPAND_TEMPLATE_ELEMS) + set(EXPAND_TEMPLATE_ELEMS_ARG "--expandTemplateElems") + endif() + if (MATERIALX_BUILD_BAKE_NAMED_VALUES) + set(BAKE_NAMED_VALUES_ARG "--bakeNamedValues") + endif() + + add_custom_command( + OUTPUT ${MATERIALX_DATA_LIBRARY_MTLX_BUILD_FILES} + COMMAND ${CMAKE_COMMAND} -E make_directory ${DATA_LIBRARY_BUILD_DIR} + COMMAND ${_MaterialXBuildLibrary} --sourceLibraryRoot ${CMAKE_CURRENT_SOURCE_DIR} --destLibraryRoot ${DATA_LIBRARY_BUILD_DIR} ${EXPAND_TEMPLATE_ELEMS_ARG} ${BAKE_NAMED_VALUES_ARG} + DEPENDS ${MATERIALX_DATA_LIBRARY_MTLX_SOURCE_FILES} MaterialXBuildLibrary + ) + + list(APPEND MATERIALX_DATA_LIBRARY_BUILD_FILES ${MATERIALX_DATA_LIBRARY_MTLX_BUILD_FILES}) + else() + foreach(SOURCE_FILE IN LISTS MATERIALX_DATA_LIBRARY_MTLX_SOURCE_FILES) + set(SOURCE_FILEPATH ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}) + set(DEST_FILEPATH ${DATA_LIBRARY_BUILD_DIR}/${SOURCE_FILE}) + add_custom_command( + OUTPUT ${DEST_FILEPATH} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SOURCE_FILEPATH} ${DEST_FILEPATH} + DEPENDS ${SOURCE_FILEPATH}) + list(APPEND MATERIALX_DATA_LIBRARY_BUILD_FILES ${DEST_FILEPATH}) + endforeach() + endif() + + foreach(SOURCE_FILE IN LISTS MATERIALX_DATA_LIBRARY_SHADER_SOURCE_FILES) set(SOURCE_FILEPATH ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}) set(DEST_FILEPATH ${DATA_LIBRARY_BUILD_DIR}/${SOURCE_FILE}) add_custom_command( diff --git a/libraries/stdlib/stdlib_defs.mtlx b/libraries/stdlib/stdlib_defs.mtlx index c1eda8047d..959efaef1f 100644 --- a/libraries/stdlib/stdlib_defs.mtlx +++ b/libraries/stdlib/stdlib_defs.mtlx @@ -12,15 +12,15 @@ - - - - - - - - - + + + + + + + + + @@ -1501,242 +1501,94 @@ Node: Add "in2" value/stream to the incoming float/integer/color/vector/matrix. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/python/MaterialXTest/main.py b/python/MaterialXTest/main.py index 7f878b3d28..a2ed00ad5e 100644 --- a/python/MaterialXTest/main.py +++ b/python/MaterialXTest/main.py @@ -25,8 +25,9 @@ [1.0, 2.0, 3.0], ['one', 'two', 'three']) +_defaultSearchPath = mx.getDefaultDataSearchPath().asString() _fileDir = os.path.dirname(os.path.abspath(__file__)) -_libraryDir = os.path.join(_fileDir, '../../libraries/stdlib/') +_libraryDir = os.path.join(_defaultSearchPath, 'libraries/stdlib/') _exampleDir = os.path.join(_fileDir, '../../resources/Materials/Examples/') _searchPath = _libraryDir + mx.PATH_LIST_SEPARATOR + _exampleDir diff --git a/source/JsMaterialX/CMakeLists.txt b/source/JsMaterialX/CMakeLists.txt index aaadafda74..242dcdd559 100644 --- a/source/JsMaterialX/CMakeLists.txt +++ b/source/JsMaterialX/CMakeLists.txt @@ -87,7 +87,7 @@ else() endif() -set(JS_LINK_FLAGS_GENSHADER "${JS_LINK_FLAGS_CORE} --preload-file ${PROJECT_SOURCE_DIR}/libraries@libraries ") +set(JS_LINK_FLAGS_GENSHADER "${JS_LINK_FLAGS_CORE} --preload-file ${PROJECT_BINARY_DIR}/libraries/DataLibraryBuild@libraries ") add_executable(JsMaterialXCore MaterialXLib.cpp ${CORE_DEPS} @@ -132,6 +132,8 @@ set_target_properties(JsMaterialXGenShader INSTALL_RPATH "${MATERIALX_UP_TWO_RPATH}" SOVERSION "${MATERIALX_MAJOR_VERSION}") +add_dependencies(JsMaterialXGenShader MaterialXBuildData) + target_link_libraries(JsMaterialXCore PUBLIC MaterialXCore PUBLIC MaterialXFormat diff --git a/source/MaterialXBuildTools/CMakeLists.txt b/source/MaterialXBuildTools/CMakeLists.txt new file mode 100644 index 0000000000..dbab900ece --- /dev/null +++ b/source/MaterialXBuildTools/CMakeLists.txt @@ -0,0 +1,19 @@ +# This subproject is necessary to support building the build tools on the host machine +# where they may be used when cross compiling for another target + +cmake_minimum_required(VERSION 3.26) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) +set(CMAKE_MACOSX_RPATH ON) + +project(MaterialXBuildTools) + +list(APPEND CMAKE_MODULE_PATH + ${PROJECT_SOURCE_DIR}/../../cmake/macros) + +include(Public) + +add_subdirectory(../MaterialXCore MaterialXCore) +add_subdirectory(../MaterialXFormat MaterialXFormat) + +add_subdirectory(buildLibrary) diff --git a/source/MaterialXBuildTools/buildLibrary/CMakeLists.txt b/source/MaterialXBuildTools/buildLibrary/CMakeLists.txt new file mode 100644 index 0000000000..bc7183aa3c --- /dev/null +++ b/source/MaterialXBuildTools/buildLibrary/CMakeLists.txt @@ -0,0 +1,10 @@ +set(TARGET_NAME MaterialXBuildLibrary) + +file(GLOB materialx_source "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") +file(GLOB materialx_headers "${CMAKE_CURRENT_SOURCE_DIR}/*.h*") + +add_executable(${TARGET_NAME} ${materialx_source} ${materialx_headers}) + +target_link_libraries(${TARGET_NAME} PRIVATE + MaterialXFormat + MaterialXCore) diff --git a/source/MaterialXBuildTools/buildLibrary/Main.cpp b/source/MaterialXBuildTools/buildLibrary/Main.cpp new file mode 100644 index 0000000000..77bd9f0f49 --- /dev/null +++ b/source/MaterialXBuildTools/buildLibrary/Main.cpp @@ -0,0 +1,201 @@ +// +// Copyright Contributors to the MaterialX Project +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include +#include + +namespace mx = MaterialX; + +#include + +const std::string cmdLineOptions = + " Options: \n" + " --sourceLibraryRoot [FILEPATH] Directory containing the source data library files.\n" + " --destLibraryRoot [FILEPATH] Directory to write the modified data library files.\n" + " --help Display the complete list of command-line options\n"; + +template void parseToken(std::string token, std::string type, T& res) +{ + if (token.empty()) + { + return; + } + + mx::ValuePtr value = mx::Value::createValueFromStrings(token, type); + if (!value) + { + std::cout << "Unable to parse token " << token << " as type " << type << std::endl; + return; + } + + res = value->asA(); +} + +mx::FilePathVec getMaterialXFiles(const mx::FilePath& libraryRoot) +{ + mx::FilePathVec result; + + auto subDirs = libraryRoot.getSubDirectories(); + for (const auto& subDir : subDirs) + { + for (const auto& file : subDir.getFilesInDirectory("mtlx")) + { + auto absFile = subDir / file; + std::string absFileStr = absFile.asString(); + std::string relFileStr = absFileStr.substr(libraryRoot.asString().size()+1); + result.emplace_back(mx::FilePath(relFileStr)); + } + } + return result; +} + +void replaceNamedValues(mx::DocumentPtr doc, mx::ConstDocumentPtr stdlib) +{ + const std::string typeValuePrefix = "Value:"; + + // replace named value "Value:" strings with concrete values + for (auto elem : doc->traverseTree()) + { + auto port = elem->asA(); + if (!port) + { + continue; + } + + if (!port->hasValueString()) + { + continue; + } + + auto valueStr = port->getValueString(); + if (mx::stringStartsWith(valueStr, typeValuePrefix)) + { + + auto typeDef = stdlib->getTypeDef(port->getType()); + if (!typeDef) + { + throw mx::Exception("Unable to find typeDef '"+port->getType()+"'"); + } + + auto valueNameStr = valueStr.substr(typeValuePrefix.size()); + if (!typeDef->hasAttribute(valueNameStr)) + { + throw mx::Exception("Unable to find named value '"+valueNameStr+"' for type '"+typeDef->getName()+"'"); + } + + port->setValueString(typeDef->getAttribute(valueNameStr)); + } + } +} + +int main(int argc, char* const argv[]) +{ + std::vector tokens; + for (int i = 1; i < argc; i++) + { + tokens.emplace_back(argv[i]); + } + + std::string sourceLibraryRootStr = ""; + std::string destLibraryRootStr = ""; + bool bakeNamedValues = false; + bool expandTemplateElems = false; + + for (size_t i = 0; i < tokens.size(); i++) + { + const std::string& token = tokens[i]; + const std::string& nextToken = i + 1 < tokens.size() ? tokens[i + 1] : mx::EMPTY_STRING; + + if (token == "--sourceLibraryRoot") + { + sourceLibraryRootStr = nextToken; + } + else if (token == "--destLibraryRoot") + { + destLibraryRootStr = nextToken; + } + else if (token == "--bakeNamedValues") + { + bakeNamedValues = true; + continue; + } + else if (token == "--expandTemplateElems") + { + expandTemplateElems = true; + continue; + } + else if (token == "--help") + { + std::cout << " MaterialXBuildLibrary version " << mx::getVersionString() << std::endl; + std::cout << cmdLineOptions << std::endl; + return 0; + } + else + { + std::cout << "Unrecognized command-line option: " << token << std::endl; + std::cout << "Launch the viewer with '--help' for a complete list of supported options." << std::endl; + continue; + } + + if (nextToken.empty()) + { + std::cout << "Expected another token following command-line option: " << token << std::endl; + } + else + { + i++; + } + } + + mx::FilePath sourceLibraryRoot = mx::FilePath(sourceLibraryRootStr); + mx::FilePath destLibrarayRoot = mx::FilePath(destLibraryRootStr); + + if (!sourceLibraryRoot.isDirectory()) + { + std::cerr << "Source Library Root is not a directory" << std::endl; + return 1; + } + + if (!destLibrarayRoot.isDirectory()) + { + std::cerr << "Destination Library Root is not a directory" << std::endl; + return 1; + } + + mx::DocumentPtr stdlib = mx::createDocument(); + mx::loadLibraries({}, mx::FileSearchPath(sourceLibraryRootStr), stdlib); + + mx::FilePathVec mtlxFiles = getMaterialXFiles(sourceLibraryRoot); + + mx::XmlReadOptions readOptions; + readOptions.readComments = true; + readOptions.readNewlines = true; + readOptions.expandTemplateElems = expandTemplateElems; + + mx::XmlWriteOptions writeOptions; + writeOptions.createDirectories = true; + + for (const auto& mtlxFile : mtlxFiles) + { + mx::DocumentPtr doc = mx::createDocument(); + + mx::FilePath sourceFile = sourceLibraryRoot / mtlxFile; + mx::FilePath destFile = destLibrarayRoot / mtlxFile; + + mx::readFromXmlFile(doc, sourceFile, mx::FileSearchPath(), &readOptions); + + if (bakeNamedValues) + { + replaceNamedValues(doc, stdlib); + } + + mx::writeToXmlFile(doc, destFile, &writeOptions); + } + + return 0; +} diff --git a/source/MaterialXCore/CMakeLists.txt b/source/MaterialXCore/CMakeLists.txt index c3c845afda..53a3b4c7b0 100644 --- a/source/MaterialXCore/CMakeLists.txt +++ b/source/MaterialXCore/CMakeLists.txt @@ -15,3 +15,9 @@ mx_add_library(MaterialXCore target_include_directories(${TARGET_NAME} PUBLIC $) + +# This define controls if the named "Value:" are dynamically evaluated at runtime +target_compile_definitions(${TARGET_NAME} + PRIVATE + MATERIALX_BUILD_BAKE_NAMED_VALUES=$ +) diff --git a/source/MaterialXCore/Element.cpp b/source/MaterialXCore/Element.cpp index e36a6fcbdf..595aa0787d 100644 --- a/source/MaterialXCore/Element.cpp +++ b/source/MaterialXCore/Element.cpp @@ -644,6 +644,36 @@ TypeDefPtr TypedElement::getTypeDef() const // // ValueElement methods // +/// Get the value string of a element. +const string ValueElement::getValueString() const +{ +#if MATERIALX_BUILD_BAKE_NAMED_VALUES + return getAttribute(VALUE_ATTRIBUTE); +#else + const string typeValuePrefix = "Value:"; + + auto valueStr = getAttribute(VALUE_ATTRIBUTE); + if (!stringStartsWith(valueStr, typeValuePrefix)) + { + return valueStr; + } + + auto typeDef = getTypeDef(); + if (!typeDef) + { + throw Exception("Unable to find typeDef '"+getType()+"'"); + } + + auto valueNameStr = valueStr.substr(typeValuePrefix.size()); + if (!typeDef->hasAttribute(valueNameStr)) + { + throw Exception("Unable to find named value '"+valueNameStr+"' for type '"+typeDef->getName()+"'"); + } + + return typeDef->getAttribute(valueNameStr); +#endif +} + string ValueElement::getResolvedValueString(StringResolverPtr resolver) const { diff --git a/source/MaterialXCore/Element.h b/source/MaterialXCore/Element.h index 89f5c87152..7c7125bf0c 100644 --- a/source/MaterialXCore/Element.h +++ b/source/MaterialXCore/Element.h @@ -954,10 +954,7 @@ class MX_CORE_API ValueElement : public TypedElement } /// Get the value string of a element. - const string& getValueString() const - { - return getAttribute(VALUE_ATTRIBUTE); - } + const string getValueString() const; /// Return the resolved value string of an element, applying any string /// substitutions that are defined at the element's scope. diff --git a/source/MaterialXFormat/XmlIo.cpp b/source/MaterialXFormat/XmlIo.cpp index 83e4721dc1..5a20f9a896 100644 --- a/source/MaterialXFormat/XmlIo.cpp +++ b/source/MaterialXFormat/XmlIo.cpp @@ -167,6 +167,70 @@ void elementFromXml(const xml_node& xmlNode, ElementPtr elem, const XmlReadOptio } } +void recursivelyReplaceStrings(ElementPtr elem, const StringMap& strReplaceMapping) +{ + for (const auto& attrName : elem->getAttributeNames()) + { + auto attrValue = elem->getAttribute(attrName); + auto newAttrValue = replaceSubstrings(attrValue, strReplaceMapping); + elem->setAttribute(attrName, newAttrValue); + } + + for (auto childElem : elem->getChildren()) + { + recursivelyReplaceStrings(childElem, strReplaceMapping); + } +} + +void expandXMLTemplateElems(DocumentPtr doc) +{ + // replace node definitions that use a TypeList + for (auto elem : doc->traverseTree()) + { + if (elem->getCategory() != "template") + continue; + + if (!elem->hasAttribute("varName")) { + // std::cerr << "