dalotia: add tensorflow C loading interface

.github/workflows/ctest.yml

@@ -31,6 +31,7 @@ jobs:
       uses: threeal/cmake-action@v1.3.0
       with:
         run-build: true
+        options: DALOTIA_WITH_TENSORFLOW=ON
         build-args: --config ${{ matrix.build_type }}
         # cxx-compiler: ${{ matrix.compiler }}
         build-dir: build-${{ matrix.build_type }}
@@ -40,3 +41,5 @@ jobs:
       with:
         test-dir: build-${{ matrix.build_type }}
         args: --output-on-failure -T memcheck
+    - name: Output Memcheck Log
+      run: for i in /home/runner/work/dalotia/dalotia/build-*/Testing/Temporary/MemoryChecker.*.log ; do echo $i ; cat $i ; done

.github/workflows/spack.yml

@@ -26,5 +26,12 @@ jobs:
       run: |
         spack spec dalotia
         spack info dalotia
-        spack dev-build dalotia@main
-
+        spack dev-build --test=root dalotia@main
+        spack load --sh dalotia@main
+    - name: info and install dalotia with tensorflow
+      shell: spack-bash {0}
+      run: |
+        spack spec dalotia~safetensorscpp+tensorflow
+        spack info dalotia~safetensorscpp+tensorflow
+        spack dev-build --test=root dalotia@main ~safetensorscpp+tensorflow
+        spack load --sh dalotia@main+tensorflow

CMakeLists.txt

@@ -1,12 +1,13 @@
 cmake_minimum_required(VERSION 3.24)
 
-project(DALOTIA CXX C)
+project(dalotia CXX C)
 
 option(DALOTIA_CPP_BUILD_EXAMPLES "Build examples" ON)
 option(DALOTIA_BUILD_TESTS "Build tests" ON)
 option(DALOTIA_WITH_CPP_PMR "use polymorphic memory resources (pmr) C++17 feature for dalotia" ON)
 option(DALOTIA_WITH_OPENMP "Build with OpenMP support" OFF)
 option(DALOTIA_WITH_SAFETENSORS_CPP "use safetensors-cpp for tensor I/O" ON)
+option(DALOTIA_WITH_TENSORFLOW "use the Tensorflow C backend for tensor I/O" OFF)
 option(DALOTIA_WITH_FORTRAN "Build Fortran interface" ON)
 if (DALOTIA_WITH_FORTRAN)
     enable_language(Fortran)
@@ -19,6 +20,10 @@ if (DALOTIA_WITH_FORTRAN)
 endif (DALOTIA_WITH_FORTRAN)
 
 set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_SOURCE_DIR}/cmake/modules")
+
+include(GNUInstallDirs)
+set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
 
 # if this is empty, will be set at config time
 # (build with --config Release or --config Debug etc.)
@@ -28,14 +33,14 @@ message(STATUS "CMAKE BUILD TYPE: ${CMAKE_BUILD_TYPE}")
 file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data/)
 
 # safetensors-cpp
+set(DALOTIA_EXTERNAL_SAFETENSORS_CPP OFF)
 if (DALOTIA_WITH_SAFETENSORS_CPP)
-  if (NOT DEFINED safetensors-cpp_DIR OR safetensors-cpp_DIR MATCHES "fetch")
+  if (NOT DEFINED safetensors-cpp_DIR OR safetensors-cpp_DIR MATCHES "fetch" AND (NOT DEFINED safetensors-cpp_FOUND OR NOT safetensors-cpp_FOUND))
     include(FetchContent)
     FetchContent_Declare(
       safetensors-cpp
       GIT_REPOSITORY https://github.com/syoyo/safetensors-cpp.git
       GIT_TAG a88953c981c6773760540592fa97f04619a8f825
-      OVERRIDE_FIND_PACKAGE
     )
     set(SAFETENSORS_CPP_CXX_EXCEPTIONS true) #TODO maybe depend on language?
     set(SAFETENSORS_CPP_BUILD_EXAMPLES false)
@@ -47,20 +52,56 @@ if (DALOTIA_WITH_SAFETENSORS_CPP)
     target_include_directories(safetensors_cpp PUBLIC  
         $<BUILD_INTERFACE:${SAFETENSORS_CPP_INCLUDE_DIR}>
       )
-    # TODO this is not good, every package should install its own targets
-    # cf. https://discourse.cmake.org/t/propagation-of-fetchcontent-targets-when-installing/2559/2
-    install(TARGETS safetensors_cpp EXPORT dalotia
-      LIBRARY DESTINATION lib
-      ARCHIVE DESTINATION lib
-      RUNTIME DESTINATION bin
-      INCLUDES DESTINATION include
-    )
   else()
     # to pass safetensors-cpp_DIR on the command line:
     find_package(safetensors-cpp REQUIRED CMAKE_FIND_ROOT_PATH_BOTH)
+    set(DALOTIA_EXTERNAL_SAFETENSORS_CPP ON)
   endif()
 endif (DALOTIA_WITH_SAFETENSORS_CPP)
 
+# tensorflow
+if (DALOTIA_WITH_TENSORFLOW)  
+  if (NOT DEFINED tensorflow_DIR OR tensorflow_DIR MATCHES "fetch")
+    include(FetchContent)
+    FetchContent_Declare(
+      tensorflow
+      URL https://storage.googleapis.com/tensorflow/versions/2.18.0/libtensorflow-gpu-linux-x86_64.tar.gz
+    )
+    FetchContent_MakeAvailable(tensorflow)
+
+    FetchContent_GetProperties(tensorflow SOURCE_DIR tensorflow_SRC_DIR)
+    set(tensorflow_INCLUDE_DIRS ${tensorflow_SRC_DIR}/include)
+    find_library(tensorflow_LIBRARIES
+      NAMES tensorflow
+      PATHS ${tensorflow_SRC_DIR}/lib
+      NO_DEFAULT_PATH
+    )
+    message(STATUS "tensorflow_LIBRARIES: ${tensorflow_LIBRARIES}")
+
+    include(FindPackageHandleStandardArgs)
+    find_package_handle_standard_args(tensorflow DEFAULT_MSG tensorflow_INCLUDE_DIRS tensorflow_LIBRARIES)
+  else()
+    find_package(tensorflow REQUIRED CMAKE_FIND_ROOT_PATH_BOTH)
+  endif()
+  if(NOT TARGET tensorflow::tensorflow)
+    add_library(tensorflow::tensorflow SHARED IMPORTED)
+    set_target_properties(
+      tensorflow::tensorflow 
+      PROPERTIES
+      IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+      IMPORTED_LOCATION "${tensorflow_LIBRARIES}"
+      INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_INCLUDE_DIRS}"
+    )# $<BUILD_INTERFACE:${tensorflow_INCLUDE_DIRS}> $<INSTALL_INTERFACE:include>
+    # cf. https://stackoverflow.com/a/41179630/7272382
+    install(FILES ${tensorflow_INCLUDE_DIRS}/tensorflow/c/c_api.h
+      DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/tensorflow/c
+    )
+    install(FILES ${tensorflow_LIBRARIES}
+      DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    )
+  endif()
+endif (DALOTIA_WITH_TENSORFLOW)
+
 add_subdirectory(src) # target dalotia_cpp is generated here
 if (DALOTIA_WITH_OPENMP)
   find_package(OpenMP)
@@ -80,25 +121,38 @@ if (DALOTIA_CPP_BUILD_EXAMPLES)
 endif (DALOTIA_CPP_BUILD_EXAMPLES)
 
 if (DALOTIA_BUILD_TESTS)
-  list(APPEND CMAKE_CTEST_ARGUMENTS "--output-on-failure")
   enable_testing()
   add_subdirectory(test)
 endif (DALOTIA_BUILD_TESTS)
 
-# install version info etc.
+
+# install dependencies, version info, etc.
 include(CMakePackageConfigHelpers)
+configure_package_config_file(
+  ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${PROJECT_NAME}-config.cmake.in
+  ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake
+  INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
+  PATH_VARS DALOTIA_EXTERNAL_SAFETENSORS_CPP
+)
+# # Install find modules
+# install(DIRECTORY cmake/modules/ DESTINATION ${INSTALL_CONFIGDIR}/modules)
+
+install(FILES
+  ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake
+  DESTINATION ${INSTALL_CONFIGDIR}
+)
 write_basic_package_version_file(
   "dalotia-config-version.cmake"
   VERSION 0.0.1
   COMPATIBILITY SameMajorVersion
 )
 install(FILES $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/dalotia-config-version.cmake
-  DESTINATION lib/cmake/dalotia
+  DESTINATION ${INSTALL_CONFIGDIR}
 )
 
 # export full library interface
-install(EXPORT dalotia
-  FILE dalotia-config.cmake
+install(EXPORT dalotia_export_set
+  FILE ${PROJECT_NAME}-targets.cmake
   NAMESPACE dalotia::
-  DESTINATION lib/cmake/dalotia
+  DESTINATION ${INSTALL_CONFIGDIR}
 )

cmake/dalotia-config.cmake.in

@@ -0,0 +1,12 @@
+get_filename_component(dalotia_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
+
+if(NOT TARGET dalotia::dalotia)
+  if(@DALOTIA_EXTERNAL_SAFETENSORS_CPP@)
+    # if we wanted to provide a FindSafetensors-cpp.cmake file:
+    set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${dalotia_CMAKE_DIR}/modules")
+    find_package(safetensors-cpp REQUIRED)
+  else()
+    include("${dalotia_CMAKE_DIR}/../safetensors-cpp-config.cmake")
+  endif()
+  include("${dalotia_CMAKE_DIR}/dalotia-targets.cmake")
+endif()

cmake/modules/Findtensorflow.cmake

@@ -0,0 +1,48 @@
+# file shamelessly taken from https://github.com/serizba/cppflow/blob/master/cmake/modules/Findtensorflow.cmake
+# so here goes their license:
+
+# MIT License
+# 
+# Copyright (c) 2019 Sergio Izquierdo
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# installation: https://izquierdo.dev/cppflow/installation.html
+
+find_path(tensorflow_INCLUDE_DIRS
+  NAMES tensorflow/c/c_api.h
+)
+mark_as_advanced(tensorflow_INCLUDE_DIRS)
+
+find_library(tensorflow_LIBRARIES
+  NAMES tensorflow
+)
+mark_as_advanced(tensorflow_LIBRARIES)
+
+
+if(NOT tensorflow_INCLUDE_DIRS)
+  message(STATUS "Could NOT find tensorflow/c/c_api.h")
+endif()
+if(NOT tensorflow_LIBRARIES)
+  message(STATUS "Could NOT find tensorflow library")
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(tensorflow DEFAULT_MSG tensorflow_INCLUDE_DIRS tensorflow_LIBRARIES)
+

data/generate_tf.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+import tf_keras as keras  # use keras version 2 by installing tf-keras
+from tf_keras import layers, models, initializers
+
+
+def residual_block(inputs, filters, kernel_size=3, stride=1):
+    x = layers.Conv2D(
+        filters,
+        kernel_size,
+        strides=stride,
+        padding="same",
+        kernel_initializer=initializers.Constant(0.1),
+    )(inputs)
+    x = layers.BatchNormalization()(x)
+    x = layers.ReLU()(x)
+
+    x = layers.Conv2D(
+        filters,
+        kernel_size,
+        strides=1,
+        padding="same",
+        kernel_initializer=initializers.Constant(0.2),
+    )(x)
+    x = layers.BatchNormalization()(x)
+
+    # Shortcut connection
+    if stride != 1 or inputs.shape[-1] != filters:
+        shortcut = layers.Conv2D(
+            filters,
+            1,
+            strides=stride,
+            padding="same",
+            kernel_initializer=initializers.Constant(0.3),
+        )(inputs)
+        shortcut = layers.BatchNormalization()(shortcut)
+    else:
+        shortcut = inputs
+
+    x = layers.Add()([x, shortcut])
+    return layers.ReLU()(x)
+
+
+def build_resnet(input_shape=(16, 16, 3), num_classes=10):
+    inputs = keras.Input(shape=input_shape)
+    x = layers.Conv2D(
+        16, 3, padding="same", kernel_initializer=initializers.Constant(0.4)
+    )(inputs)
+    x = layers.BatchNormalization()(x)
+    x = layers.ReLU()(x)
+
+    x = residual_block(x, 16)
+
+    x = layers.GlobalAveragePooling2D()(x)
+    outputs = layers.Dense(
+        num_classes, activation="softmax", kernel_initializer=initializers.GlorotNormal(seed=133)
+    )(x)
+
+    return models.Model(inputs, outputs)
+
+
+if __name__ == "__main__":
+    model = build_resnet()
+    model.save("tensorflow_model", save_format="tf")
+
+    for layer in model.layers:
+        print(f"{layer.name}:")
+        for weight in layer.weights:
+            print(f"  {weight.name} = {weight.numpy()}")