[tmva][sofie] Disentangle PyMVA and SOFIE

So far, the part of SOFIE that uses the C Python API was included in
PyMVA for convenience, but the functionality is completely unrelated.

Moving this code to SOFIE itself means we can fully build and test SOFIE
without enabling `tmva-pymva`.

The only subtelty is that we want to be able to disable these SOFIE
features that require linking against `libpython`, because this is not
always allowed (e.g. in the Python wheels). Therefore, this part is only
built if we build also the other ROOT libraries that links against
`libpython` besides PyMVA, wich is `TPython`.

Author: guitargeek

cmake/modules/SearchInstalledSoftware.cmake

@@ -705,7 +705,7 @@ if(pyroot AND NOT (tpython OR tmva-pymva))
 elseif(tpython OR tmva-pymva)
   list(APPEND python_components Development)
 endif()
-if(tmva-pymva)
+if(tmva-pymva OR tmva-sofie)
   list(APPEND python_components NumPy)
 endif()
 find_package(Python3 3.9 COMPONENTS ${python_components})
@@ -1796,7 +1796,7 @@ if(tmva)
       endif()
     endif()
   endif()
-  if(tmva-pymva)
+  if(tmva-pymva OR tmva-sofie)
     if(fail-on-missing AND (NOT Python3_NumPy_FOUND OR NOT Python3_Development_FOUND))
       message(SEND_ERROR "TMVA: numpy python package or Python development package not found and tmva-pymva component required"
                           " (python executable: ${Python3_EXECUTABLE})")

tmva/pymva/CMakeLists.txt

@@ -17,17 +17,13 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
     TMVA/MethodPyKeras.h
     TMVA/MethodPyRandomForest.h
     TMVA/MethodPyTorch.h
-    TMVA/RModelParser_Keras.h
-    TMVA/RModelParser_PyTorch.h
     TMVA/PyMethodBase.h
   SOURCES
     src/MethodPyAdaBoost.cxx
     src/MethodPyGTB.cxx
     src/MethodPyKeras.cxx
     src/MethodPyRandomForest.cxx
     src/MethodPyTorch.cxx
-    src/RModelParser_Keras.cxx
-    src/RModelParser_PyTorch.cxx
     src/PyMethodBase.cxx
   LIBRARIES
     Python3::NumPy
@@ -38,7 +34,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
     Thread
     RIO
     TMVA
-    ROOTTMVASofie
 )
 
 ROOT_ADD_TEST_SUBDIRECTORY(test)

tmva/pymva/inc/LinkDef.h

@@ -15,8 +15,4 @@
 #pragma link C++ class TMVA::MethodPyGTB+;
 #pragma link C++ class TMVA::MethodPyKeras+;
 #pragma link C++ class TMVA::MethodPyTorch+;
-#pragma link C++ namespace TMVA::Experimental::SOFIE::PyKeras;
-#pragma link C++ function TMVA::Experimental::SOFIE::PyKeras::Parse+;
-#pragma link C++ namespace TMVA::Experimental::SOFIE::PyTorch;
-#pragma link C++ function TMVA::Experimental::SOFIE::PyTorch::Parse+;
 #endif

tmva/pymva/test/CMakeLists.txt

@@ -11,7 +11,7 @@
 
 project(pymva-tests)
 
-set(Libraries Core MathCore TMVA PyMVA ROOTTMVASofie)
+set(Libraries Core MathCore TMVA PyMVA)
 
 # Look for needed python modules
 ROOT_FIND_PYTHON_MODULE(torch)
@@ -55,10 +55,6 @@ endif(ROOT_SKLEARN_FOUND)
 
 # Enable tests based on available python modules
 if(ROOT_TORCH_FOUND)
-   configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY)
-   configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY)
-   configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY)
-   configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY)
    # Test PyTorch: Binary classification
 
    if (ROOT_SKLEARN_FOUND)
@@ -80,27 +76,9 @@ if(ROOT_TORCH_FOUND)
       LIBRARIES ${Libraries})
    ROOT_ADD_TEST(PyMVA-Torch-Multiclass COMMAND testPyTorchMulticlass DEPENDS ${PyMVA-Torch-Multiclass-depends})
 
-   # Test RModelParser_PyTorch
-
-   if(BLAS_FOUND)
-     ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C
-           LIBRARIES
-           ROOTTMVASofie
-           TMVA
-           Python3::NumPy
-           Python3::Python
-           BLAS::BLAS
-           INCLUDE_DIRS
-           SYSTEM
-           ${CMAKE_CURRENT_BINARY_DIR}
-        )
-   endif()
-
 endif(ROOT_TORCH_FOUND)
 
 if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSORFLOW_FOUND))
-   configure_file(generateKerasModels.py generateKerasModels.py COPYONLY)
-   configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY)
 
    if (ROOT_TORCH_FOUND)
       set(PyMVA-Keras-Classification-depends PyMVA-Torch-Classification)
@@ -127,19 +105,4 @@ if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSOR
    ROOT_EXECUTABLE(testPyKerasMulticlass testPyKerasMulticlass.C
       LIBRARIES ${Libraries})
    ROOT_ADD_TEST(PyMVA-Keras-Multiclass COMMAND testPyKerasMulticlass DEPENDS ${PyMVA-Keras-Multiclass-depends})
-
-   if(BLAS_FOUND)
-      ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C
-         LIBRARIES
-         ROOTTMVASofie
-         PyMVA
-         Python3::NumPy
-         Python3::Python
-         BLAS::BLAS
-         INCLUDE_DIRS
-         SYSTEM
-         ${CMAKE_CURRENT_BINARY_DIR}
-     )
-  endif()
-
 endif()

tmva/sofie/test/CMakeLists.txt

@@ -107,6 +107,8 @@ endif()
 
 # Look for needed Python modules
 ROOT_FIND_PYTHON_MODULE(torch)
+ROOT_FIND_PYTHON_MODULE(keras)
+
 if (ROOT_TORCH_FOUND)
   configure_file(Conv1dModelGenerator.py  Conv1dModelGenerator.py COPYONLY)
   configure_file(Conv2dModelGenerator.py  Conv2dModelGenerator.py COPYONLY)
@@ -127,6 +129,46 @@ if (ROOT_TORCH_FOUND)
   endif()
 endif()
 
+# Any reatures that link against libpython are disabled if built with tpython=OFF
+if (tpython AND ROOT_TORCH_FOUND AND BLAS_FOUND)
+  configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY)
+  configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY)
+  configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY)
+  configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY)
+
+  # Test RModelParser_PyTorch
+  ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C
+        LIBRARIES
+        ROOTTMVASofie
+        TMVA
+        Python3::NumPy
+        Python3::Python
+        BLAS::BLAS
+        INCLUDE_DIRS
+        SYSTEM
+        ${CMAKE_CURRENT_BINARY_DIR}
+  )
+endif()
+
+
+# Any reatures that link against libpython are disabled if built with tpython=OFF
+if (tpython AND ROOT_KERAS_FOUND AND BLAS_FOUND)
+  configure_file(generateKerasModels.py generateKerasModels.py COPYONLY)
+  configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY)
+
+  ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C
+     LIBRARIES
+     ROOTTMVASofie
+     Python3::NumPy
+     Python3::Python
+     BLAS::BLAS
+     INCLUDE_DIRS
+     SYSTEM
+     ${CMAKE_CURRENT_BINARY_DIR}
+  )
+endif()
+
+
 ROOT_EXECUTABLE(emitGNN GNN/EmitGNN.cxx LIBRARIES ROOTTMVASofie)
 ROOT_ADD_TEST(tmva-sofie-EmitGNN COMMAND emitGNN)
 

tmva/pymva/test/TestRModelParserKeras.C renamed to tmva/sofie/test/TestRModelParserKeras.C

@@ -7,10 +7,22 @@
 
 #include "TSystem.h"
 #include "TMVA/RSofieReader.hxx"
-#include "TMVA/PyMethodBase.h"
 
 constexpr float DEFAULT_TOLERANCE = 1e-6f;
 
+namespace {
+
+// Utility functions (taken from PyMethodBase in PyMVA)
+
+const char *PyStringAsString(PyObject *string)
+{
+   PyObject *encodedString = PyUnicode_AsUTF8String(string);
+   const char *cstring = PyBytes_AsString(encodedString);
+   return cstring;
+}
+
+} // namespace
+
 void GenerateModels() {
 
    FILE* fKerasModels = fopen("generateKerasModels.py", "r");
@@ -529,7 +541,7 @@ TEST(RModel, CUSTOM_OP)
     // get input name for custom (it is output of one before last)
     PyRun_String("outputName = model.get_layer(index=len(model.layers)-2).output.name",Py_single_input,fGlobalNS,fLocalNS);
     PyObject *pOutputName = PyDict_GetItemString(fLocalNS, "outputName");
-    std::string outputName = TMVA::PyMethodBase::PyStringAsString(pOutputName);
+    std::string outputName = PyStringAsString(pOutputName);
     TMVA::Experimental:: RSofieReader r;
     r.AddCustomOperator(/*OpName*/ "Scale_by_2",
                         /*input tensor names where to insert custom op */std::string("{\"" + outputName + "\"}"),
@@ -550,4 +562,4 @@ TEST(RModel, CUSTOM_OP)
     for (size_t i = 0; i < outputCustomOp.size(); ++i) {
         EXPECT_LE(std::abs(outputCustomOp[i] - pOutputCustomOp[i]), TOLERANCE);
     }
-}
+}