From 460fc97cb317700066564d7f6db8b78af41ab68d Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Sat, 20 Sep 2025 17:35:35 +0530 Subject: [PATCH 01/15] Out-Of-Process on CppInterOp --- .github/actions/Build_LLVM/action.yml | 12 + .../Build_and_Test_CppInterOp/action.yml | 3 + .github/workflows/main.yml | 39 +- CMakeLists.txt | 12 + README.md | 62 ++ docs/InstallationAndUsage.rst | 41 + include/CppInterOp/CppInterOp.h | 7 + lib/CppInterOp/Compatibility.h | 70 +- lib/CppInterOp/CppInterOp.cpp | 91 +- lib/CppInterOp/CppInterOpInterpreter.h | 99 +- patches/llvm/clang20-2-out-of-process.patch | 940 ++++++++++++++++++ unittests/CppInterOp/CMakeLists.txt | 5 +- unittests/CppInterOp/CUDATest.cpp | 8 +- .../CppInterOp/DynamicLibraryManagerTest.cpp | 7 +- unittests/CppInterOp/EnumReflectionTest.cpp | 2 +- .../CppInterOp/FunctionReflectionTest.cpp | 73 +- unittests/CppInterOp/InterpreterTest.cpp | 77 +- unittests/CppInterOp/JitTest.cpp | 34 + unittests/CppInterOp/ScopeReflectionTest.cpp | 16 +- unittests/CppInterOp/TypeReflectionTest.cpp | 10 +- unittests/CppInterOp/Utils.cpp | 18 +- unittests/CppInterOp/Utils.h | 3 + .../CppInterOp/VariableReflectionTest.cpp | 14 +- unittests/CppInterOp/main.cpp | 12 + 24 files changed, 1556 insertions(+), 99 deletions(-) create mode 100644 patches/llvm/clang20-2-out-of-process.patch diff --git a/.github/actions/Build_LLVM/action.yml b/.github/actions/Build_LLVM/action.yml index 53fd301a8..38fcf6c85 100644 --- a/.github/actions/Build_LLVM/action.yml +++ b/.github/actions/Build_LLVM/action.yml @@ -43,6 +43,10 @@ runs: ninja LLVMOrcDebugging -j ${{ env.ncpus }} ninja clingInterpreter -j ${{ env.ncpus }} else + if [[ "${{ matrix.oop-jit }}" == "On" ]]; then + git apply -v ../patches/llvm/clang20-2-out-of-process.patch + echo "Apply clang20-2-out-of-process.patch:" + fi cd build cmake -DLLVM_ENABLE_PROJECTS="${{ matrix.llvm_enable_projects}}" \ -DLLVM_TARGETS_TO_BUILD="${{ matrix.llvm_targets_to_build }}" \ @@ -58,6 +62,14 @@ runs: -DLLVM_INCLUDE_TESTS=OFF \ ../llvm ninja clang clangInterpreter clangStaticAnalyzerCore -j ${{ env.ncpus }} + if [[ "${{ matrix.oop-jit }}" == "On" ]]; then + if [[ "${{ matrix.os }}" == macos* ]]; then + SUFFIX="_osx" + elif [[ "${{ matrix.os }}" == ubuntu* ]]; then + SUFFIX="-x86_64" + fi + ninja clang-repl llvm-jitlink-executor orc_rt${SUFFIX} -j ${{ env.ncpus }} + fi cd ./tools/ rm -rf $(find . -maxdepth 1 ! -name "clang" ! -name ".") cd .. diff --git a/.github/actions/Build_and_Test_CppInterOp/action.yml b/.github/actions/Build_and_Test_CppInterOp/action.yml index b97d2e8ae..c1387b4d9 100644 --- a/.github/actions/Build_and_Test_CppInterOp/action.yml +++ b/.github/actions/Build_and_Test_CppInterOp/action.yml @@ -48,6 +48,7 @@ runs: -DCODE_COVERAGE=${{ env.CODE_COVERAGE }} \ -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ -DLLVM_ENABLE_WERROR=On \ + -DLLVM_BUILT_WITH_OOP_JIT=${{ matrix.oop-jit }} \ ../ fi docs_on=$(echo "${{ matrix.documentation }}" | tr '[:lower:]' '[:upper:]') @@ -60,6 +61,8 @@ runs: if [[ "${os}" != "macos"* ]]; then valgrind --show-error-list=yes --track-origins=yes --error-exitcode=1 unittests/CppInterOp/CppInterOpTests/unittests/bin/${{ env.BUILD_TYPE }}/CppInterOpTests fi + if [[ "${{ matrix.oop-jit }}" == "On" ]]; then + ./unittests/CppInterOp/CppInterOpTests/unittests/bin/${{ env.BUILD_TYPE }}/CppInterOpTests --use-oop-jit fi echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV echo "CPPINTEROP_BUILD_DIR=$CPPINTEROP_BUILD_DIR" >> $GITHUB_ENV diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac9450d0b..e0889decf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,15 +23,6 @@ jobs: matrix: include: # Ubuntu Arm Jobs - - name: ubu22-arm-gcc12-clang-repl-20-coverage - os: ubuntu-22.04-arm - compiler: gcc-12 - clang-runtime: '20' - cling: Off - cppyy: Off - llvm_enable_projects: "clang" - llvm_targets_to_build: "host;NVPTX" - coverage: true - name: ubu24-arm-gcc12-clang-repl-20 os: ubuntu-24.04-arm compiler: gcc-12 @@ -66,6 +57,16 @@ jobs: llvm_enable_projects: "clang" llvm_targets_to_build: "host;NVPTX" # Ubuntu X86 Jobs + - name: ubu22-x86-gcc12-clang-repl-20-coverage + os: ubuntu-22.04 + compiler: gcc-12 + clang-runtime: '20' + cling: Off + cppyy: Off + llvm_enable_projects: "clang;compiler-rt" + llvm_targets_to_build: "host;NVPTX" + coverage: true + oop-jit: On - name: ubu24-x86-gcc12-clang-repl-20 os: ubuntu-24.04 compiler: gcc-12 @@ -74,6 +75,15 @@ jobs: cppyy: Off llvm_enable_projects: "clang" llvm_targets_to_build: "host;NVPTX" + - name: ubu24-x86-gcc12-clang-repl-20-out-of-process + os: ubuntu-24.04 + compiler: gcc-12 + clang-runtime: '20' + cling: Off + cppyy: Off + llvm_enable_projects: "clang;compiler-rt" + llvm_targets_to_build: "host;NVPTX" + oop-jit: On - name: ubu24-x86-gcc12-clang-repl-19-cppyy os: ubuntu-24.04 compiler: gcc-12 @@ -100,6 +110,15 @@ jobs: llvm_enable_projects: "clang" llvm_targets_to_build: "host;NVPTX" # MacOS Arm Jobs + - name: osx15-arm-clang-clang-repl-20-out-of-process + os: macos-15 + compiler: clang + clang-runtime: '20' + cling: Off + cppyy: Off + llvm_enable_projects: "clang;compiler-rt" + llvm_targets_to_build: "host" + oop-jit: On - name: osx15-arm-clang-clang-repl-20 os: macos-15 compiler: clang @@ -220,7 +239,7 @@ jobs: path: | llvm-project ${{ matrix.cling=='On' && 'cling' || '' }} - key: ${{ env.CLING_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x-patch-${{ hashFiles(format('patches/llvm/clang{0}-*.patch', matrix.clang-runtime)) || 'none' }} + key: ${{ env.CLING_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x-patch-${{ hashFiles(format('patches/llvm/clang{0}-*.patch', matrix.clang-runtime)) || 'none' }}${{ matrix.oop-jit == 'On' && '-oop' || '' }} - name: Setup default Build Type uses: ./.github/actions/Miscellaneous/Select_Default_Build_Type diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d1aa034d..9a1012a28 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -298,6 +298,18 @@ include_directories(SYSTEM ${LLVM_INCLUDE_DIRS}) separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS}) add_definitions(${LLVM_DEFINITIONS_LIST}) +string(REGEX REPLACE "/build/lib/cmake/llvm$" "" LLVM_SOURCE_DIR "${LLVM_DIR}") +add_definitions(-DLLVM_SOURCE_DIR="${LLVM_SOURCE_DIR}") + +if(LLVM_BUILT_WITH_OOP_JIT) + if((CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") OR + (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")) + add_definitions(-DLLVM_BUILT_WITH_OOP_JIT) + else() + message(FATAL_ERROR "LLVM_BUILT_WITH_OOP_JIT is only supported on Darwin arm64 or Linux x86_64. Build aborted.") + endif() +endif() + # If the llvm sources are present add them with higher priority. if (LLVM_BUILD_MAIN_SRC_DIR) # LLVM_INCLUDE_DIRS contains the include paths to both LLVM's source and diff --git a/README.md b/README.md index e14a94d94..abd5ee0fc 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,13 @@ git clone --depth=1 --branch release/20.x https://github.com/llvm/llvm-project.g cd llvm-project ``` +If you want to have out-of-process JIT execution enabled in CppInterOp, then apply this patch on Linux and MacOS environment. +> Note that this patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux and MacOS only. + +```bash +git apply -v ../CppInterOp/patches/llvm/clang20-2-out-of-process.patch +``` + ##### Build Clang-REPL Clang-REPL is an interpreter that CppInterOp works alongside. Build Clang (and @@ -202,6 +209,33 @@ and clone cppyy-backend repository where we will be installing the CppInterOp li git clone --depth=1 https://github.com/compiler-research/cppyy-backend.git ``` +##### Build Clang-REPL with Out-of-Process JIT Execution + +To have ``Out-of-Process JIT Execution`` enabled, run following commands to build clang and clang-repl to support this feature: +> Only for Linux x86_64 and Macos amr64 +```bash +mkdir build +cd build +cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt" \ + -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ + -DCLANG_ENABLE_ARCMT=OFF \ + -DCLANG_ENABLE_FORMAT=OFF \ + -DCLANG_ENABLE_BOOTSTRAP=OFF \ + ../llvm +``` + +## For Linux x86_64 +```bash +cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all) +``` +## For MacOS arm64 +```bash +cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu) +``` + #### Build Cling and related dependencies The Cling interpreter and depends on its own customised version of `llvm-project`, @@ -265,6 +299,34 @@ Now CppInterOp can be built. This can be done by executing ```bash mkdir CppInterOp/build/ cd CppInterOp/build/ +``` + +On Windows execute + +```powershell +mkdir CppInterOp\build\ +cd CppInterOp\build\ +``` + +Now if you want to build CppInterOp with Clang-REPL then execute the following commands on Linux and MacOS + +```bash +cmake -DBUILD_SHARED_LIBS=ON -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. +cmake --build . --target install --parallel $(nproc --all) +``` + +and + +> Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled. + +```powershell +cmake -DLLVM_DIR=$env:LLVM_DIR\build\lib\cmake\llvm -DClang_DIR=$env:LLVM_DIR\build\lib\cmake\clang -DCMAKE_INSTALL_PREFIX=$env:CPPINTEROP_DIR .. +cmake --build . --target install --parallel $env:ncpus +``` + +on Windows. If alternatively you would like to install CppInterOp with Cling then execute the following commands on Linux and MacOS. + +```bash cmake -DBUILD_SHARED_LIBS=ON -DCPPINTEROP_USE_CLING=ON -DCPPINTEROP_USE_REPL=Off -DCling_DIR=$LLVM_DIR/build/tools/cling -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. cmake --build . --target install --parallel $(nproc --all) ``` diff --git a/docs/InstallationAndUsage.rst b/docs/InstallationAndUsage.rst index 3ec6470b4..2924f6118 100644 --- a/docs/InstallationAndUsage.rst +++ b/docs/InstallationAndUsage.rst @@ -41,6 +41,15 @@ Clone the 20.x release of the LLVM project repository. git clone --depth=1 --branch release/20.x https://github.com/llvm/llvm-project.git cd llvm-project +If you want to have out-of-process JIT execution enabled in CppInterOp, then apply this patch on Linux and MacOS environment. +.. note:: + + This patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux and MacOS only. + +.. code:: bash + + git apply -v ../CppInterOp/patches/llvm/clang20-2-out-of-process.patch + ****************** Build Clang-REPL ****************** @@ -99,6 +108,34 @@ On Windows you execute the following $env:LLVM_DIR= $PWD.Path cd ..\ +*************************************************** +Build Clang-REPL with Out-of-Process JIT Execution +*************************************************** + +To have `Out-of-Process JIT Execution` enabled, run following commands to build clang and clang-repl to support this feature: + +.. note:: + + Only for Linux x86_64 and Macos arm64 + +.. code:: bash + mkdir build + cd build + cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt" \ + -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ + -DCLANG_ENABLE_ARCMT=OFF \ + -DCLANG_ENABLE_FORMAT=OFF \ + -DCLANG_ENABLE_BOOTSTRAP=OFF \ + ../llvm + + # For Linux x86_64 + cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all) + # For MacOS arm64 + cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu) + ************************************** Build Cling and related dependencies ************************************** @@ -263,6 +300,10 @@ commands on Linux and MacOS cmake -DBUILD_SHARED_LIBS=ON -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. cmake --build . --target install --parallel $(nproc --all) +.. note:: + + Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled. + and .. code:: powershell diff --git a/include/CppInterOp/CppInterOp.h b/include/CppInterOp/CppInterOp.h index 5e6122fe6..cc3cce8f8 100644 --- a/include/CppInterOp/CppInterOp.h +++ b/include/CppInterOp/CppInterOp.h @@ -19,6 +19,7 @@ #include #include #include +#include #include // The cross-platform CPPINTEROP_API macro definition @@ -964,6 +965,12 @@ CPPINTEROP_API void CodeComplete(std::vector& Results, ///\returns 0 on success, non-zero on failure. CPPINTEROP_API int Undo(unsigned N = 1, TInterp_t interp = nullptr); +#ifndef _WIN32 +/// Returns the process ID of the executor process. +/// \returns the PID of the executor process. +CPPINTEROP_API pid_t GetExecutorPID(); +#endif + } // end namespace Cpp #endif // CPPINTEROP_CPPINTEROP_H diff --git a/lib/CppInterOp/Compatibility.h b/lib/CppInterOp/Compatibility.h index 24c822dec..69374440e 100644 --- a/lib/CppInterOp/Compatibility.h +++ b/lib/CppInterOp/Compatibility.h @@ -202,10 +202,24 @@ inline void codeComplete(std::vector& Results, #include "llvm/Support/Error.h" +#ifdef LLVM_BUILT_WITH_OOP_JIT +#include "clang/Basic/Version.h" +#include "llvm/TargetParser/Host.h" + +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" + +#include +#endif + +#include + +static const llvm::ExitOnError ExitOnError; + namespace compat { inline std::unique_ptr -createClangInterpreter(std::vector& args) { +createClangInterpreter(std::vector& args, int stdin_fd = 0, + int stdout_fd = 1, int stderr_fd = 2) { auto has_arg = [](const char* x, llvm::StringRef match = "cuda") { llvm::StringRef Arg = x; Arg = Arg.trim().ltrim('-'); @@ -219,6 +233,15 @@ createClangInterpreter(std::vector& args) { bool CudaEnabled = !gpu_args.empty(); #endif +#if defined(_WIN32) + bool outOfProcess = false; +#else + bool outOfProcess = + std::any_of(args.begin(), args.end(), [](const char* arg) { + return llvm::StringRef(arg).trim() == "--use-oop-jit"; + }); +#endif + clang::IncrementalCompilerBuilder CB; CB.SetCompilerArgs({args.begin(), it}); @@ -243,11 +266,54 @@ createClangInterpreter(std::vector& args) { (*ciOrErr)->LoadRequestedPlugins(); if (CudaEnabled) DeviceCI->LoadRequestedPlugins(); + +#ifdef LLVM_BUILT_WITH_OOP_JIT + + clang::Interpreter::JITConfig OutOfProcessConfig; + if (outOfProcess) { + OutOfProcessConfig.IsOutOfProcess = true; + OutOfProcessConfig.OOPExecutor = + std::string(LLVM_SOURCE_DIR) + "/build/bin/llvm-jitlink-executor"; + OutOfProcessConfig.UseSharedMemory = false; + OutOfProcessConfig.SlabAllocateSize = 0; + OutOfProcessConfig.CustomizeFork = [=] { // Lambda defined inline + auto redirect = [](int from, int to) { + if (from != to) { + dup2(from, to); + close(from); + } + }; + + redirect(stdin_fd, STDIN_FILENO); + redirect(stdout_fd, STDOUT_FILENO); + redirect(stderr_fd, STDERR_FILENO); + + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + }; + OutOfProcessConfig.OrcRuntimePath = + std::string(LLVM_SOURCE_DIR) + + "/build/lib/clang/20/lib/darwin/liborc_rt_osx.a"; + } + auto innerOrErr = + CudaEnabled + ? clang::Interpreter::createWithCUDA(std::move(*ciOrErr), + std::move(DeviceCI)) + : clang::Interpreter::create(std::move(*ciOrErr), OutOfProcessConfig); +#else + if (outOfProcess) { + llvm::errs() + << "[CreateClangInterpreter]: No compatibility with out-of-process " + "JIT. Running in-process JIT execution." + << "(To enable recompile CppInterOp with patch applied and change " + "VERSION file to 1.8.1;dev." + << "\n"; + } auto innerOrErr = CudaEnabled ? clang::Interpreter::createWithCUDA(std::move(*ciOrErr), std::move(DeviceCI)) : clang::Interpreter::create(std::move(*ciOrErr)); - +#endif if (!innerOrErr) { llvm::logAllUnhandledErrors(innerOrErr.takeError(), llvm::errs(), "Failed to build Interpreter:"); diff --git a/lib/CppInterOp/CppInterOp.cpp b/lib/CppInterOp/CppInterOp.cpp index f1527b4a8..789c48f68 100755 --- a/lib/CppInterOp/CppInterOp.cpp +++ b/lib/CppInterOp/CppInterOp.cpp @@ -62,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,10 @@ #include #include #include +#include +#ifndef _WIN32 +#include +#endif #include #include #include @@ -80,6 +85,7 @@ #include #ifndef STDOUT_FILENO #define STDOUT_FILENO 1 +#define STDERR_FILENO 2 // For exec(). #include #define popen(x, y) (_popen(x, y)) @@ -3402,8 +3408,9 @@ TInterp_t CreateInterpreter(const std::vector& Args /*={}*/, #ifdef CPPINTEROP_USE_CLING auto I = new compat::Interpreter(ClingArgv.size(), &ClingArgv[0]); #else - auto Interp = compat::Interpreter::create(static_cast(ClingArgv.size()), - ClingArgv.data()); + auto Interp = + compat::Interpreter::create(static_cast(ClingArgv.size()), + ClingArgv.data(), nullptr, {}, nullptr, true); if (!Interp) return nullptr; auto* I = Interp.release(); @@ -4269,12 +4276,11 @@ bool Destruct(TCppObject_t This, TCppConstScope_t scope, } class StreamCaptureInfo { - struct file_deleter { - void operator()(FILE* fp) { pclose(fp); } - }; - std::unique_ptr m_TempFile; + FILE* m_TempFile; int m_FD = -1; int m_DupFD = -1; + int mode = -1; + bool m_OwnsFile = false; public: #ifdef _MSC_VER @@ -4289,20 +4295,47 @@ class StreamCaptureInfo { }()}, m_FD(FD) { #else - StreamCaptureInfo(int FD) : m_TempFile{tmpfile()}, m_FD(FD) { + StreamCaptureInfo(int FD) : mode(FD) { +#endif +#if !defined(CPPINTEROP_USE_CLING) && !defined(_WIN32) + auto& I = getInterp(NULLPTR); + if (I.isOutOfProcess() && FD == STDOUT_FILENO) { + m_TempFile = I.getTempFileForOOP(FD); + ::fflush(m_TempFile); + m_FD = fileno(m_TempFile); + m_OwnsFile = false; + } else { + m_TempFile = tmpfile(); + m_FD = FD; + m_OwnsFile = true; + } +#else + m_TempFile = tmpfile(); + m_FD = FD; + m_OwnsFile = true; + (void)mode; #endif if (!m_TempFile) { perror("StreamCaptureInfo: Unable to create temp file"); return; } - m_DupFD = dup(FD); + m_DupFD = dup(m_FD); // Flush now or can drop the buffer when dup2 is called with Fd later. // This seems only necessary when piping stdout or stderr, but do it // for ttys to avoid over complicated code for minimal benefit. - ::fflush(FD == STDOUT_FILENO ? stdout : stderr); - if (dup2(fileno(m_TempFile.get()), FD) < 0) + if (m_FD == STDOUT_FILENO) { + ::fflush(stdout); + } else if (m_FD == STDERR_FILENO) { + ::fflush(stderr); + } else { +#ifndef _WIN32 + fsync(m_FD); +#endif + } + // ::fflush(FD == STDOUT_FILENO ? stdout : stderr); + if (dup2(fileno(m_TempFile), m_FD) < 0) perror("StreamCaptureInfo:"); } StreamCaptureInfo(const StreamCaptureInfo&) = delete; @@ -4310,7 +4343,12 @@ class StreamCaptureInfo { StreamCaptureInfo(StreamCaptureInfo&&) = delete; StreamCaptureInfo& operator=(StreamCaptureInfo&&) = delete; - ~StreamCaptureInfo() { assert(m_DupFD == -1 && "Captured output not used?"); } + ~StreamCaptureInfo() { + assert(m_DupFD == -1 && "Captured output not used?"); + if (m_TempFile && m_OwnsFile) { + fclose(m_TempFile); + } + } std::string GetCapturedString() { assert(m_DupFD != -1 && "Multiple calls to GetCapturedString"); @@ -4319,11 +4357,11 @@ class StreamCaptureInfo { if (dup2(m_DupFD, m_FD) < 0) perror("StreamCaptureInfo:"); // Go to the end of the file. - if (fseek(m_TempFile.get(), 0L, SEEK_END) != 0) + if (fseek(m_TempFile, 0L, SEEK_END) != 0) perror("StreamCaptureInfo:"); // Get the size of the file. - long bufsize = ftell(m_TempFile.get()); + long bufsize = ftell(m_TempFile); if (bufsize == -1) perror("StreamCaptureInfo:"); @@ -4331,13 +4369,12 @@ class StreamCaptureInfo { std::unique_ptr content(new char[bufsize + 1]); // Go back to the start of the file. - if (fseek(m_TempFile.get(), 0L, SEEK_SET) != 0) + if (fseek(m_TempFile, 0L, SEEK_SET) != 0) perror("StreamCaptureInfo:"); // Read the entire file into memory. - size_t newLen = - fread(content.get(), sizeof(char), bufsize, m_TempFile.get()); - if (ferror(m_TempFile.get()) != 0) + size_t newLen = fread(content.get(), sizeof(char), bufsize, m_TempFile); + if (ferror(m_TempFile) != 0) fputs("Error reading file", stderr); else content[newLen++] = '\0'; // Just to be safe. @@ -4345,6 +4382,15 @@ class StreamCaptureInfo { std::string result = content.get(); close(m_DupFD); m_DupFD = -1; +#if !defined(_WIN32) && !defined(CPPINTEROP_USE_CLING) + auto& I = getInterp(NULLPTR); + if (I.isOutOfProcess() && mode != STDERR_FILENO) { + if (ftruncate(m_FD, 0) != 0) + perror("ftruncate"); + if (lseek(m_FD, 0, SEEK_SET) == -1) + perror("lseek"); + } +#endif return result; } }; @@ -4388,4 +4434,15 @@ int Undo(unsigned N, TInterp_t interp) { #endif } +#ifndef _WIN32 +pid_t GetExecutorPID() { +#ifdef LLVM_BUILT_WITH_OOP_JIT + auto& I = getInterp(NULLPTR); + return I.getOutOfProcessExecutorPID(); +#endif + return 0; +} + +#endif + } // end namespace Cpp diff --git a/lib/CppInterOp/CppInterOpInterpreter.h b/lib/CppInterOp/CppInterOpInterpreter.h index 4833c0bd5..8aa88edff 100644 --- a/lib/CppInterOp/CppInterOpInterpreter.h +++ b/lib/CppInterOp/CppInterOpInterpreter.h @@ -39,6 +39,12 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Triple.h" +#ifndef _WIN32 +#include +#endif +#include +#include +#include #include #include @@ -140,10 +146,35 @@ namespace Cpp { /// CppInterOp Interpreter /// class Interpreter { + +private: + struct FileDeleter { + void operator()(FILE* f /* owns */) { + if (f) + fclose(f); + } + }; + struct IOContext { + std::unique_ptr stdin_file; + std::unique_ptr stdout_file; + std::unique_ptr stderr_file; + bool outOfProcess = false; + + bool initializeTempFiles() { + stdin_file.reset(tmpfile()); // NOLINT(cppcoreguidelines-owning-memory) + stdout_file.reset(tmpfile()); // NOLINT(cppcoreguidelines-owning-memory) + stderr_file.reset(tmpfile()); // NOLINT(cppcoreguidelines-owning-memory) + return stdin_file && stdout_file && stderr_file; + } + }; + std::unique_ptr inner; + std::unique_ptr io_context; public: - Interpreter(std::unique_ptr CI) : inner(std::move(CI)) {} + Interpreter(std::unique_ptr CI, + std::unique_ptr ctx = nullptr) + : inner(std::move(CI)), io_context(std::move(ctx)) {} static std::unique_ptr create(int argc, const char* const* argv, const char* llvmdir = nullptr, @@ -157,13 +188,44 @@ class Interpreter { llvm::InitializeAllAsmPrinters(); std::vector vargs(argv + 1, argv + argc); - auto CI = compat::createClangInterpreter(vargs); + + auto io_ctx = std::make_unique(); + + int stdin_fd = 0; + int stdout_fd = 1; + int stderr_fd = 2; + +#if defined(_WIN32) + io_ctx->outOfProcess = false; +#else + io_ctx->outOfProcess = + std::any_of(vargs.begin(), vargs.end(), [](const char* arg) { + return llvm::StringRef(arg).trim() == "--use-oop-jit"; + }); + + if (io_ctx->outOfProcess) { + bool init = io_ctx->initializeTempFiles(); + if (!init) { + llvm::errs() << "Can't start out-of-process JIT execution. Continuing " + "with in-process JIT execution.\n"; + io_ctx->outOfProcess = false; + } else { + stdin_fd = fileno(io_ctx->stdin_file.get()); + stdout_fd = fileno(io_ctx->stdout_file.get()); + stderr_fd = fileno(io_ctx->stderr_file.get()); + } + } +#endif + + auto CI = + compat::createClangInterpreter(vargs, stdin_fd, stdout_fd, stderr_fd); if (!CI) { llvm::errs() << "Interpreter creation failed\n"; return nullptr; } - return std::unique_ptr(new Interpreter(std::move(CI))); + return std::unique_ptr( + new Interpreter(std::move(CI), std::move(io_ctx))); } ~Interpreter() {} @@ -171,6 +233,28 @@ class Interpreter { operator const clang::Interpreter&() const { return *inner; } operator clang::Interpreter&() { return *inner; } + bool isOutOfProcess() const { + return io_context ? io_context->outOfProcess : false; + } + +#ifndef _WIN32 + FILE* getTempFileForOOP(int FD) { + if (!io_context) + return nullptr; + switch (FD) { + case (STDIN_FILENO): + return io_context->stdin_file.get(); + case (STDOUT_FILENO): + return io_context->stdout_file.get(); + case (STDERR_FILENO): + return io_context->stderr_file.get(); + default: + llvm::errs() << "No temp file for the FD\n"; + return nullptr; + } + } +#endif + ///\brief Describes the return result of the different routines that do the /// incremental compilation. /// @@ -229,6 +313,15 @@ class Interpreter { return llvm::orc::ExecutorAddr(*AddrOrErr); } + pid_t getOutOfProcessExecutorPID() const { +#ifndef _WIN32 +#ifdef LLVM_BUILT_WITH_OOP_JIT + return inner->getOutOfProcessExecutorPID(); +#endif +#endif + return 0; + } + /// \returns the \c ExecutorAddr of a given name as written in the object /// file. llvm::Expected diff --git a/patches/llvm/clang20-2-out-of-process.patch b/patches/llvm/clang20-2-out-of-process.patch new file mode 100644 index 000000000..b86f1b4bf --- /dev/null +++ b/patches/llvm/clang20-2-out-of-process.patch @@ -0,0 +1,940 @@ +diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h +index f8663e3193a1..1f9553e6809a 100644 +--- a/clang/include/clang/Interpreter/Interpreter.h ++++ b/clang/include/clang/Interpreter/Interpreter.h +@@ -20,8 +20,10 @@ + + #include "llvm/ADT/DenseMap.h" + #include "llvm/ExecutionEngine/JITSymbol.h" ++#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" + #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + #include "llvm/Support/Error.h" ++#include + #include + #include + +@@ -35,6 +37,10 @@ class ThreadSafeContext; + + namespace clang { + ++namespace driver { ++class ToolChain; ++} // namespace driver ++ + class CompilerInstance; + class CodeGenerator; + class CXXRecordDecl; +@@ -119,15 +125,40 @@ class Interpreter { + /// An optional compiler instance for CUDA offloading + std::unique_ptr DeviceCI; + ++public: ++ struct JITConfig { ++ /// Indicates whether out-of-process JIT execution is enabled. ++ bool IsOutOfProcess = false; ++ /// Path to the out-of-process JIT executor. ++ std::string OOPExecutor = ""; ++ std::string OOPExecutorConnect = ""; ++ /// Indicates whether to use shared memory for communication. ++ bool UseSharedMemory = false; ++ /// Representing the slab allocation size for memory management in kb. ++ unsigned SlabAllocateSize = 0; ++ /// Path to the ORC runtime library. ++ std::string OrcRuntimePath = ""; ++ /// PID of the out-of-process JIT executor. ++ uint32_t ExecutorPID = 0; ++ /// Custom lambda to be executed inside child process/executor ++ std::function CustomizeFork = nullptr; ++ ++ JITConfig() ++ : IsOutOfProcess(false), OOPExecutor(""), OOPExecutorConnect(""), ++ UseSharedMemory(false), SlabAllocateSize(0), OrcRuntimePath(""), ++ ExecutorPID(0), CustomizeFork(nullptr) {} ++ }; ++ + protected: + // Derived classes can use an extended interface of the Interpreter. + Interpreter(std::unique_ptr Instance, llvm::Error &Err, + std::unique_ptr JITBuilder = nullptr, +- std::unique_ptr Consumer = nullptr); ++ std::unique_ptr Consumer = nullptr, ++ JITConfig Config = JITConfig()); + + // Create the internal IncrementalExecutor, or re-create it after calling + // ResetExecutor(). +- llvm::Error CreateExecutor(); ++ llvm::Error CreateExecutor(JITConfig Config = JITConfig()); + + // Delete the internal IncrementalExecutor. This causes a hard shutdown of the + // JIT engine. In particular, it doesn't run cleanup or destructors. +@@ -136,10 +167,19 @@ protected: + public: + virtual ~Interpreter(); + static llvm::Expected> +- create(std::unique_ptr CI); ++ create(std::unique_ptr CI, JITConfig Config = {}); + static llvm::Expected> + createWithCUDA(std::unique_ptr CI, + std::unique_ptr DCI); ++ static llvm::Expected> ++ createLLJITBuilder(std::unique_ptr EPC, ++ llvm::StringRef OrcRuntimePath); ++ static llvm::Expected< ++ std::pair, uint32_t>> ++ outOfProcessJITBuilder(JITConfig Config); ++ static llvm::Expected ++ getOrcRuntimePath(const driver::ToolChain &TC); ++ + const ASTContext &getASTContext() const; + ASTContext &getASTContext(); + const CompilerInstance *getCompilerInstance() const; +@@ -170,6 +210,8 @@ public: + llvm::Expected + getSymbolAddressFromLinkerName(llvm::StringRef LinkerName) const; + ++ uint32_t getOutOfProcessExecutorPID() const; ++ + const llvm::SmallVectorImpl &getValuePrintingInfo() const { + return ValuePrintingInfo; + } +diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp +index 4d2adecaafce..45620fcd358c 100644 +--- a/clang/lib/Interpreter/IncrementalExecutor.cpp ++++ b/clang/lib/Interpreter/IncrementalExecutor.cpp +@@ -15,19 +15,36 @@ + #include "clang/Basic/TargetInfo.h" + #include "clang/Basic/TargetOptions.h" + #include "clang/Interpreter/PartialTranslationUnit.h" ++#include "llvm/ADT/StringExtras.h" + #include "llvm/ExecutionEngine/ExecutionEngine.h" + #include "llvm/ExecutionEngine/Orc/CompileUtils.h" ++#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" + #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" ++#include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h" ++#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" + #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" + #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" + #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" + #include "llvm/ExecutionEngine/Orc/LLJIT.h" ++#include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h" + #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" ++#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" ++#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" + #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" + #include "llvm/ExecutionEngine/SectionMemoryManager.h" + #include "llvm/IR/Module.h" ++#include "llvm/Support/FileSystem.h" + #include "llvm/Support/ManagedStatic.h" ++#include "llvm/Support/Path.h" + #include "llvm/Support/TargetSelect.h" ++#include "llvm/TargetParser/Host.h" ++ ++#ifdef LLVM_ON_UNIX ++#include ++#include ++#include ++#include ++#endif // LLVM_ON_UNIX + + // Force linking some of the runtimes that helps attaching to a debugger. + LLVM_ATTRIBUTE_USED void linkComponents() { +@@ -55,8 +72,9 @@ IncrementalExecutor::createDefaultJITBuilder( + + IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, + llvm::orc::LLJITBuilder &JITBuilder, ++ Interpreter::JITConfig Config, + llvm::Error &Err) +- : TSCtx(TSC) { ++ : TSCtx(TSC), OutOfProcessChildPid(Config.ExecutorPID) { + using namespace llvm::orc; + llvm::ErrorAsOutParameter EAO(&Err); + +@@ -118,4 +136,229 @@ IncrementalExecutor::getSymbolAddress(llvm::StringRef Name, + return SymOrErr->getAddress(); + } + ++Expected> ++createSharedMemoryManager(llvm::orc::SimpleRemoteEPC &SREPC, ++ unsigned SlabAllocateSize) { ++ llvm::orc::SharedMemoryMapper::SymbolAddrs SAs; ++ if (auto Err = SREPC.getBootstrapSymbols( ++ {{SAs.Instance, ++ llvm::orc::rt::ExecutorSharedMemoryMapperServiceInstanceName}, ++ {SAs.Reserve, ++ llvm::orc::rt::ExecutorSharedMemoryMapperServiceReserveWrapperName}, ++ {SAs.Initialize, ++ llvm::orc::rt:: ++ ExecutorSharedMemoryMapperServiceInitializeWrapperName}, ++ {SAs.Deinitialize, ++ llvm::orc::rt:: ++ ExecutorSharedMemoryMapperServiceDeinitializeWrapperName}, ++ {SAs.Release, ++ llvm::orc::rt:: ++ ExecutorSharedMemoryMapperServiceReleaseWrapperName}})) ++ return std::move(Err); ++ ++ size_t SlabSize; ++ if (llvm::Triple(llvm::sys::getProcessTriple()).isOSWindows()) ++ SlabSize = 1024 * 1024; ++ else ++ SlabSize = 1024 * 1024 * 1024; ++ ++ if (SlabAllocateSize > 0) ++ SlabSize = SlabAllocateSize; ++ ++ return llvm::orc::MapperJITLinkMemoryManager::CreateWithMapper< ++ llvm::orc::SharedMemoryMapper>(SlabSize, SREPC, SAs); ++} ++ ++llvm::Expected, uint32_t>> ++IncrementalExecutor::launchExecutor(llvm::StringRef ExecutablePath, ++ bool UseSharedMemory, ++ unsigned SlabAllocateSize, ++ std::function CustomizeFork) { ++#ifndef LLVM_ON_UNIX ++ // FIXME: Add support for Windows. ++ return llvm::make_error( ++ "-" + ExecutablePath + " not supported on non-unix platforms", ++ llvm::inconvertibleErrorCode()); ++#elif !LLVM_ENABLE_THREADS ++ // Out of process mode using SimpleRemoteEPC depends on threads. ++ return llvm::make_error( ++ "-" + ExecutablePath + ++ " requires threads, but LLVM was built with " ++ "LLVM_ENABLE_THREADS=Off", ++ llvm::inconvertibleErrorCode()); ++#else ++ ++ if (!llvm::sys::fs::can_execute(ExecutablePath)) ++ return llvm::make_error( ++ llvm::formatv("Specified executor invalid: {0}", ExecutablePath), ++ llvm::inconvertibleErrorCode()); ++ ++ constexpr int ReadEnd = 0; ++ constexpr int WriteEnd = 1; ++ ++ // Pipe FDs. ++ int ToExecutor[2]; ++ int FromExecutor[2]; ++ ++ uint32_t ChildPID; ++ ++ // Create pipes to/from the executor.. ++ if (pipe(ToExecutor) != 0 || pipe(FromExecutor) != 0) ++ return llvm::make_error( ++ "Unable to create pipe for executor", llvm::inconvertibleErrorCode()); ++ ++ ChildPID = fork(); ++ ++ if (ChildPID == 0) { ++ // In the child... ++ ++ // Close the parent ends of the pipes ++ close(ToExecutor[WriteEnd]); ++ close(FromExecutor[ReadEnd]); ++ ++ if (CustomizeFork) ++ CustomizeFork(); ++ ++ // Execute the child process. ++ std::unique_ptr ExecutorPath, FDSpecifier; ++ { ++ ExecutorPath = std::make_unique(ExecutablePath.size() + 1); ++ strcpy(ExecutorPath.get(), ExecutablePath.data()); ++ ++ std::string FDSpecifierStr("filedescs="); ++ FDSpecifierStr += llvm::utostr(ToExecutor[ReadEnd]); ++ FDSpecifierStr += ','; ++ FDSpecifierStr += llvm::utostr(FromExecutor[WriteEnd]); ++ FDSpecifier = std::make_unique(FDSpecifierStr.size() + 1); ++ strcpy(FDSpecifier.get(), FDSpecifierStr.c_str()); ++ } ++ ++ char *const Args[] = {ExecutorPath.get(), FDSpecifier.get(), nullptr}; ++ int RC = execvp(ExecutorPath.get(), Args); ++ if (RC != 0) { ++ llvm::errs() << "unable to launch out-of-process executor \"" ++ << ExecutorPath.get() << "\"\n"; ++ exit(1); ++ } ++ } ++ // else we're the parent... ++ ++ // Close the child ends of the pipes ++ close(ToExecutor[ReadEnd]); ++ close(FromExecutor[WriteEnd]); ++ ++ llvm::orc::SimpleRemoteEPC::Setup S = llvm::orc::SimpleRemoteEPC::Setup(); ++ if (UseSharedMemory) ++ S.CreateMemoryManager = ++ [SlabAllocateSize](llvm::orc::SimpleRemoteEPC &EPC) { ++ return createSharedMemoryManager(EPC, SlabAllocateSize); ++ }; ++ ++ auto EPCOrErr = ++ llvm::orc::SimpleRemoteEPC::Create( ++ std::make_unique( ++ std::nullopt), ++ std::move(S), FromExecutor[ReadEnd], ToExecutor[WriteEnd]); ++ if (!EPCOrErr) ++ return EPCOrErr.takeError(); ++ return std::make_pair(std::move(*EPCOrErr), ChildPID); ++#endif ++} ++ ++#if LLVM_ON_UNIX && LLVM_ENABLE_THREADS ++ ++static Expected connectTCPSocketImpl(std::string Host, ++ std::string PortStr) { ++ addrinfo *AI; ++ addrinfo Hints{}; ++ Hints.ai_family = AF_INET; ++ Hints.ai_socktype = SOCK_STREAM; ++ Hints.ai_flags = AI_NUMERICSERV; ++ ++ if (int EC = getaddrinfo(Host.c_str(), PortStr.c_str(), &Hints, &AI)) ++ return llvm::make_error( ++ llvm::formatv("address resolution failed ({0})", strerror(EC)), ++ llvm::inconvertibleErrorCode()); ++ // Cycle through the returned addrinfo structures and connect to the first ++ // reachable endpoint. ++ int SockFD; ++ addrinfo *Server; ++ for (Server = AI; Server != nullptr; Server = Server->ai_next) { ++ // socket might fail, e.g. if the address family is not supported. Skip to ++ // the next addrinfo structure in such a case. ++ if ((SockFD = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol)) < 0) ++ continue; ++ ++ // If connect returns null, we exit the loop with a working socket. ++ if (connect(SockFD, Server->ai_addr, Server->ai_addrlen) == 0) ++ break; ++ ++ close(SockFD); ++ } ++ freeaddrinfo(AI); ++ ++ // If we reached the end of the loop without connecting to a valid endpoint, ++ // dump the last error that was logged in socket() or connect(). ++ if (Server == nullptr) ++ return llvm::make_error("invalid hostname", ++ llvm::inconvertibleErrorCode()); ++ ++ return SockFD; ++} ++ ++llvm::Expected> ++IncrementalExecutor::connectTCPSocket(llvm::StringRef NetworkAddress, ++ bool UseSharedMemory, ++ unsigned SlabAllocateSize) { ++#ifndef LLVM_ON_UNIX ++ // FIXME: Add TCP support for Windows. ++ return llvm::make_error( ++ "-" + NetworkAddress + " not supported on non-unix platforms", ++ llvm::inconvertibleErrorCode()); ++#elif !LLVM_ENABLE_THREADS ++ // Out of process mode using SimpleRemoteEPC depends on threads. ++ return llvm::make_error( ++ "-" + NetworkAddress + ++ " requires threads, but LLVM was built with " ++ "LLVM_ENABLE_THREADS=Off", ++ llvm::inconvertibleErrorCode()); ++#else ++ ++ auto CreateErr = [NetworkAddress](Twine Details) { ++ return llvm::make_error( ++ formatv("Failed to connect TCP socket '{0}': {1}", NetworkAddress, ++ Details), ++ llvm::inconvertibleErrorCode()); ++ }; ++ ++ StringRef Host, PortStr; ++ std::tie(Host, PortStr) = NetworkAddress.split(':'); ++ if (Host.empty()) ++ return CreateErr("Host name for -" + NetworkAddress + " can not be empty"); ++ if (PortStr.empty()) ++ return CreateErr("Port number in -" + NetworkAddress + " can not be empty"); ++ int Port = 0; ++ if (PortStr.getAsInteger(10, Port)) ++ return CreateErr("Port number '" + PortStr + "' is not a valid integer"); ++ ++ Expected SockFD = connectTCPSocketImpl(Host.str(), PortStr.str()); ++ if (!SockFD) ++ return SockFD.takeError(); ++ ++ llvm::orc::SimpleRemoteEPC::Setup S = llvm::orc::SimpleRemoteEPC::Setup(); ++ if (UseSharedMemory) ++ S.CreateMemoryManager = ++ [SlabAllocateSize](llvm::orc::SimpleRemoteEPC &EPC) { ++ return createSharedMemoryManager(EPC, SlabAllocateSize); ++ }; ++ ++ return llvm::orc::SimpleRemoteEPC::Create< ++ llvm::orc::FDSimpleRemoteEPCTransport>( ++ std::make_unique( ++ std::nullopt), ++ std::move(S), *SockFD, *SockFD); ++#endif ++} ++#endif // _WIN32 ++ + } // namespace clang +diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h +index 71d71bc3883e..56e83378f004 100644 +--- a/clang/lib/Interpreter/IncrementalExecutor.h ++++ b/clang/lib/Interpreter/IncrementalExecutor.h +@@ -13,13 +13,20 @@ + #ifndef LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H + #define LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H + ++#include "clang/Interpreter/Interpreter.h" + #include "llvm/ADT/DenseMap.h" + #include "llvm/ADT/StringRef.h" ++#include "llvm/ExecutionEngine/Orc/Core.h" ++#include "llvm/ExecutionEngine/Orc/Core.h" + #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" ++#include "llvm/ExecutionEngine/Orc/Layer.h" + #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" ++#include "llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h" ++#include "llvm/Support/Error.h" + ++#include + #include +- ++#include + namespace llvm { + class Error; + namespace orc { +@@ -39,6 +46,7 @@ class IncrementalExecutor { + using CtorDtorIterator = llvm::orc::CtorDtorIterator; + std::unique_ptr Jit; + llvm::orc::ThreadSafeContext &TSCtx; ++ uint32_t OutOfProcessChildPid = -1; + + llvm::DenseMap + ResourceTrackers; +@@ -50,7 +58,8 @@ public: + enum SymbolNameKind { IRName, LinkerName }; + + IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, +- llvm::orc::LLJITBuilder &JITBuilder, llvm::Error &Err); ++ llvm::orc::LLJITBuilder &JITBuilder, ++ Interpreter::JITConfig Config, llvm::Error &Err); + virtual ~IncrementalExecutor(); + + virtual llvm::Error addModule(PartialTranslationUnit &PTU); +@@ -62,8 +71,22 @@ public: + + llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; } + ++ uint32_t getOutOfProcessChildPid() const { return OutOfProcessChildPid; } ++ + static llvm::Expected> + createDefaultJITBuilder(llvm::orc::JITTargetMachineBuilder JTMB); ++ ++ static llvm::Expected< ++ std::pair, uint32_t>> ++ launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory, ++ unsigned SlabAllocateSize, ++ std::function CustomizeFork = nullptr); ++ ++#if LLVM_ON_UNIX && LLVM_ENABLE_THREADS ++ static llvm::Expected> ++ connectTCPSocket(llvm::StringRef NetworkAddress, bool UseSharedMemory, ++ unsigned SlabAllocateSize); ++#endif + }; + + } // end namespace clang +diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp +index 3b81f9d701b4..63fb0e175ac8 100644 +--- a/clang/lib/Interpreter/Interpreter.cpp ++++ b/clang/lib/Interpreter/Interpreter.cpp +@@ -46,6 +46,7 @@ + #include "clang/Sema/Lookup.h" + #include "clang/Serialization/ObjectFilePCHContainerReader.h" + #include "llvm/ExecutionEngine/JITSymbol.h" ++#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" + #include "llvm/ExecutionEngine/Orc/LLJIT.h" + #include "llvm/IR/Module.h" + #include "llvm/Support/Errc.h" +@@ -365,7 +366,8 @@ public: + Interpreter::Interpreter(std::unique_ptr Instance, + llvm::Error &ErrOut, + std::unique_ptr JITBuilder, +- std::unique_ptr Consumer) ++ std::unique_ptr Consumer, ++ JITConfig Config) + : JITBuilder(std::move(JITBuilder)) { + CI = std::move(Instance); + llvm::ErrorAsOutParameter EAO(&ErrOut); +@@ -395,7 +397,7 @@ Interpreter::Interpreter(std::unique_ptr Instance, + ASTContext &C = CI->getASTContext(); + RegisterPTU(C.getTranslationUnitDecl(), std::move(M)); + } +- if (llvm::Error Err = CreateExecutor()) { ++ if (llvm::Error Err = CreateExecutor(Config)) { + ErrOut = joinErrors(std::move(ErrOut), std::move(Err)); + return; + } +@@ -454,22 +456,120 @@ const char *const Runtimes = R"( + EXTERN_C void __clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ...); + )"; + ++llvm::Expected, uint32_t>> ++Interpreter::outOfProcessJITBuilder(JITConfig Config) { ++ std::unique_ptr EPC; ++ uint32_t childPid = -1; ++ if (!Config.OOPExecutor.empty()) { ++ // Launch an out-of-process executor locally in a child process. ++ auto ResultOrErr = IncrementalExecutor::launchExecutor( ++ Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize, ++ Config.CustomizeFork); ++ if (!ResultOrErr) ++ return ResultOrErr.takeError(); ++ childPid = ResultOrErr->second; ++ auto EPCOrErr = std::move(ResultOrErr->first); ++ EPC = std::move(EPCOrErr); ++ } else if (Config.OOPExecutorConnect != "") { ++#if LLVM_ON_UNIX && LLVM_ENABLE_THREADS ++ auto EPCOrErr = IncrementalExecutor::connectTCPSocket( ++ Config.OOPExecutorConnect, Config.UseSharedMemory, ++ Config.SlabAllocateSize); ++ if (!EPCOrErr) ++ return EPCOrErr.takeError(); ++ EPC = std::move(*EPCOrErr); ++#else ++ return llvm::make_error( ++ "Out-of-process JIT over TCP is not supported on this platform", ++ std::error_code()); ++#endif ++ } ++ ++ std::unique_ptr JB; ++ if (EPC) { ++ auto JBOrErr = clang::Interpreter::createLLJITBuilder( ++ std::move(EPC), Config.OrcRuntimePath); ++ if (!JBOrErr) ++ return JBOrErr.takeError(); ++ JB = std::move(*JBOrErr); ++ } ++ ++ return std::make_pair(std::move(JB), childPid); ++} ++ ++llvm::Expected ++Interpreter::getOrcRuntimePath(const driver::ToolChain &TC) { ++ std::optional CompilerRTPath = TC.getCompilerRTPath(); ++ std::optional ResourceDir = TC.getRuntimePath(); ++ ++ if (!CompilerRTPath) { ++ return llvm::make_error("CompilerRT path not found", ++ std::error_code()); ++ } ++ ++ const std::array OrcRTLibNames = { ++ "liborc_rt.a", "liborc_rt_osx.a", "liborc_rt-x86_64.a"}; ++ ++ for (const char *LibName : OrcRTLibNames) { ++ llvm::SmallString<256> CandidatePath((*CompilerRTPath).c_str()); ++ llvm::sys::path::append(CandidatePath, LibName); ++ ++ if (llvm::sys::fs::exists(CandidatePath)) { ++ return CandidatePath.str().str(); ++ } ++ } ++ ++ return llvm::make_error( ++ llvm::Twine("OrcRuntime library not found in: ") + (*CompilerRTPath), ++ std::error_code()); ++} ++ + llvm::Expected> +-Interpreter::create(std::unique_ptr CI) { ++Interpreter::create(std::unique_ptr CI, JITConfig Config) { + llvm::Error Err = llvm::Error::success(); +- auto Interp = +- std::unique_ptr(new Interpreter(std::move(CI), Err)); +- if (Err) +- return std::move(Err); ++ ++ std::unique_ptr JB; ++ ++ if (Config.IsOutOfProcess) { ++ const TargetInfo &TI = CI->getTarget(); ++ const llvm::Triple &Triple = TI.getTriple(); ++ ++ DiagnosticsEngine &Diags = CI->getDiagnostics(); ++ std::string BinaryName = llvm::sys::fs::getMainExecutable(nullptr, nullptr); ++ driver::Driver Driver(BinaryName, Triple.str(), Diags); ++ // Need fake args to get the driver to create a compilation. ++ std::vector Args = {"clang", "--version"}; ++ std::unique_ptr C( ++ Driver.BuildCompilation(Args)); ++ if (!C) { ++ return llvm::make_error( ++ "Failed to create driver compilation for out-of-process JIT", ++ std::error_code()); ++ } ++ if (Config.OrcRuntimePath == "") { ++ const clang::driver::ToolChain &TC = C->getDefaultToolChain(); ++ ++ auto OrcRuntimePathOrErr = getOrcRuntimePath(TC); ++ if (!OrcRuntimePathOrErr) { ++ return OrcRuntimePathOrErr.takeError(); ++ } ++ ++ Config.OrcRuntimePath = *OrcRuntimePathOrErr; ++ } ++ } ++ ++ auto Interp = std::unique_ptr(new Interpreter( ++ std::move(CI), Err, std::move(JB), /*Consumer=*/nullptr, Config)); ++ if (auto E = std::move(Err)) ++ return std::move(E); + + // Add runtime code and set a marker to hide it from user code. Undo will not + // go through that. +- auto PTU = Interp->Parse(Runtimes); +- if (!PTU) +- return PTU.takeError(); ++ if (auto E = Interp->ParseAndExecute(Runtimes)) ++ return std::move(E); ++ + Interp->markUserCodeStart(); + +- Interp->ValuePrintingInfo.resize(4); + return std::move(Interp); + } + +@@ -551,6 +651,12 @@ size_t Interpreter::getEffectivePTUSize() const { + return PTUs.size() - InitPTUSize; + } + ++uint32_t Interpreter::getOutOfProcessExecutorPID() const { ++ if (IncrExecutor) ++ return IncrExecutor->getOutOfProcessChildPid(); ++ return -1; ++} ++ + PartialTranslationUnit & + Interpreter::RegisterPTU(TranslationUnitDecl *TU, + std::unique_ptr M /*={}*/, +@@ -617,7 +723,26 @@ createJITTargetMachineBuilder(const std::string &TT) { + return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT)); + } + +-llvm::Error Interpreter::CreateExecutor() { ++llvm::Expected> ++Interpreter::createLLJITBuilder( ++ std::unique_ptr EPC, ++ llvm::StringRef OrcRuntimePath) { ++ const std::string &TT = EPC->getTargetTriple().getTriple(); ++ auto JTMB = createJITTargetMachineBuilder(TT); ++ if (!JTMB) ++ return JTMB.takeError(); ++ auto JB = IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB)); ++ if (!JB) ++ return JB.takeError(); ++ ++ (*JB)->setExecutorProcessControl(std::move(EPC)); ++ (*JB)->setPlatformSetUp( ++ llvm::orc::ExecutorNativePlatform(OrcRuntimePath.str())); ++ ++ return std::move(*JB); ++} ++ ++llvm::Error Interpreter::CreateExecutor(JITConfig Config) { + if (IncrExecutor) + return llvm::make_error("Operation failed. " + "Execution engine exists", +@@ -626,8 +751,26 @@ llvm::Error Interpreter::CreateExecutor() { + return llvm::make_error("Operation failed. " + "No code generator available", + std::error_code()); ++ ++ const std::string &TT = getCompilerInstance()->getTargetOpts().Triple; ++ llvm::Triple TargetTriple(TT); ++ bool IsWindowsTarget = TargetTriple.isOSWindows(); ++ ++ if (!IsWindowsTarget && Config.IsOutOfProcess) { ++ if (!JITBuilder) { ++ auto ResOrErr = outOfProcessJITBuilder(Config); ++ if (!ResOrErr) ++ return ResOrErr.takeError(); ++ JITBuilder = std::move(ResOrErr->first); ++ Config.ExecutorPID = ResOrErr->second; ++ } ++ if (!JITBuilder) ++ return llvm::make_error( ++ "Operation failed. No LLJITBuilder for out-of-process JIT", ++ std::error_code()); ++ } ++ + if (!JITBuilder) { +- const std::string &TT = getCompilerInstance()->getTargetOpts().Triple; + auto JTMB = createJITTargetMachineBuilder(TT); + if (!JTMB) + return JTMB.takeError(); +@@ -638,11 +781,15 @@ llvm::Error Interpreter::CreateExecutor() { + } + + llvm::Error Err = llvm::Error::success(); ++ ++ // Fix: Declare Executor as the appropriate unique_ptr type ++ std::unique_ptr Executor; ++ + #ifdef __EMSCRIPTEN__ +- auto Executor = std::make_unique(*TSCtx); ++ Executor = std::make_unique(*TSCtx); + #else +- auto Executor = +- std::make_unique(*TSCtx, *JITBuilder, Err); ++ Executor = ++ std::make_unique(*TSCtx, *JITBuilder, Config, Err); + #endif + if (!Err) + IncrExecutor = std::move(Executor); +diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp +index 7af8e4f25d99..c1a7ec397917 100644 +--- a/clang/tools/clang-repl/ClangRepl.cpp ++++ b/clang/tools/clang-repl/ClangRepl.cpp +@@ -11,6 +11,8 @@ + //===----------------------------------------------------------------------===// + + #include "clang/Basic/Diagnostic.h" ++#include "clang/Basic/Version.h" ++#include "clang/Config/config.h" + #include "clang/Frontend/CompilerInstance.h" + #include "clang/Frontend/FrontendDiagnostic.h" + #include "clang/Interpreter/CodeCompletion.h" +@@ -18,14 +20,27 @@ + #include "clang/Lex/Preprocessor.h" + #include "clang/Sema/Sema.h" + ++#include "llvm/ADT/SmallString.h" ++#include "llvm/ADT/StringRef.h" + #include "llvm/ExecutionEngine/Orc/LLJIT.h" + #include "llvm/LineEditor/LineEditor.h" + #include "llvm/Support/CommandLine.h" ++#include "llvm/Support/FileSystem.h" + #include "llvm/Support/ManagedStatic.h" // llvm_shutdown ++#include "llvm/Support/Path.h" + #include "llvm/Support/Signals.h" + #include "llvm/Support/TargetSelect.h" ++#include "llvm/Support/VirtualFileSystem.h" ++#include "llvm/Support/raw_ostream.h" ++#include "llvm/TargetParser/Host.h" ++#include "llvm/TargetParser/Triple.h" + #include + ++#include ++#include ++ ++#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" ++ + // Disable LSan for this test. + // FIXME: Re-enable once we can assume GCC 13.2 or higher. + // https://llvm.org/github.com/llvm/llvm-project/issues/67586. +@@ -34,10 +49,36 @@ + LLVM_ATTRIBUTE_USED int __lsan_is_turned_off() { return 1; } + #endif + ++#define DEBUG_TYPE "clang-repl" ++ + static llvm::cl::opt CudaEnabled("cuda", llvm::cl::Hidden); + static llvm::cl::opt CudaPath("cuda-path", llvm::cl::Hidden); + static llvm::cl::opt OffloadArch("offload-arch", llvm::cl::Hidden); +- ++static llvm::cl::OptionCategory OOPCategory("Out-of-process Execution Options"); ++static llvm::cl::opt SlabAllocateSizeString( ++ "slab-allocate", ++ llvm::cl::desc("Allocate from a slab of the given size " ++ "(allowable suffixes: Kb, Mb, Gb. default = " ++ "Kb)"), ++ llvm::cl::init(""), llvm::cl::cat(OOPCategory)); ++static llvm::cl::opt ++ OOPExecutor("oop-executor", ++ llvm::cl::desc("Launch an out-of-process executor to run code"), ++ llvm::cl::init(""), llvm::cl::ValueOptional, ++ llvm::cl::cat(OOPCategory)); ++static llvm::cl::opt OOPExecutorConnect( ++ "oop-executor-connect", ++ llvm::cl::desc( ++ "Connect to an out-of-process executor through a TCP socket"), ++ llvm::cl::value_desc(":")); ++static llvm::cl::opt ++ OrcRuntimePath("orc-runtime", llvm::cl::desc("Path to the ORC runtime"), ++ llvm::cl::init(""), llvm::cl::ValueOptional, ++ llvm::cl::cat(OOPCategory)); ++static llvm::cl::opt UseSharedMemory( ++ "use-shared-memory", ++ llvm::cl::desc("Use shared memory to transfer generated code and data"), ++ llvm::cl::init(false), llvm::cl::cat(OOPCategory)); + static llvm::cl::list + ClangArgs("Xcc", + llvm::cl::desc("Argument to pass to the CompilerInvocation"), +@@ -47,6 +88,79 @@ static llvm::cl::opt OptHostSupportsJit("host-supports-jit", + static llvm::cl::list OptInputs(llvm::cl::Positional, + llvm::cl::desc("[code to run]")); + ++static llvm::Error sanitizeOopArguments(const char *ArgV0) { ++ // Only one of -oop-executor and -oop-executor-connect can be used. ++ if (!!OOPExecutor.getNumOccurrences() && ++ !!OOPExecutorConnect.getNumOccurrences()) ++ return llvm::make_error( ++ "Only one of -" + OOPExecutor.ArgStr + " and -" + ++ OOPExecutorConnect.ArgStr + " can be specified", ++ llvm::inconvertibleErrorCode()); ++ ++ llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); ++ // TODO: Remove once out-of-process execution support is implemented for ++ // non-Unix platforms. ++ if ((!SystemTriple.isOSBinFormatELF() && ++ !SystemTriple.isOSBinFormatMachO()) && ++ (OOPExecutor.getNumOccurrences() || ++ OOPExecutorConnect.getNumOccurrences())) ++ return llvm::make_error( ++ "Out-of-process execution is only supported on Unix platforms", ++ llvm::inconvertibleErrorCode()); ++ ++ // If -slab-allocate is passed, check that we're not trying to use it in ++ // -oop-executor or -oop-executor-connect mode. ++ // ++ // FIXME: Remove once we enable remote slab allocation. ++ if (SlabAllocateSizeString != "") { ++ if (OOPExecutor.getNumOccurrences() || ++ OOPExecutorConnect.getNumOccurrences()) ++ return llvm::make_error( ++ "-slab-allocate cannot be used with -oop-executor or " ++ "-oop-executor-connect", ++ llvm::inconvertibleErrorCode()); ++ } ++ ++ // Out-of-process executors require the ORC runtime. ORC Runtime Path ++ // resolution is done in Interpreter.cpp. ++ ++ // If -oop-executor was used but no value was specified then use a sensible ++ // default. ++ if (!!OOPExecutor.getNumOccurrences() && OOPExecutor.empty()) { ++ llvm::SmallString<256> OOPExecutorPath(llvm::sys::fs::getMainExecutable( ++ ArgV0, reinterpret_cast(&sanitizeOopArguments))); ++ llvm::sys::path::remove_filename(OOPExecutorPath); ++ llvm::sys::path::append(OOPExecutorPath, "llvm-jitlink-executor"); ++ OOPExecutor = OOPExecutorPath.str().str(); ++ } ++ ++ return llvm::Error::success(); ++} ++ ++static llvm::Expected getSlabAllocSize(llvm::StringRef SizeString) { ++ SizeString = SizeString.trim(); ++ ++ uint64_t Units = 1024; ++ ++ if (SizeString.ends_with_insensitive("kb")) ++ SizeString = SizeString.drop_back(2).rtrim(); ++ else if (SizeString.ends_with_insensitive("mb")) { ++ Units = 1024 * 1024; ++ SizeString = SizeString.drop_back(2).rtrim(); ++ } else if (SizeString.ends_with_insensitive("gb")) { ++ Units = 1024 * 1024 * 1024; ++ SizeString = SizeString.drop_back(2).rtrim(); ++ } else if (SizeString.empty()) ++ return 0; ++ ++ uint64_t SlabSize = 0; ++ if (SizeString.getAsInteger(10, SlabSize)) ++ return llvm::make_error( ++ "Invalid numeric format for slab size", llvm::inconvertibleErrorCode()); ++ ++ return SlabSize * Units; ++} ++ + static void LLVMErrorHandler(void *UserData, const char *Message, + bool GenCrashDiag) { + auto &Diags = *static_cast(UserData); +@@ -86,7 +200,7 @@ struct ReplListCompleter { + clang::Interpreter &MainInterp; + ReplListCompleter(clang::IncrementalCompilerBuilder &CB, + clang::Interpreter &Interp) +- : CB(CB), MainInterp(Interp){}; ++ : CB(CB), MainInterp(Interp) {}; + + std::vector operator()(llvm::StringRef Buffer, + size_t Pos) const; +@@ -183,6 +297,19 @@ int main(int argc, const char **argv) { + DeviceCI = ExitOnErr(CB.CreateCudaDevice()); + } + ++ ExitOnErr(sanitizeOopArguments(argv[0])); ++ ++ clang::Interpreter::JITConfig Config; ++ Config.IsOutOfProcess = !OOPExecutor.empty() || !OOPExecutorConnect.empty(); ++ Config.OOPExecutor = OOPExecutor; ++ auto SizeOrErr = getSlabAllocSize(SlabAllocateSizeString); ++ if (!SizeOrErr) { ++ llvm::logAllUnhandledErrors(SizeOrErr.takeError(), llvm::errs(), "error: "); ++ return EXIT_FAILURE; ++ } ++ Config.SlabAllocateSize = *SizeOrErr; ++ Config.UseSharedMemory = UseSharedMemory; ++ + // FIXME: Investigate if we could use runToolOnCodeWithArgs from tooling. It + // can replace the boilerplate code for creation of the compiler instance. + std::unique_ptr CI; +@@ -214,8 +341,9 @@ int main(int argc, const char **argv) { + auto CudaRuntimeLibPath = CudaPath + "/lib/libcudart.so"; + ExitOnErr(Interp->LoadDynamicLibrary(CudaRuntimeLibPath.c_str())); + } +- } else +- Interp = ExitOnErr(clang::Interpreter::create(std::move(CI))); ++ } else { ++ Interp = ExitOnErr(clang::Interpreter::create(std::move(CI), Config)); ++ } + + bool HasError = false; + +@@ -243,15 +371,34 @@ int main(int argc, const char **argv) { + } + + Input += L; ++ // If we add more % commands, there should be better architecture than ++ // this. + if (Input == R"(%quit)") { + break; + } + if (Input == R"(%undo)") { + if (auto Err = Interp->Undo()) + llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); ++ } else if (Input == R"(%help)") { ++ llvm::outs() << "%help\t\tlist clang-repl %commands\n" ++ << "%undo\t\tundo the previous input\n" ++ << "%lib\t\tlink a dynamic library\n" ++ << "%quit\t\texit clang-repl\n"; ++ } else if (Input == R"(%lib)") { ++ auto Err = llvm::make_error( ++ "%lib expects 1 argument: the path to a dynamic library\n", ++ std::error_code()); ++ llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); + } else if (Input.rfind("%lib ", 0) == 0) { + if (auto Err = Interp->LoadDynamicLibrary(Input.data() + 5)) + llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); ++ } else if (Input[0] == '%') { ++ auto Err = llvm::make_error( ++ llvm::formatv( ++ "Invalid % command \"{0}\", use \"%help\" to list commands\n", ++ Input), ++ std::error_code()); ++ llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); + } else if (auto Err = Interp->ParseAndExecute(Input)) { + llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); + } +@@ -267,4 +414,4 @@ int main(int argc, const char **argv) { + llvm::remove_fatal_error_handler(); + + return checkDiagErrors(Interp->getCompilerInstance(), HasError); +-} ++} +\ No newline at end of file + diff --git a/unittests/CppInterOp/CMakeLists.txt b/unittests/CppInterOp/CMakeLists.txt index 843846f25..1b6566bb9 100644 --- a/unittests/CppInterOp/CMakeLists.txt +++ b/unittests/CppInterOp/CMakeLists.txt @@ -4,13 +4,14 @@ if (EMSCRIPTEN) # Omitting CUDATest.cpp since Emscripten build currently has no GPU support # For Emscripten builds linking to gtest_main will not suffice for gtest to run # the tests and an explicitly main.cpp is needed - set(EXTRA_TEST_SOURCE_FILES main.cpp) else() # Do not need main.cpp for native builds, but we do have GPU support for native builds set(EXTRA_TEST_SOURCE_FILES CUDATest.cpp) set(EXTRA_PATH_TEST_BINARIES /CppInterOpTests/unittests/bin/$/) endif() +set(EXTRA_TEST_SOURCE_FILES main.cpp) + add_cppinterop_unittest(CppInterOpTests EnumReflectionTest.cpp FunctionReflectionTest.cpp @@ -89,7 +90,7 @@ if (NOT EMSCRIPTEN) set(EXTRA_PATH_TEST_BINARIES /TestSharedLib/unittests/bin/$/) endif() -add_cppinterop_unittest(DynamicLibraryManagerTests DynamicLibraryManagerTest.cpp ${EXTRA_TEST_SOURCE_FILES}) +add_cppinterop_unittest(DynamicLibraryManagerTests DynamicLibraryManagerTest.cpp ${EXTRA_TEST_SOURCE_FILES} Utils.cpp) target_link_libraries(DynamicLibraryManagerTests PRIVATE diff --git a/unittests/CppInterOp/CUDATest.cpp b/unittests/CppInterOp/CUDATest.cpp index 45b41c94d..9eb42b6ad 100644 --- a/unittests/CppInterOp/CUDATest.cpp +++ b/unittests/CppInterOp/CUDATest.cpp @@ -14,7 +14,7 @@ static bool HasCudaSDK() { // FIXME: Enable this for cling. return false; #endif - if (!Cpp::CreateInterpreter({}, {"--cuda"})) + if (!TestUtils::CreateInterpreter({}, {"--cuda"})) return false; return Cpp::Declare("__global__ void test_func() {}" "test_func<<<1,1>>>();") == 0; @@ -32,7 +32,7 @@ static bool HasCudaRuntime() { if (!HasCudaSDK()) return false; - if (!Cpp::CreateInterpreter({}, {"--cuda"})) + if (!TestUtils::CreateInterpreter({}, {"--cuda"})) return false; if (Cpp::Declare("__global__ void test_func() {}" "test_func<<<1,1>>>();")) @@ -54,7 +54,7 @@ TEST(CUDATest, Sanity) { #endif if (!HasCudaSDK()) GTEST_SKIP() << "Skipping CUDA tests as CUDA SDK not found"; - EXPECT_TRUE(Cpp::CreateInterpreter({}, {"--cuda"})); + EXPECT_TRUE(TestUtils::CreateInterpreter({}, {"--cuda"})); } TEST(CUDATest, CUDAH) { @@ -64,7 +64,7 @@ TEST(CUDATest, CUDAH) { if (!HasCudaSDK()) GTEST_SKIP() << "Skipping CUDA tests as CUDA SDK not found"; - Cpp::CreateInterpreter({}, {"--cuda"}); + TestUtils::CreateInterpreter({}, {"--cuda"}); bool success = !Cpp::Declare("#include "); EXPECT_TRUE(success); } diff --git a/unittests/CppInterOp/DynamicLibraryManagerTest.cpp b/unittests/CppInterOp/DynamicLibraryManagerTest.cpp index 030c61416..89d61c3c9 100644 --- a/unittests/CppInterOp/DynamicLibraryManagerTest.cpp +++ b/unittests/CppInterOp/DynamicLibraryManagerTest.cpp @@ -1,3 +1,4 @@ +#include "Utils.h" #include "CppInterOp/CppInterOp.h" #include "clang/Basic/Version.h" @@ -29,7 +30,7 @@ TEST(DynamicLibraryManagerTest, Sanity1) { GTEST_SKIP() << "Test fails with Cling on Windows"; #endif - EXPECT_TRUE(Cpp::CreateInterpreter()); + EXPECT_TRUE(TestUtils::CreateInterpreter()); EXPECT_FALSE(Cpp::GetFunctionAddress("ret_zero")); std::string BinaryPath = GetExecutablePath(/*Argv0=*/nullptr); @@ -76,7 +77,7 @@ TEST(DynamicLibraryManagerTest, Sanity2) { GTEST_SKIP() << "Test fails with Cling on Windows"; #endif - auto* I = Cpp::CreateInterpreter(); + auto* I = TestUtils::CreateInterpreter(); EXPECT_TRUE(I); EXPECT_FALSE(Cpp::GetFunctionAddress("ret_one", I)); @@ -124,7 +125,7 @@ TEST(DynamicLibraryManagerTest, BasicSymbolLookup) { #endif #endif - ASSERT_TRUE(Cpp::CreateInterpreter()); + ASSERT_TRUE(TestUtils::CreateInterpreter()); EXPECT_FALSE(Cpp::GetFunctionAddress("ret_zero")); // Load the library manually. Use known preload path (MEMFS path) diff --git a/unittests/CppInterOp/EnumReflectionTest.cpp b/unittests/CppInterOp/EnumReflectionTest.cpp index f4b3d47de..3b29aa312 100644 --- a/unittests/CppInterOp/EnumReflectionTest.cpp +++ b/unittests/CppInterOp/EnumReflectionTest.cpp @@ -338,7 +338,7 @@ TEST(EnumReflectionTest, GetEnums) { int myVariable; )"; - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); Interp->declare(code); std::vector enumNames1, enumNames2, enumNames3, enumNames4; Cpp::TCppScope_t globalscope = Cpp::GetScope("", 0); diff --git a/unittests/CppInterOp/FunctionReflectionTest.cpp b/unittests/CppInterOp/FunctionReflectionTest.cpp index caf0d3391..87616c133 100644 --- a/unittests/CppInterOp/FunctionReflectionTest.cpp +++ b/unittests/CppInterOp/FunctionReflectionTest.cpp @@ -701,7 +701,7 @@ TEST(FunctionReflectionTest, InstantiateTemplateFunctionFromString) { if (llvm::sys::RunningOnValgrind()) GTEST_SKIP() << "XFAIL due to Valgrind report"; std::vector interpreter_args = { "-include", "new" }; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); std::string code = R"(#include )"; Interp->process(code); const char* str = "std::make_unique"; @@ -1451,6 +1451,11 @@ TEST(FunctionReflectionTest, GetFunctionAddress) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif + + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } + std::vector Decls; std::string code = "int f1(int i) { return i * i; }"; std::vector interpreter_args = {"-include", "new"}; @@ -1519,6 +1524,10 @@ TEST(FunctionReflectionTest, JitCallAdvanced) { if (llvm::sys::RunningOnValgrind()) GTEST_SKIP() << "XFAIL due to Valgrind report"; + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } + Cpp::JitCall JC = Cpp::MakeFunctionCallable(nullptr); EXPECT_TRUE(JC.getKind() == Cpp::JitCall::kUnknown); @@ -1567,6 +1576,10 @@ TEST(FunctionReflectionTest, JitCallDebug) { if (llvm::sys::RunningOnValgrind()) GTEST_SKIP() << "XFAIL due to Valgrind report"; + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } + std::vector Decls, SubDecls; std::string code = R"( class C { @@ -1660,6 +1673,9 @@ TEST(FunctionReflectionTest, GetFunctionCallWrapper) { #if defined(CPPINTEROP_USE_CLING) && defined(_WIN32) GTEST_SKIP() << "Disabled, invoking functions containing printf does not work with Cling on Windows"; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector Decls; std::string code = R"( int f1(int i) { return i * i; } @@ -2321,6 +2337,9 @@ TEST(FunctionReflectionTest, Construct) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector interpreter_args = {"-include", "new"}; std::vector Decls, SubDecls; @@ -2402,8 +2421,11 @@ TEST(FunctionReflectionTest, ConstructPOD) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Interp->declare(R"( namespace PODS { @@ -2445,9 +2467,12 @@ TEST(FunctionReflectionTest, ConstructNested) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Interp->declare(R"( #include @@ -2505,8 +2530,11 @@ TEST(FunctionReflectionTest, ConstructArray) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); Interp->declare(R"( #include @@ -2558,9 +2586,12 @@ TEST(FunctionReflectionTest, Destruct) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Interp->declare(R"( #include @@ -2629,9 +2660,12 @@ TEST(FunctionReflectionTest, DestructArray) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Interp->declare(R"( #include @@ -2703,7 +2737,7 @@ TEST(FunctionReflectionTest, UndoTest) { #ifdef EMSCRIPTEN GTEST_SKIP() << "Test fails for Emscipten builds"; #else - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); EXPECT_EQ(Cpp::Process("int a = 5;"), 0); EXPECT_EQ(Cpp::Process("int b = 10;"), 0); EXPECT_EQ(Cpp::Process("int x = 5;"), 0); @@ -2727,7 +2761,10 @@ TEST(FunctionReflectionTest, FailingTest1) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif - Cpp::CreateInterpreter(); +#ifdef EMSCRIPTEN_SHARED_LIBRARY + GTEST_SKIP() << "Test fails for Emscipten shared library builds"; +#endif + TestUtils::CreateInterpreter(); EXPECT_FALSE(Cpp::Declare(R"( class WithOutEqualOp1 {}; class WithOutEqualOp2 {}; @@ -2754,3 +2791,23 @@ TEST(FunctionReflectionTest, FailingTest1) { EXPECT_FALSE(Cpp::Declare("int x = 1;")); EXPECT_FALSE(Cpp::Declare("int y = x;")); } + +#ifndef _WIN32 +TEST(FunctionReflectionTest, GetExecutorPIDTest) { +#ifdef EMSCRIPTEN + GTEST_SKIP() << "Test fails for Emscipten builds"; +#endif +#ifdef CPPINTEROP_USE_CLING + GTEST_SKIP() << "Test fails for cling builds"; +#endif + if (llvm::sys::RunningOnValgrind()) + GTEST_SKIP() << "XFAIL due to Valgrind report"; + TestUtils::CreateInterpreter(); + pid_t pid = Cpp::GetExecutorPID(); + if (TestUtils::use_oop_jit()) { + EXPECT_NE(pid, 0); + } else { + EXPECT_EQ(pid, 0); + } +} +#endif \ No newline at end of file diff --git a/unittests/CppInterOp/InterpreterTest.cpp b/unittests/CppInterOp/InterpreterTest.cpp index 5800fe64e..551f9cf2f 100644 --- a/unittests/CppInterOp/InterpreterTest.cpp +++ b/unittests/CppInterOp/InterpreterTest.cpp @@ -40,7 +40,7 @@ TEST(InterpreterTest, DISABLED_DebugFlag) { #else TEST(InterpreterTest, DebugFlag) { #endif // NDEBUG - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); EXPECT_FALSE(Cpp::IsDebugOutputEnabled()); std::string cerrs; testing::internal::CaptureStderr(); @@ -52,13 +52,14 @@ TEST(InterpreterTest, DebugFlag) { testing::internal::CaptureStderr(); Cpp::Process("int b = 12;"); cerrs = testing::internal::GetCapturedStderr(); - EXPECT_STRNE(cerrs.c_str(), ""); + EXPECT_STREQ(cerrs.c_str(), ""); Cpp::EnableDebugOutput(false); EXPECT_FALSE(Cpp::IsDebugOutputEnabled()); testing::internal::CaptureStderr(); Cpp::Process("int c = 12;"); cerrs = testing::internal::GetCapturedStderr(); + std::cout << cerrs << std::endl; EXPECT_STREQ(cerrs.c_str(), ""); } @@ -71,6 +72,9 @@ TEST(InterpreterTest, Evaluate) { #endif if (llvm::sys::RunningOnValgrind()) GTEST_SKIP() << "XFAIL due to Valgrind report"; + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } // EXPECT_TRUE(Cpp::Evaluate(I, "") == 0); //EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); // Due to a deficiency in the clang-repl implementation to get the value we @@ -85,9 +89,12 @@ TEST(InterpreterTest, Evaluate) { } TEST(InterpreterTest, DeleteInterpreter) { - auto* I1 = Cpp::CreateInterpreter(); - auto* I2 = Cpp::CreateInterpreter(); - auto* I3 = Cpp::CreateInterpreter(); + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } + auto* I1 = TestUtils::CreateInterpreter(); + auto* I2 = TestUtils::CreateInterpreter(); + auto* I3 = TestUtils::CreateInterpreter(); EXPECT_TRUE(I1 && I2 && I3) << "Failed to create interpreters"; EXPECT_EQ(I3, Cpp::GetInterpreter()) << "I3 is not active"; @@ -106,10 +113,13 @@ TEST(InterpreterTest, ActivateInterpreter) { #ifdef EMSCRIPTEN_STATIC_LIBRARY GTEST_SKIP() << "Test fails for Emscipten static library build"; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } EXPECT_FALSE(Cpp::ActivateInterpreter(nullptr)); - auto* Cpp14 = Cpp::CreateInterpreter({"-std=c++14"}); - auto* Cpp17 = Cpp::CreateInterpreter({"-std=c++17"}); - auto* Cpp20 = Cpp::CreateInterpreter({"-std=c++20"}); + auto* Cpp14 = TestUtils::CreateInterpreter({"-std=c++14"}); + auto* Cpp17 = TestUtils::CreateInterpreter({"-std=c++17"}); + auto* Cpp20 = TestUtils::CreateInterpreter({"-std=c++20"}); EXPECT_TRUE(Cpp14 && Cpp17 && Cpp20); EXPECT_TRUE(Cpp::Evaluate("__cplusplus") == 202002L) @@ -138,7 +148,7 @@ TEST(InterpreterTest, Process) { if (llvm::sys::RunningOnValgrind()) GTEST_SKIP() << "XFAIL due to Valgrind report"; std::vector interpreter_args = { "-include", "new" }; - auto* I = Cpp::CreateInterpreter(interpreter_args); + auto* I = TestUtils::CreateInterpreter(interpreter_args); EXPECT_TRUE(Cpp::Process("") == 0); EXPECT_TRUE(Cpp::Process("int a = 12;") == 0); EXPECT_FALSE(Cpp::Process("error_here;") == 0); @@ -146,22 +156,27 @@ TEST(InterpreterTest, Process) { EXPECT_FALSE(Cpp::Process("int f(); int res = f();") == 0); // C API - auto* CXI = clang_createInterpreterFromRawPtr(I); - clang_Interpreter_declare(CXI, "#include ", false); - clang_Interpreter_process(CXI, "int c = 42;"); - auto* CXV = clang_createValue(); - auto Res = clang_Interpreter_evaluate(CXI, "c", CXV); - EXPECT_EQ(Res, CXError_Success); - clang_Value_dispose(CXV); - clang_Interpreter_dispose(CXI); - auto* OldI = Cpp::TakeInterpreter(); - EXPECT_EQ(OldI, I); + if (!TestUtils::use_oop_jit()) { + auto* CXI = clang_createInterpreterFromRawPtr(I); + clang_Interpreter_declare(CXI, "#include ", false); + clang_Interpreter_process(CXI, "int c = 42;"); + auto* CXV = clang_createValue(); + auto Res = clang_Interpreter_evaluate(CXI, "c", CXV); + EXPECT_EQ(Res, CXError_Success); + clang_Value_dispose(CXV); + clang_Interpreter_dispose(CXI); + auto* OldI = Cpp::TakeInterpreter(); + EXPECT_EQ(OldI, I); + } } TEST(InterpreterTest, EmscriptenExceptionHandling) { #ifndef EMSCRIPTEN GTEST_SKIP() << "This test is intended to check exception handling for Emscripten builds."; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector Args = { "-std=c++20", @@ -172,7 +187,7 @@ TEST(InterpreterTest, EmscriptenExceptionHandling) { "-mllvm", "-enable-emscripten-sjlj" }; - Cpp::CreateInterpreter(Args); + TestUtils::CreateInterpreter(Args); const char* tryCatchCode = R"( try { @@ -186,7 +201,7 @@ TEST(InterpreterTest, EmscriptenExceptionHandling) { } TEST(InterpreterTest, CreateInterpreter) { - auto* I = Cpp::CreateInterpreter(); + auto* I = TestUtils::CreateInterpreter(); EXPECT_TRUE(I); // Check if the default standard is c++14 @@ -198,7 +213,7 @@ TEST(InterpreterTest, CreateInterpreter) { EXPECT_TRUE(Cpp::GetNamed("cpp14")); EXPECT_FALSE(Cpp::GetNamed("cppUnknown")); - I = Cpp::CreateInterpreter({"-std=c++17"}); + I = TestUtils::CreateInterpreter({"-std=c++17"}); Cpp::Declare("#if __cplusplus==201703L\n" "int cpp17() { return 2017; }\n" "#else\n" @@ -250,7 +265,7 @@ TEST(InterpreterTest, DISABLED_DetectResourceDir) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif - auto* I = Cpp::CreateInterpreter(); + auto* I = TestUtils::CreateInterpreter(); EXPECT_STRNE(Cpp::DetectResourceDir().c_str(), Cpp::GetResourceDir()); llvm::SmallString<256> Clang(LLVM_BINARY_DIR); llvm::sys::path::append(Clang, "bin", "clang"); @@ -275,6 +290,9 @@ TEST(InterpreterTest, DetectSystemCompilerIncludePaths) { } TEST(InterpreterTest, IncludePaths) { + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector includes; Cpp::GetIncludePaths(includes); EXPECT_FALSE(includes.empty()); @@ -300,7 +318,7 @@ TEST(InterpreterTest, IncludePaths) { TEST(InterpreterTest, CodeCompletion) { #if CLANG_VERSION_MAJOR >= 18 || defined(CPPINTEROP_USE_CLING) - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); std::vector cc; Cpp::Declare("int foo = 12;"); Cpp::CodeComplete(cc, "f", 1, 2); @@ -380,11 +398,14 @@ TEST(InterpreterTest, MultipleInterpreter) { GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif GTEST_SKIP() << "Test does not consistently pass so skipping for now"; + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } // delete all old interpreters while (Cpp::DeleteInterpreter()) ; std::vector interpreter_args = {"-include", "new"}; - auto* I1 = Cpp::CreateInterpreter(interpreter_args); + auto* I1 = TestUtils::CreateInterpreter(interpreter_args); EXPECT_TRUE(I1); auto F = [](Cpp::TInterp_t I) { @@ -429,9 +450,9 @@ TEST(InterpreterTest, MultipleInterpreter) { }; F(I1); - auto* I2 = Cpp::CreateInterpreter(interpreter_args); - auto* I3 = Cpp::CreateInterpreter(interpreter_args); - auto* I4 = Cpp::CreateInterpreter(interpreter_args); + auto* I2 = TestUtils::CreateInterpreter(interpreter_args); + auto* I3 = TestUtils::CreateInterpreter(interpreter_args); + auto* I4 = TestUtils::CreateInterpreter(interpreter_args); EXPECT_TRUE(I2); EXPECT_TRUE(I3); EXPECT_TRUE(I4); diff --git a/unittests/CppInterOp/JitTest.cpp b/unittests/CppInterOp/JitTest.cpp index a1b6909b7..7db47c766 100644 --- a/unittests/CppInterOp/JitTest.cpp +++ b/unittests/CppInterOp/JitTest.cpp @@ -20,6 +20,9 @@ TEST(JitTest, InsertOrReplaceJitSymbol) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } std::vector Decls; std::string code = R"( extern "C" int printf(const char*,...); @@ -40,6 +43,9 @@ TEST(JitTest, InsertOrReplaceJitSymbol) { } TEST(Streams, StreamRedirect) { + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } // printf and etc are fine here. // NOLINTBEGIN(cppcoreguidelines-pro-type-vararg) Cpp::BeginStdStreamCapture(Cpp::kStdOut); @@ -70,3 +76,31 @@ TEST(Streams, StreamRedirect) { EXPECT_STREQ(cerrs.c_str(), "Err\nStdErr\n"); // NOLINTEND(cppcoreguidelines-pro-type-vararg) } + +TEST(Streams, StreamRedirectJIT) { +#ifdef EMSCRIPTEN + GTEST_SKIP() << "Test fails for Emscipten builds"; +#endif + if (llvm::sys::RunningOnValgrind()) + GTEST_SKIP() << "XFAIL due to Valgrind report"; +#ifdef _WIN32 + GTEST_SKIP() << "Disabled on Windows. Needs fixing."; +#endif +#ifdef CPPINTEROP_USE_CLING + GTEST_SKIP() << "Test fails for cling builds"; +#endif + TestUtils::CreateInterpreter(); + + Cpp::BeginStdStreamCapture(Cpp::kStdOut); + Cpp::BeginStdStreamCapture(Cpp::kStdErr); + Interp->process(R"( + #include + printf("%s\n", "Hello World"); + fflush(stdout); + )"); + std::string CapturedStringErr = Cpp::EndStdStreamCapture(); + std::string CapturedStringOut = Cpp::EndStdStreamCapture(); + + EXPECT_STREQ(CapturedStringOut.c_str(), "Hello World\n"); + EXPECT_STREQ(CapturedStringErr.c_str(), ""); +} diff --git a/unittests/CppInterOp/ScopeReflectionTest.cpp b/unittests/CppInterOp/ScopeReflectionTest.cpp index 53b2ee894..e686c05ba 100644 --- a/unittests/CppInterOp/ScopeReflectionTest.cpp +++ b/unittests/CppInterOp/ScopeReflectionTest.cpp @@ -176,7 +176,7 @@ TEST(ScopeReflectionTest, IsBuiltin) { std::vector interpreter_args = { "-include", "new" }; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); ASTContext &C = Interp->getCI()->getASTContext(); EXPECT_TRUE(Cpp::IsBuiltin(C.BoolTy.getAsOpaquePtr())); EXPECT_TRUE(Cpp::IsBuiltin(C.CharTy.getAsOpaquePtr())); @@ -476,7 +476,7 @@ TEST(ScopeReflectionTest, GetScope) { typedef N::C T; )"; - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); Interp->declare(code); Cpp::TCppScope_t tu = Cpp::GetScope("", 0); Cpp::TCppScope_t ns_N = Cpp::GetScope("N", 0); @@ -501,7 +501,7 @@ TEST(ScopeReflectionTest, GetScopefromCompleteName) { } )"; - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); Interp->declare(code); EXPECT_EQ(Cpp::GetQualifiedName(Cpp::GetScopeFromCompleteName("N1")), "N1"); @@ -528,7 +528,7 @@ TEST(ScopeReflectionTest, GetNamed) { std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Interp->declare(code); Cpp::TCppScope_t ns_N1 = Cpp::GetNamed("N1", nullptr); @@ -567,7 +567,7 @@ TEST(ScopeReflectionTest, GetParentScope) { } )"; - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); Interp->declare(code); Cpp::TCppScope_t ns_N1 = Cpp::GetNamed("N1"); @@ -917,7 +917,7 @@ TEST(ScopeReflectionTest, InstantiateTemplateFunctionFromString) { if (llvm::sys::RunningOnValgrind()) GTEST_SKIP() << "XFAIL due to Valgrind report"; std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); std::string code = R"(#include )"; Interp->process(code); const char* str = "std::make_unique"; @@ -1066,7 +1066,7 @@ TEST(ScopeReflectionTest, IncludeVector) { #include )"; std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Interp->declare(code); } @@ -1074,7 +1074,7 @@ TEST(ScopeReflectionTest, GetOperator) { if (llvm::sys::RunningOnValgrind()) GTEST_SKIP() << "XFAIL due to Valgrind report"; - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); std::string code = R"( class MyClass { diff --git a/unittests/CppInterOp/TypeReflectionTest.cpp b/unittests/CppInterOp/TypeReflectionTest.cpp index 215d30f10..dc1e75af2 100644 --- a/unittests/CppInterOp/TypeReflectionTest.cpp +++ b/unittests/CppInterOp/TypeReflectionTest.cpp @@ -109,7 +109,7 @@ TEST(TypeReflectionTest, GetCanonicalType) { } TEST(TypeReflectionTest, GetType) { - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); std::string code = R"( class A {}; @@ -346,7 +346,7 @@ TEST(TypeReflectionTest, IsUnderlyingTypeRecordType) { } TEST(TypeReflectionTest, GetComplexType) { - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); auto get_complex_type_as_string = [&](const std::string &element_type) { auto ElementQT = Cpp::GetType(element_type); @@ -559,7 +559,7 @@ TEST(TypeReflectionTest, IsSmartPtrType) { GTEST_SKIP() << "XFAIL due to Valgrind report"; std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Interp->declare(R"( #include @@ -598,7 +598,7 @@ TEST(TypeReflectionTest, IsSmartPtrType) { TEST(TypeReflectionTest, IsFunctionPointerType) { std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Interp->declare(R"( typedef int (*int_func)(int, int); @@ -622,7 +622,7 @@ TEST(TypeReflectionTest, OperatorSpelling) { } TEST(TypeReflectionTest, TypeQualifiers) { - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); Cpp::Declare(R"( int *a; int *__restrict__ b; diff --git a/unittests/CppInterOp/Utils.cpp b/unittests/CppInterOp/Utils.cpp index e048b0df5..1f29ce3fc 100644 --- a/unittests/CppInterOp/Utils.cpp +++ b/unittests/CppInterOp/Utils.cpp @@ -6,6 +6,7 @@ #include "clang/AST/DeclCXX.h" #include "clang/Basic/Version.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Interpreter/PartialTranslationUnit.h" #include "clang/Sema/Lookup.h" #include "clang/Sema/Sema.h" @@ -16,11 +17,16 @@ using namespace clang; using namespace llvm; +bool& TestUtils::use_oop_jit() { + static bool flag = false; + return flag; +} + void TestUtils::GetAllTopLevelDecls( const std::string& code, std::vector& Decls, bool filter_implicitGenerated /* = false */, const std::vector& interpreter_args /* = {} */) { - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); #ifdef CPPINTEROP_USE_CLING cling::Transaction *T = nullptr; Interp->declare(code, &T); @@ -57,6 +63,16 @@ void TestUtils::GetAllSubDecls(Decl *D, std::vector& SubDecls, } } +TInterp_t +TestUtils::CreateInterpreter(const std::vector& Args, + const std::vector& GpuArgs) { + auto mergedArgs = Args; + if (TestUtils::use_oop_jit()) { + mergedArgs.push_back("--use-oop-jit"); + } + return Cpp::CreateInterpreter(mergedArgs, GpuArgs); +} + const char* get_c_string(CXString string) { return static_cast(string.data); } diff --git a/unittests/CppInterOp/Utils.h b/unittests/CppInterOp/Utils.h index 2b7b12590..c51459eed 100644 --- a/unittests/CppInterOp/Utils.h +++ b/unittests/CppInterOp/Utils.h @@ -20,12 +20,15 @@ namespace clang { } #define Interp (static_cast(Cpp::GetInterpreter())) namespace TestUtils { +bool& use_oop_jit(); void GetAllTopLevelDecls(const std::string& code, std::vector& Decls, bool filter_implicitGenerated = false, const std::vector& interpreter_args = {}); void GetAllSubDecls(clang::Decl* D, std::vector& SubDecls, bool filter_implicitGenerated = false); +TInterp_t CreateInterpreter(const std::vector& Args = {}, + const std::vector& GpuArgs = {}); } // end namespace TestUtils const char* get_c_string(CXString string); diff --git a/unittests/CppInterOp/VariableReflectionTest.cpp b/unittests/CppInterOp/VariableReflectionTest.cpp index 5c9829316..a22f619d1 100644 --- a/unittests/CppInterOp/VariableReflectionTest.cpp +++ b/unittests/CppInterOp/VariableReflectionTest.cpp @@ -180,7 +180,7 @@ TEST(VariableReflectionTest, GetTypeAsString) { } )"; - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); EXPECT_EQ(Cpp::Declare(code.c_str()), 0); Cpp::TCppScope_t wrapper = @@ -390,7 +390,7 @@ TEST(VariableReflectionTest, VariableOffsetsWithInheritance) { GTEST_SKIP() << "XFAIL due to Valgrind report"; std::vector interpreter_args = {"-include", "new"}; - Cpp::CreateInterpreter(interpreter_args); + TestUtils::CreateInterpreter(interpreter_args); Cpp::Declare("#include"); @@ -567,7 +567,7 @@ TEST(VariableReflectionTest, StaticConstExprDatamember) { GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); Cpp::Declare(R"( class MyClass { @@ -639,7 +639,7 @@ TEST(VariableReflectionTest, StaticConstExprDatamember) { } TEST(VariableReflectionTest, GetEnumConstantDatamembers) { - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); Cpp::Declare(R"( class MyEnumClass { @@ -663,7 +663,7 @@ TEST(VariableReflectionTest, GetEnumConstantDatamembers) { } TEST(VariableReflectionTest, Is_Get_Pointer) { - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); std::vector Decls; std::string code = R"( class A {}; @@ -695,7 +695,7 @@ TEST(VariableReflectionTest, Is_Get_Pointer) { } TEST(VariableReflectionTest, Is_Get_Reference) { - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); std::vector Decls; std::string code = R"( class A {}; @@ -733,7 +733,7 @@ TEST(VariableReflectionTest, Is_Get_Reference) { } TEST(VariableReflectionTest, GetPointerType) { - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); std::vector Decls; std::string code = R"( class A {}; diff --git a/unittests/CppInterOp/main.cpp b/unittests/CppInterOp/main.cpp index 09799f80a..792bcff09 100644 --- a/unittests/CppInterOp/main.cpp +++ b/unittests/CppInterOp/main.cpp @@ -1,6 +1,18 @@ +#include "Utils.h" #include +#include + +static void parseCustomArguments(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + std::string arg(argv[i]); + if (arg == "--use-oop-jit") { + TestUtils::use_oop_jit() = true; + } + } +} int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); + parseCustomArguments(argc, argv); return RUN_ALL_TESTS(); } \ No newline at end of file From 72fee63148696b2ed687836ed7cb14446e431637 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Sat, 20 Sep 2025 18:59:31 +0530 Subject: [PATCH 02/15] Fixing syntax error in CI --- .github/actions/Build_and_Test_CppInterOp/action.yml | 1 + Emscripten-build-instructions.md | 3 ++- README.md | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/actions/Build_and_Test_CppInterOp/action.yml b/.github/actions/Build_and_Test_CppInterOp/action.yml index c1387b4d9..20e28a4a2 100644 --- a/.github/actions/Build_and_Test_CppInterOp/action.yml +++ b/.github/actions/Build_and_Test_CppInterOp/action.yml @@ -63,6 +63,7 @@ runs: fi if [[ "${{ matrix.oop-jit }}" == "On" ]]; then ./unittests/CppInterOp/CppInterOpTests/unittests/bin/${{ env.BUILD_TYPE }}/CppInterOpTests --use-oop-jit + fi fi echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV echo "CPPINTEROP_BUILD_DIR=$CPPINTEROP_BUILD_DIR" >> $GITHUB_ENV diff --git a/Emscripten-build-instructions.md b/Emscripten-build-instructions.md index 0854fed2d..d546604ed 100644 --- a/Emscripten-build-instructions.md +++ b/Emscripten-build-instructions.md @@ -20,6 +20,7 @@ cd ./CppInterOp-wasm To create a wasm build of CppInterOp we make use of the emsdk toolchain. This can be installed by executing (we only currently support version 3.1.73) + ```bash git clone https://github.com/emscripten-core/emsdk.git ./emsdk/emsdk install 3.1.73 @@ -69,6 +70,7 @@ git apply -v emscripten-clang20-3-enable_exception_handling.patch We are now in a position to build an emscripten build of llvm by executing the following on Linux and osx + ```bash mkdir native_build cd native_build @@ -221,7 +223,6 @@ emcmake cmake -DCMAKE_BUILD_TYPE=Release ` It is possible to run the Emscripten tests in a headless browser. To do this we will first move to the tests directory - ```bash cd ./unittests/CppInterOp/ ``` diff --git a/README.md b/README.md index abd5ee0fc..8b60e91c7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # CppInterOp +
[![Build Status](https://github.com/compiler-research/CppInterOp/actions/workflows/main.yml/badge.svg)](https://github.com/compiler-research/CppInterOp/actions/workflows/main.yml) @@ -213,6 +214,7 @@ git clone --depth=1 https://github.com/compiler-research/cppyy-backend.git To have ``Out-of-Process JIT Execution`` enabled, run following commands to build clang and clang-repl to support this feature: > Only for Linux x86_64 and Macos amr64 + ```bash mkdir build cd build @@ -228,10 +230,13 @@ cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt" \ ``` ## For Linux x86_64 + ```bash cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all) ``` + ## For MacOS arm64 + ```bash cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu) ``` From d77c457d4695e9e3fff736c8efa4f5082bb292c6 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Sat, 20 Sep 2025 19:06:04 +0530 Subject: [PATCH 03/15] Adding strne in debug flag --- unittests/CppInterOp/InterpreterTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/CppInterOp/InterpreterTest.cpp b/unittests/CppInterOp/InterpreterTest.cpp index 551f9cf2f..f86edd186 100644 --- a/unittests/CppInterOp/InterpreterTest.cpp +++ b/unittests/CppInterOp/InterpreterTest.cpp @@ -52,7 +52,7 @@ TEST(InterpreterTest, DebugFlag) { testing::internal::CaptureStderr(); Cpp::Process("int b = 12;"); cerrs = testing::internal::GetCapturedStderr(); - EXPECT_STREQ(cerrs.c_str(), ""); + EXPECT_STRNE(cerrs.c_str(), ""); Cpp::EnableDebugOutput(false); EXPECT_FALSE(Cpp::IsDebugOutputEnabled()); From 67424336d3d85819fb191b72c62108b10d015500 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Sat, 20 Sep 2025 19:51:10 +0530 Subject: [PATCH 04/15] Creating Interpreter in Evaluate --- unittests/CppInterOp/InterpreterTest.cpp | 70 ++++++++++++------------ 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/unittests/CppInterOp/InterpreterTest.cpp b/unittests/CppInterOp/InterpreterTest.cpp index f86edd186..28cd3799d 100644 --- a/unittests/CppInterOp/InterpreterTest.cpp +++ b/unittests/CppInterOp/InterpreterTest.cpp @@ -76,16 +76,21 @@ TEST(InterpreterTest, Evaluate) { GTEST_SKIP() << "Test fails for OOP JIT builds"; } // EXPECT_TRUE(Cpp::Evaluate(I, "") == 0); - //EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); + // EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); // Due to a deficiency in the clang-repl implementation to get the value we // always must omit the ; + + // this needs to be added because TestUtils::CreateInterpreter creates + // Cpp::CreateInterpreter. But, the later goes out of scope(destroyed) at the + // end of every test due to TestUtils::CreateInterpreter. + TestUtils::CreateInterpreter(); EXPECT_TRUE(Cpp::Evaluate("__cplusplus") == 201402); bool HadError; EXPECT_TRUE(Cpp::Evaluate("#error", &HadError) == (intptr_t)~0UL); EXPECT_TRUE(HadError); EXPECT_EQ(Cpp::Evaluate("int i = 11; ++i", &HadError), 12); - EXPECT_FALSE(HadError) ; + EXPECT_FALSE(HadError); } TEST(InterpreterTest, DeleteInterpreter) { @@ -147,7 +152,7 @@ TEST(InterpreterTest, Process) { #endif if (llvm::sys::RunningOnValgrind()) GTEST_SKIP() << "XFAIL due to Valgrind report"; - std::vector interpreter_args = { "-include", "new" }; + std::vector interpreter_args = {"-include", "new"}; auto* I = TestUtils::CreateInterpreter(interpreter_args); EXPECT_TRUE(Cpp::Process("") == 0); EXPECT_TRUE(Cpp::Process("int a = 12;") == 0); @@ -172,20 +177,18 @@ TEST(InterpreterTest, Process) { TEST(InterpreterTest, EmscriptenExceptionHandling) { #ifndef EMSCRIPTEN - GTEST_SKIP() << "This test is intended to check exception handling for Emscripten builds."; + GTEST_SKIP() << "This test is intended to check exception handling for " + "Emscripten builds."; #endif if (TestUtils::use_oop_jit()) { GTEST_SKIP() << "Test fails for OOP JIT builds"; } std::vector Args = { - "-std=c++20", - "-v", - "-fexceptions", - "-fcxx-exceptions", - "-mllvm", "-enable-emscripten-cxx-exceptions", - "-mllvm", "-enable-emscripten-sjlj" - }; + "-std=c++20", "-v", + "-fexceptions", "-fcxx-exceptions", + "-mllvm", "-enable-emscripten-cxx-exceptions", + "-mllvm", "-enable-emscripten-sjlj"}; TestUtils::CreateInterpreter(Args); @@ -206,23 +209,22 @@ TEST(InterpreterTest, CreateInterpreter) { // Check if the default standard is c++14 Cpp::Declare("#if __cplusplus==201402L\n" - "int cpp14() { return 2014; }\n" - "#else\n" - "void cppUnknown() {}\n" - "#endif"); + "int cpp14() { return 2014; }\n" + "#else\n" + "void cppUnknown() {}\n" + "#endif"); EXPECT_TRUE(Cpp::GetNamed("cpp14")); EXPECT_FALSE(Cpp::GetNamed("cppUnknown")); I = TestUtils::CreateInterpreter({"-std=c++17"}); Cpp::Declare("#if __cplusplus==201703L\n" - "int cpp17() { return 2017; }\n" - "#else\n" - "void cppUnknown() {}\n" - "#endif"); + "int cpp17() { return 2017; }\n" + "#else\n" + "void cppUnknown() {}\n" + "#endif"); EXPECT_TRUE(Cpp::GetNamed("cpp17")); EXPECT_FALSE(Cpp::GetNamed("cppUnknown")); - #ifndef CPPINTEROP_USE_CLING // C API auto* CXI = clang_createInterpreterFromRawPtr(I); @@ -238,7 +240,7 @@ TEST(InterpreterTest, CreateInterpreter) { #ifndef CPPINTEROP_USE_CLING TEST(InterpreterTest, CreateInterpreterCAPI) { const char* argv[] = {"-std=c++17"}; - auto *CXI = clang_createInterpreter(argv, 1); + auto* CXI = clang_createInterpreter(argv, 1); auto CLI = clang_Interpreter_getClangInterpreter(CXI); EXPECT_TRUE(CLI); clang_Interpreter_dispose(CXI); @@ -249,7 +251,7 @@ TEST(InterpreterTest, CreateInterpreterCAPIFailure) { GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif const char* argv[] = {"-fsyntax-only", "-Xclang", "-invalid-plugin"}; - auto *CXI = clang_createInterpreter(argv, 3); + auto* CXI = clang_createInterpreter(argv, 3); EXPECT_EQ(CXI, nullptr); } #endif @@ -313,7 +315,7 @@ TEST(InterpreterTest, IncludePaths) { Cpp::AddIncludePath("/non/existent/"); Cpp::GetIncludePaths(includes); EXPECT_NE(std::find(includes.begin(), includes.end(), "/non/existent/"), - std::end(includes)); + std::end(includes)); } TEST(InterpreterTest, CodeCompletion) { @@ -338,8 +340,8 @@ TEST(InterpreterTest, CodeCompletion) { TEST(InterpreterTest, ExternalInterpreterTest) { -if (llvm::sys::RunningOnValgrind()) - GTEST_SKIP() << "XFAIL due to Valgrind report"; + if (llvm::sys::RunningOnValgrind()) + GTEST_SKIP() << "XFAIL due to Valgrind report"; #ifndef CPPINTEROP_USE_CLING llvm::ExitOnError ExitOnErr; @@ -359,15 +361,15 @@ if (llvm::sys::RunningOnValgrind()) #endif // CPPINTEROP_USE_REPL #ifdef CPPINTEROP_USE_CLING - std::string MainExecutableName = sys::fs::getMainExecutable(nullptr, nullptr); - llvm::SmallString<128> P(LLVM_BINARY_DIR); - llvm::sys::path::append(P, CLANG_INSTALL_LIBDIR_BASENAME, "clang", - CLANG_VERSION_MAJOR_STRING); - std::string ResourceDir = std::string(P.str()); - std::vector ClingArgv = {"-resource-dir", ResourceDir.c_str(), - "-std=c++14"}; - ClingArgv.insert(ClingArgv.begin(), MainExecutableName.c_str()); - auto *ExtInterp = new compat::Interpreter(ClingArgv.size(), &ClingArgv[0]); + std::string MainExecutableName = sys::fs::getMainExecutable(nullptr, nullptr); + llvm::SmallString<128> P(LLVM_BINARY_DIR); + llvm::sys::path::append(P, CLANG_INSTALL_LIBDIR_BASENAME, "clang", + CLANG_VERSION_MAJOR_STRING); + std::string ResourceDir = std::string(P.str()); + std::vector ClingArgv = {"-resource-dir", ResourceDir.c_str(), + "-std=c++14"}; + ClingArgv.insert(ClingArgv.begin(), MainExecutableName.c_str()); + auto* ExtInterp = new compat::Interpreter(ClingArgv.size(), &ClingArgv[0]); #endif EXPECT_NE(ExtInterp, nullptr); From 6cf1da4c542f1de7d018e0119e6e07460dc50d6d Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Sat, 20 Sep 2025 20:02:44 +0530 Subject: [PATCH 05/15] Creating Interpreter in Evaluate --- unittests/CppInterOp/InterpreterTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/CppInterOp/InterpreterTest.cpp b/unittests/CppInterOp/InterpreterTest.cpp index 28cd3799d..a6c944b65 100644 --- a/unittests/CppInterOp/InterpreterTest.cpp +++ b/unittests/CppInterOp/InterpreterTest.cpp @@ -83,7 +83,7 @@ TEST(InterpreterTest, Evaluate) { // this needs to be added because TestUtils::CreateInterpreter creates // Cpp::CreateInterpreter. But, the later goes out of scope(destroyed) at the // end of every test due to TestUtils::CreateInterpreter. - TestUtils::CreateInterpreter(); + Cpp::CreateInterpreter(); EXPECT_TRUE(Cpp::Evaluate("__cplusplus") == 201402); bool HadError; From 7217df907e27d7db54369d5ee91c9779c2427749 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Sat, 20 Sep 2025 20:17:18 +0530 Subject: [PATCH 06/15] [experimental] removing evaluate test --- unittests/CppInterOp/InterpreterTest.cpp | 58 ++++++++++++------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/unittests/CppInterOp/InterpreterTest.cpp b/unittests/CppInterOp/InterpreterTest.cpp index a6c944b65..104f7d03a 100644 --- a/unittests/CppInterOp/InterpreterTest.cpp +++ b/unittests/CppInterOp/InterpreterTest.cpp @@ -63,35 +63,35 @@ TEST(InterpreterTest, DebugFlag) { EXPECT_STREQ(cerrs.c_str(), ""); } -TEST(InterpreterTest, Evaluate) { -#ifdef EMSCRIPTEN - GTEST_SKIP() << "Test fails for Emscipten builds"; -#endif -#ifdef _WIN32 - GTEST_SKIP() << "Disabled on Windows. Needs fixing."; -#endif - if (llvm::sys::RunningOnValgrind()) - GTEST_SKIP() << "XFAIL due to Valgrind report"; - if (TestUtils::use_oop_jit()) { - GTEST_SKIP() << "Test fails for OOP JIT builds"; - } - // EXPECT_TRUE(Cpp::Evaluate(I, "") == 0); - // EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); - // Due to a deficiency in the clang-repl implementation to get the value we - // always must omit the ; - - // this needs to be added because TestUtils::CreateInterpreter creates - // Cpp::CreateInterpreter. But, the later goes out of scope(destroyed) at the - // end of every test due to TestUtils::CreateInterpreter. - Cpp::CreateInterpreter(); - EXPECT_TRUE(Cpp::Evaluate("__cplusplus") == 201402); - - bool HadError; - EXPECT_TRUE(Cpp::Evaluate("#error", &HadError) == (intptr_t)~0UL); - EXPECT_TRUE(HadError); - EXPECT_EQ(Cpp::Evaluate("int i = 11; ++i", &HadError), 12); - EXPECT_FALSE(HadError); -} +// TEST(InterpreterTest, Evaluate) { +// #ifdef EMSCRIPTEN +// GTEST_SKIP() << "Test fails for Emscipten builds"; +// #endif +// #ifdef _WIN32 +// GTEST_SKIP() << "Disabled on Windows. Needs fixing."; +// #endif +// if (llvm::sys::RunningOnValgrind()) +// GTEST_SKIP() << "XFAIL due to Valgrind report"; +// if (TestUtils::use_oop_jit()) { +// GTEST_SKIP() << "Test fails for OOP JIT builds"; +// } +// // EXPECT_TRUE(Cpp::Evaluate(I, "") == 0); +// // EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); +// // Due to a deficiency in the clang-repl implementation to get the value we +// // always must omit the ; + +// // this needs to be added because TestUtils::CreateInterpreter creates +// // Cpp::CreateInterpreter. But, the later goes out of scope(destroyed) at the +// // end of every test due to TestUtils::CreateInterpreter. +// Cpp::CreateInterpreter(); +// EXPECT_TRUE(Cpp::Evaluate("__cplusplus") == 201402); + +// bool HadError; +// EXPECT_TRUE(Cpp::Evaluate("#error", &HadError) == (intptr_t)~0UL); +// EXPECT_TRUE(HadError); +// EXPECT_EQ(Cpp::Evaluate("int i = 11; ++i", &HadError), 12); +// EXPECT_FALSE(HadError); +// } TEST(InterpreterTest, DeleteInterpreter) { if (TestUtils::use_oop_jit()) { From 334b02e925ade96d192eccbfb09cf93c21565ec5 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Wed, 1 Oct 2025 22:56:55 +0530 Subject: [PATCH 07/15] Fixing llvm 20 patch --- .github/actions/Build_LLVM/action.yml | 2 +- README.md | 16 ++--- lib/CppInterOp/CppInterOpInterpreter.h | 4 +- ...s.patch => clang20-3-out-of-process.patch} | 26 ++++----- unittests/CppInterOp/InterpreterTest.cpp | 58 +++++++++---------- 5 files changed, 51 insertions(+), 55 deletions(-) rename patches/llvm/{clang20-2-out-of-process.patch => clang20-3-out-of-process.patch} (98%) diff --git a/.github/actions/Build_LLVM/action.yml b/.github/actions/Build_LLVM/action.yml index 38fcf6c85..2a6529604 100644 --- a/.github/actions/Build_LLVM/action.yml +++ b/.github/actions/Build_LLVM/action.yml @@ -68,7 +68,7 @@ runs: elif [[ "${{ matrix.os }}" == ubuntu* ]]; then SUFFIX="-x86_64" fi - ninja clang-repl llvm-jitlink-executor orc_rt${SUFFIX} -j ${{ env.ncpus }} + ninja llvm-jitlink-executor orc_rt${SUFFIX} -j ${{ env.ncpus }} fi cd ./tools/ rm -rf $(find . -maxdepth 1 ! -name "clang" ! -name ".") diff --git a/README.md b/README.md index 8b60e91c7..6f13ab1a1 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ cd llvm-project ``` If you want to have out-of-process JIT execution enabled in CppInterOp, then apply this patch on Linux and MacOS environment. -> Note that this patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux and MacOS only. +> Note that this patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux-x86_64 and MacOS-Darwin only. ```bash git apply -v ../CppInterOp/patches/llvm/clang20-2-out-of-process.patch @@ -306,13 +306,6 @@ mkdir CppInterOp/build/ cd CppInterOp/build/ ``` -On Windows execute - -```powershell -mkdir CppInterOp\build\ -cd CppInterOp\build\ -``` - Now if you want to build CppInterOp with Clang-REPL then execute the following commands on Linux and MacOS ```bash @@ -324,6 +317,13 @@ and > Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled. +On Windows execute + +```powershell +mkdir CppInterOp\build\ +cd CppInterOp\build\ +``` + ```powershell cmake -DLLVM_DIR=$env:LLVM_DIR\build\lib\cmake\llvm -DClang_DIR=$env:LLVM_DIR\build\lib\cmake\clang -DCMAKE_INSTALL_PREFIX=$env:CPPINTEROP_DIR .. cmake --build . --target install --parallel $env:ncpus diff --git a/lib/CppInterOp/CppInterOpInterpreter.h b/lib/CppInterOp/CppInterOpInterpreter.h index 8aa88edff..e390836da 100644 --- a/lib/CppInterOp/CppInterOpInterpreter.h +++ b/lib/CppInterOp/CppInterOpInterpreter.h @@ -313,14 +313,14 @@ class Interpreter { return llvm::orc::ExecutorAddr(*AddrOrErr); } - pid_t getOutOfProcessExecutorPID() const { #ifndef _WIN32 + pid_t getOutOfProcessExecutorPID() const { #ifdef LLVM_BUILT_WITH_OOP_JIT return inner->getOutOfProcessExecutorPID(); -#endif #endif return 0; } +#endif /// \returns the \c ExecutorAddr of a given name as written in the object /// file. diff --git a/patches/llvm/clang20-2-out-of-process.patch b/patches/llvm/clang20-3-out-of-process.patch similarity index 98% rename from patches/llvm/clang20-2-out-of-process.patch rename to patches/llvm/clang20-3-out-of-process.patch index b86f1b4bf..d4a93ffe8 100644 --- a/patches/llvm/clang20-2-out-of-process.patch +++ b/patches/llvm/clang20-3-out-of-process.patch @@ -1,5 +1,5 @@ diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h -index f8663e3193a1..1f9553e6809a 100644 +index f8663e319..1f9553e68 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -20,8 +20,10 @@ @@ -98,7 +98,7 @@ index f8663e3193a1..1f9553e6809a 100644 return ValuePrintingInfo; } diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp -index 4d2adecaafce..45620fcd358c 100644 +index 4d2adecaa..45620fcd3 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -15,19 +15,36 @@ @@ -380,7 +380,7 @@ index 4d2adecaafce..45620fcd358c 100644 + } // namespace clang diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h -index 71d71bc3883e..56e83378f004 100644 +index 71d71bc38..56e83378f 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.h +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -13,13 +13,20 @@ @@ -447,7 +447,7 @@ index 71d71bc3883e..56e83378f004 100644 } // end namespace clang diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp -index 3b81f9d701b4..63fb0e175ac8 100644 +index 3b81f9d70..6ef46a942 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -46,6 +46,7 @@ @@ -477,7 +477,7 @@ index 3b81f9d701b4..63fb0e175ac8 100644 ErrOut = joinErrors(std::move(ErrOut), std::move(Err)); return; } -@@ -454,22 +456,120 @@ const char *const Runtimes = R"( +@@ -454,19 +456,118 @@ const char *const Runtimes = R"( EXTERN_C void __clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ...); )"; @@ -603,11 +603,8 @@ index 3b81f9d701b4..63fb0e175ac8 100644 + Interp->markUserCodeStart(); -- Interp->ValuePrintingInfo.resize(4); - return std::move(Interp); - } - -@@ -551,6 +651,12 @@ size_t Interpreter::getEffectivePTUSize() const { + Interp->ValuePrintingInfo.resize(4); +@@ -551,6 +652,12 @@ size_t Interpreter::getEffectivePTUSize() const { return PTUs.size() - InitPTUSize; } @@ -620,7 +617,7 @@ index 3b81f9d701b4..63fb0e175ac8 100644 PartialTranslationUnit & Interpreter::RegisterPTU(TranslationUnitDecl *TU, std::unique_ptr M /*={}*/, -@@ -617,7 +723,26 @@ createJITTargetMachineBuilder(const std::string &TT) { +@@ -617,7 +724,26 @@ createJITTargetMachineBuilder(const std::string &TT) { return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT)); } @@ -648,7 +645,7 @@ index 3b81f9d701b4..63fb0e175ac8 100644 if (IncrExecutor) return llvm::make_error("Operation failed. " "Execution engine exists", -@@ -626,8 +751,26 @@ llvm::Error Interpreter::CreateExecutor() { +@@ -626,8 +752,26 @@ llvm::Error Interpreter::CreateExecutor() { return llvm::make_error("Operation failed. " "No code generator available", std::error_code()); @@ -676,7 +673,7 @@ index 3b81f9d701b4..63fb0e175ac8 100644 auto JTMB = createJITTargetMachineBuilder(TT); if (!JTMB) return JTMB.takeError(); -@@ -638,11 +781,15 @@ llvm::Error Interpreter::CreateExecutor() { +@@ -638,11 +782,15 @@ llvm::Error Interpreter::CreateExecutor() { } llvm::Error Err = llvm::Error::success(); @@ -696,7 +693,7 @@ index 3b81f9d701b4..63fb0e175ac8 100644 if (!Err) IncrExecutor = std::move(Executor); diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp -index 7af8e4f25d99..c1a7ec397917 100644 +index 7af8e4f25..c1a7ec397 100644 --- a/clang/tools/clang-repl/ClangRepl.cpp +++ b/clang/tools/clang-repl/ClangRepl.cpp @@ -11,6 +11,8 @@ @@ -937,4 +934,3 @@ index 7af8e4f25d99..c1a7ec397917 100644 -} +} \ No newline at end of file - diff --git a/unittests/CppInterOp/InterpreterTest.cpp b/unittests/CppInterOp/InterpreterTest.cpp index 104f7d03a..a6c944b65 100644 --- a/unittests/CppInterOp/InterpreterTest.cpp +++ b/unittests/CppInterOp/InterpreterTest.cpp @@ -63,35 +63,35 @@ TEST(InterpreterTest, DebugFlag) { EXPECT_STREQ(cerrs.c_str(), ""); } -// TEST(InterpreterTest, Evaluate) { -// #ifdef EMSCRIPTEN -// GTEST_SKIP() << "Test fails for Emscipten builds"; -// #endif -// #ifdef _WIN32 -// GTEST_SKIP() << "Disabled on Windows. Needs fixing."; -// #endif -// if (llvm::sys::RunningOnValgrind()) -// GTEST_SKIP() << "XFAIL due to Valgrind report"; -// if (TestUtils::use_oop_jit()) { -// GTEST_SKIP() << "Test fails for OOP JIT builds"; -// } -// // EXPECT_TRUE(Cpp::Evaluate(I, "") == 0); -// // EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); -// // Due to a deficiency in the clang-repl implementation to get the value we -// // always must omit the ; - -// // this needs to be added because TestUtils::CreateInterpreter creates -// // Cpp::CreateInterpreter. But, the later goes out of scope(destroyed) at the -// // end of every test due to TestUtils::CreateInterpreter. -// Cpp::CreateInterpreter(); -// EXPECT_TRUE(Cpp::Evaluate("__cplusplus") == 201402); - -// bool HadError; -// EXPECT_TRUE(Cpp::Evaluate("#error", &HadError) == (intptr_t)~0UL); -// EXPECT_TRUE(HadError); -// EXPECT_EQ(Cpp::Evaluate("int i = 11; ++i", &HadError), 12); -// EXPECT_FALSE(HadError); -// } +TEST(InterpreterTest, Evaluate) { +#ifdef EMSCRIPTEN + GTEST_SKIP() << "Test fails for Emscipten builds"; +#endif +#ifdef _WIN32 + GTEST_SKIP() << "Disabled on Windows. Needs fixing."; +#endif + if (llvm::sys::RunningOnValgrind()) + GTEST_SKIP() << "XFAIL due to Valgrind report"; + if (TestUtils::use_oop_jit()) { + GTEST_SKIP() << "Test fails for OOP JIT builds"; + } + // EXPECT_TRUE(Cpp::Evaluate(I, "") == 0); + // EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); + // Due to a deficiency in the clang-repl implementation to get the value we + // always must omit the ; + + // this needs to be added because TestUtils::CreateInterpreter creates + // Cpp::CreateInterpreter. But, the later goes out of scope(destroyed) at the + // end of every test due to TestUtils::CreateInterpreter. + Cpp::CreateInterpreter(); + EXPECT_TRUE(Cpp::Evaluate("__cplusplus") == 201402); + + bool HadError; + EXPECT_TRUE(Cpp::Evaluate("#error", &HadError) == (intptr_t)~0UL); + EXPECT_TRUE(HadError); + EXPECT_EQ(Cpp::Evaluate("int i = 11; ++i", &HadError), 12); + EXPECT_FALSE(HadError); +} TEST(InterpreterTest, DeleteInterpreter) { if (TestUtils::use_oop_jit()) { From 3d9669db708d7880f3bcef6b00e40746b00a41cb Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Wed, 1 Oct 2025 23:03:21 +0530 Subject: [PATCH 08/15] Changed patch name --- .github/actions/Build_LLVM/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/Build_LLVM/action.yml b/.github/actions/Build_LLVM/action.yml index 2a6529604..468d0fe88 100644 --- a/.github/actions/Build_LLVM/action.yml +++ b/.github/actions/Build_LLVM/action.yml @@ -44,8 +44,8 @@ runs: ninja clingInterpreter -j ${{ env.ncpus }} else if [[ "${{ matrix.oop-jit }}" == "On" ]]; then - git apply -v ../patches/llvm/clang20-2-out-of-process.patch - echo "Apply clang20-2-out-of-process.patch:" + git apply -v ../patches/llvm/clang20-1-out-of-process.patch + echo "Apply clang20-1-out-of-process.patch:" fi cd build cmake -DLLVM_ENABLE_PROJECTS="${{ matrix.llvm_enable_projects}}" \ From ab50e0410a849b3762678d05e35a094f9af4f076 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Wed, 1 Oct 2025 23:09:32 +0530 Subject: [PATCH 09/15] Changed patch name --- ...ng20-3-out-of-process.patch => clang20-1-out-of-process.patch} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename patches/llvm/{clang20-3-out-of-process.patch => clang20-1-out-of-process.patch} (100%) diff --git a/patches/llvm/clang20-3-out-of-process.patch b/patches/llvm/clang20-1-out-of-process.patch similarity index 100% rename from patches/llvm/clang20-3-out-of-process.patch rename to patches/llvm/clang20-1-out-of-process.patch From c9a44261ebbe271bc3659a06fb678ab3bdd0df32 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Wed, 1 Oct 2025 23:27:21 +0530 Subject: [PATCH 10/15] Linting fix and patch name change --- README.md | 87 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 6f13ab1a1..aabcd81a3 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,45 @@ export LLVM_DIR=$PWD cd ../ ``` +##### Build Clang-REPL with Out-of-Process JIT Execution + +To have ``Out-of-Process JIT Execution`` enabled, run following commands to build clang and clang-repl to support this feature: +> Only for Linux x86_64 and Macos amr64 + +```bash +mkdir build +cd build +cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt" \ + -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ + -DCLANG_ENABLE_ARCMT=OFF \ + -DCLANG_ENABLE_FORMAT=OFF \ + -DCLANG_ENABLE_BOOTSTRAP=OFF \ + ../llvm +``` + +###### For Linux x86_64 + +```bash +cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all) +``` + +###### For MacOS arm64 + +```bash +cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu) +``` + +Note the 'llvm-project' directory location by executing + +```bash +cd ../ +export LLVM_DIR=$PWD +cd ../ +``` + #### Environment variables You will need to define the following environment variables for the build of CppInterOp and cppyy (as they clear for a new session, it is recommended that you also add these to your .bashrc in linux, .bash_profile if on MacOS). On Linux and MacOS you define as follows @@ -175,6 +214,10 @@ cmake -DBUILD_SHARED_LIBS=ON -DCPPINTEROP_USE_CLING=ON -DCPPINTEROP_USE_REPL=Off cmake --build . --target install --parallel $(nproc --all) ``` +and + +> Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled. + #### Testing CppInterOp To test the built CppInterOp execute the following command in the CppInterOP build folder on Linux and MacOS @@ -210,38 +253,7 @@ and clone cppyy-backend repository where we will be installing the CppInterOp li git clone --depth=1 https://github.com/compiler-research/cppyy-backend.git ``` -##### Build Clang-REPL with Out-of-Process JIT Execution - -To have ``Out-of-Process JIT Execution`` enabled, run following commands to build clang and clang-repl to support this feature: -> Only for Linux x86_64 and Macos amr64 - -```bash -mkdir build -cd build -cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt" \ - -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ - -DCLANG_ENABLE_ARCMT=OFF \ - -DCLANG_ENABLE_FORMAT=OFF \ - -DCLANG_ENABLE_BOOTSTRAP=OFF \ - ../llvm -``` - -## For Linux x86_64 - -```bash -cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all) -``` - -## For MacOS arm64 - -```bash -cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu) -``` - -#### Build Cling and related dependencies +### Build Cling and related dependencies The Cling interpreter and depends on its own customised version of `llvm-project`, hosted under the `root-project` (see the git path below). @@ -306,17 +318,6 @@ mkdir CppInterOp/build/ cd CppInterOp/build/ ``` -Now if you want to build CppInterOp with Clang-REPL then execute the following commands on Linux and MacOS - -```bash -cmake -DBUILD_SHARED_LIBS=ON -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. -cmake --build . --target install --parallel $(nproc --all) -``` - -and - -> Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled. - On Windows execute ```powershell From 9e18da633641f60fbdcca867c1342758e5f0f602 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Thu, 2 Oct 2025 01:44:28 +0530 Subject: [PATCH 11/15] Changing llvm patch --- patches/llvm/clang20-1-out-of-process.patch | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/patches/llvm/clang20-1-out-of-process.patch b/patches/llvm/clang20-1-out-of-process.patch index d4a93ffe8..198afb99a 100644 --- a/patches/llvm/clang20-1-out-of-process.patch +++ b/patches/llvm/clang20-1-out-of-process.patch @@ -934,3 +934,33 @@ index 7af8e4f25..c1a7ec397 100644 -} +} \ No newline at end of file +diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +index 972c24abc..a75a0afa7 100644 +--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp ++++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +@@ -635,16 +635,19 @@ Error ORCPlatformSupport::initialize(orc::JITDylib &JD) { + int32_t result; + auto E = ES.callSPSWrapper(WrapperAddr->getAddress(), + result, DSOHandles[&JD]); +- if (result) ++ if (E) ++ return E; ++ else if (result) + return make_error("dlupdate failed", + inconvertibleErrorCode()); +- return E; +- } +- return ES.callSPSWrapper(WrapperAddr->getAddress(), +- DSOHandles[&JD], JD.getName(), +- int32_t(ORC_RT_RTLD_LAZY)); ++ } else ++ return ES.callSPSWrapper(WrapperAddr->getAddress(), ++ DSOHandles[&JD], JD.getName(), ++ int32_t(ORC_RT_RTLD_LAZY)); + } else + return WrapperAddr.takeError(); ++ ++ return Error::success(); + } + + Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) { From 748d7c06d82e494030ecec0a86a191488e7f86a7 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Thu, 2 Oct 2025 07:59:40 +0530 Subject: [PATCH 12/15] OrcRuntime path hardcoding removed --- lib/CppInterOp/Compatibility.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/CppInterOp/Compatibility.h b/lib/CppInterOp/Compatibility.h index 69374440e..fd4432380 100644 --- a/lib/CppInterOp/Compatibility.h +++ b/lib/CppInterOp/Compatibility.h @@ -291,9 +291,17 @@ createClangInterpreter(std::vector& args, int stdin_fd = 0, setvbuf(stdout, nullptr, _IONBF, 0); setvbuf(stderr, nullptr, _IONBF, 0); }; - OutOfProcessConfig.OrcRuntimePath = - std::string(LLVM_SOURCE_DIR) + - "/build/lib/clang/20/lib/darwin/liborc_rt_osx.a"; +#ifdef __APPLE__ + std::string OrcRuntimePath = + std::string(LLVM_SOURCE_DIR) + "/build/lib/clang/" + + std::to_string(LLVM_VERSION_MAJOR) + "/lib/darwin/liborc_rt_osx.a"; +#else + std::string OrcRuntimePath = std::string(LLVM_SOURCE_DIR) + + "/build/lib/clang/" + + std::to_string(LLVM_VERSION_MAJOR) + + "/lib/x86_64-unknown-linux-gnu/liborc_rt.a"; +#endif + OutOfProcessConfig.OrcRuntimePath = OrcRuntimePath; } auto innerOrErr = CudaEnabled From bf2637eeb7bd7aa85e7893414c510e258ccb94d1 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Thu, 2 Oct 2025 17:52:07 +0530 Subject: [PATCH 13/15] Resolving comments and formatting --- .../Build_and_Test_CppInterOp/action.yml | 3 -- CMakeLists.txt | 4 +- Emscripten-build-instructions.md | 3 +- README.md | 2 +- docs/DevelopersDocumentation.rst | 41 +++++++++++++++++++ docs/InstallationAndUsage.rst | 2 +- lib/CppInterOp/Compatibility.h | 8 ++-- unittests/CMakeLists.txt | 10 +++++ unittests/CppInterOp/CMakeLists.txt | 7 ++-- unittests/CppInterOp/InterpreterTest.cpp | 8 +--- 10 files changed, 65 insertions(+), 23 deletions(-) diff --git a/.github/actions/Build_and_Test_CppInterOp/action.yml b/.github/actions/Build_and_Test_CppInterOp/action.yml index 20e28a4a2..5e9da2c54 100644 --- a/.github/actions/Build_and_Test_CppInterOp/action.yml +++ b/.github/actions/Build_and_Test_CppInterOp/action.yml @@ -61,9 +61,6 @@ runs: if [[ "${os}" != "macos"* ]]; then valgrind --show-error-list=yes --track-origins=yes --error-exitcode=1 unittests/CppInterOp/CppInterOpTests/unittests/bin/${{ env.BUILD_TYPE }}/CppInterOpTests fi - if [[ "${{ matrix.oop-jit }}" == "On" ]]; then - ./unittests/CppInterOp/CppInterOpTests/unittests/bin/${{ env.BUILD_TYPE }}/CppInterOpTests --use-oop-jit - fi fi echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV echo "CPPINTEROP_BUILD_DIR=$CPPINTEROP_BUILD_DIR" >> $GITHUB_ENV diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a1012a28..fa55fb0b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -298,8 +298,8 @@ include_directories(SYSTEM ${LLVM_INCLUDE_DIRS}) separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS}) add_definitions(${LLVM_DEFINITIONS_LIST}) -string(REGEX REPLACE "/build/lib/cmake/llvm$" "" LLVM_SOURCE_DIR "${LLVM_DIR}") -add_definitions(-DLLVM_SOURCE_DIR="${LLVM_SOURCE_DIR}") +string(REGEX REPLACE "/lib/cmake/llvm$" "" LLVM_BUILD_DIR "${LLVM_DIR}") +add_definitions(-DLLVM_BUILD_DIR="${LLVM_BUILD_DIR}") if(LLVM_BUILT_WITH_OOP_JIT) if((CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") OR diff --git a/Emscripten-build-instructions.md b/Emscripten-build-instructions.md index d546604ed..0854fed2d 100644 --- a/Emscripten-build-instructions.md +++ b/Emscripten-build-instructions.md @@ -20,7 +20,6 @@ cd ./CppInterOp-wasm To create a wasm build of CppInterOp we make use of the emsdk toolchain. This can be installed by executing (we only currently support version 3.1.73) - ```bash git clone https://github.com/emscripten-core/emsdk.git ./emsdk/emsdk install 3.1.73 @@ -70,7 +69,6 @@ git apply -v emscripten-clang20-3-enable_exception_handling.patch We are now in a position to build an emscripten build of llvm by executing the following on Linux and osx - ```bash mkdir native_build cd native_build @@ -223,6 +221,7 @@ emcmake cmake -DCMAKE_BUILD_TYPE=Release ` It is possible to run the Emscripten tests in a headless browser. To do this we will first move to the tests directory + ```bash cd ./unittests/CppInterOp/ ``` diff --git a/README.md b/README.md index aabcd81a3..72fdd2b86 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ If you want to have out-of-process JIT execution enabled in CppInterOp, then app > Note that this patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux-x86_64 and MacOS-Darwin only. ```bash -git apply -v ../CppInterOp/patches/llvm/clang20-2-out-of-process.patch +git apply -v ../CppInterOp/patches/llvm/clang20-1-out-of-process.patch ``` ##### Build Clang-REPL diff --git a/docs/DevelopersDocumentation.rst b/docs/DevelopersDocumentation.rst index e586cc5d8..08f73f95f 100644 --- a/docs/DevelopersDocumentation.rst +++ b/docs/DevelopersDocumentation.rst @@ -41,6 +41,15 @@ Clone the 20.x release of the LLVM project repository. git clone --depth=1 --branch release/20.x https://github.com/llvm/llvm-project.git cd llvm-project +If you want to have out-of-process JIT execution enabled in CppInterOp, then apply this patch on Linux and MacOS environment. +.. note:: + + This patch will not work for Windows because out-of-process JIT execution is currently implemented for Linux and MacOS only. + +.. code:: bash + + git apply -v ../CppInterOp/patches/llvm/clang20-1-out-of-process.patch + ****************** Build Clang-REPL ****************** @@ -99,6 +108,34 @@ On Windows you execute the following $env:LLVM_DIR= $PWD.Path cd ..\ +*************************************************** +Build Clang-REPL with Out-of-Process JIT Execution +*************************************************** + +To have `Out-of-Process JIT Execution` enabled, run following commands to build clang and clang-repl to support this feature: + +.. note:: + + Only for Linux x86_64 and Macos arm64 + +.. code:: bash + mkdir build + cd build + cmake -DLLVM_ENABLE_PROJECTS="clang;compiler-rt" \ + -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ + -DCLANG_ENABLE_ARCMT=OFF \ + -DCLANG_ENABLE_FORMAT=OFF \ + -DCLANG_ENABLE_BOOTSTRAP=OFF \ + ../llvm + + # For Linux x86_64 + cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt-x86_64 --parallel $(nproc --all) + # For MacOS arm64 + cmake --build . --target clang clang-repl llvm-jitlink-executor orc_rt_osx --parallel $(sysctl -n hw.ncpu) + ************************************** Build Cling and related dependencies ************************************** @@ -261,6 +298,10 @@ commands on Linux and MacOS cmake -DBUILD_SHARED_LIBS=ON -DLLVM_DIR=$LLVM_DIR/build/lib/cmake/llvm -DClang_DIR=$LLVM_DIR/build/lib/cmake/clang -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. cmake --build . --target install --parallel $(nproc --all) +.. note:: + + Do make sure to pass ``DLLVM_BUILT_WITH_OOP_JIT=ON``, if you want to have out-of-process JIT execution feature enabled. + and .. code:: powershell diff --git a/docs/InstallationAndUsage.rst b/docs/InstallationAndUsage.rst index 2924f6118..9bc4c5ade 100644 --- a/docs/InstallationAndUsage.rst +++ b/docs/InstallationAndUsage.rst @@ -48,7 +48,7 @@ If you want to have out-of-process JIT execution enabled in CppInterOp, then app .. code:: bash - git apply -v ../CppInterOp/patches/llvm/clang20-2-out-of-process.patch + git apply -v ../CppInterOp/patches/llvm/clang20-1-out-of-process.patch ****************** Build Clang-REPL diff --git a/lib/CppInterOp/Compatibility.h b/lib/CppInterOp/Compatibility.h index fd4432380..45eb5d6be 100644 --- a/lib/CppInterOp/Compatibility.h +++ b/lib/CppInterOp/Compatibility.h @@ -273,7 +273,7 @@ createClangInterpreter(std::vector& args, int stdin_fd = 0, if (outOfProcess) { OutOfProcessConfig.IsOutOfProcess = true; OutOfProcessConfig.OOPExecutor = - std::string(LLVM_SOURCE_DIR) + "/build/bin/llvm-jitlink-executor"; + std::string(LLVM_BUILD_DIR) + "/bin/llvm-jitlink-executor"; OutOfProcessConfig.UseSharedMemory = false; OutOfProcessConfig.SlabAllocateSize = 0; OutOfProcessConfig.CustomizeFork = [=] { // Lambda defined inline @@ -293,11 +293,11 @@ createClangInterpreter(std::vector& args, int stdin_fd = 0, }; #ifdef __APPLE__ std::string OrcRuntimePath = - std::string(LLVM_SOURCE_DIR) + "/build/lib/clang/" + + std::string(LLVM_BUILD_DIR) + "/lib/clang/" + std::to_string(LLVM_VERSION_MAJOR) + "/lib/darwin/liborc_rt_osx.a"; #else - std::string OrcRuntimePath = std::string(LLVM_SOURCE_DIR) + - "/build/lib/clang/" + + std::string OrcRuntimePath = std::string(LLVM_BUILD_DIR) + + "/lib/clang/" + std::to_string(LLVM_VERSION_MAJOR) + "/lib/x86_64-unknown-linux-gnu/liborc_rt.a"; #endif diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index f13b85f68..4ba940ae7 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -46,12 +46,22 @@ if(EMSCRIPTEN) add_test(NAME cppinterop-${name} COMMAND $ENV{EMSDK_NODE} ${name}.js) else() add_test(NAME cppinterop-${name} COMMAND ${name}) + if (LLVM_BUILT_WITH_OOP_JIT) + add_test(NAME cppinterop-oop-${name} COMMAND ${name} --use-oop-jit) + endif() endif() set_tests_properties(cppinterop-${name} PROPERTIES TIMEOUT "${TIMEOUT_VALUE}" ENVIRONMENT "CPLUS_INCLUDE_PATH=${CMAKE_BINARY_DIR}/etc" LABELS DEPENDS) + if (LLVM_BUILT_WITH_OOP_JIT) + set_tests_properties(cppinterop-oop-${name} PROPERTIES + TIMEOUT "${TIMEOUT_VALUE}" + ENVIRONMENT "CPLUS_INCLUDE_PATH=${CMAKE_BINARY_DIR}/etc" + LABELS + DEPENDS) + endif() # FIXME: Just call gtest_add_tests this function is available. #gtest_add_tests(${name} "${Arguments}" AUTO) endfunction() diff --git a/unittests/CppInterOp/CMakeLists.txt b/unittests/CppInterOp/CMakeLists.txt index 1b6566bb9..a87373a8c 100644 --- a/unittests/CppInterOp/CMakeLists.txt +++ b/unittests/CppInterOp/CMakeLists.txt @@ -10,7 +10,7 @@ else() set(EXTRA_PATH_TEST_BINARIES /CppInterOpTests/unittests/bin/$/) endif() -set(EXTRA_TEST_SOURCE_FILES main.cpp) +set(EXTRA_TEST_SOURCE_FILES Utils.cpp main.cpp) add_cppinterop_unittest(CppInterOpTests EnumReflectionTest.cpp @@ -19,7 +19,6 @@ add_cppinterop_unittest(CppInterOpTests JitTest.cpp ScopeReflectionTest.cpp TypeReflectionTest.cpp - Utils.cpp VariableReflectionTest.cpp ${EXTRA_TEST_SOURCE_FILES} ) @@ -86,11 +85,11 @@ export_executable_symbols(CppInterOpTests) unset(LLVM_LINK_COMPONENTS) if (NOT EMSCRIPTEN) - set(EXTRA_TEST_SOURCE_FILES "") + set(EXTRA_TEST_SOURCE_FILES Utils.cpp) set(EXTRA_PATH_TEST_BINARIES /TestSharedLib/unittests/bin/$/) endif() -add_cppinterop_unittest(DynamicLibraryManagerTests DynamicLibraryManagerTest.cpp ${EXTRA_TEST_SOURCE_FILES} Utils.cpp) +add_cppinterop_unittest(DynamicLibraryManagerTests DynamicLibraryManagerTest.cpp ${EXTRA_TEST_SOURCE_FILES}) target_link_libraries(DynamicLibraryManagerTests PRIVATE diff --git a/unittests/CppInterOp/InterpreterTest.cpp b/unittests/CppInterOp/InterpreterTest.cpp index a6c944b65..3f10548f9 100644 --- a/unittests/CppInterOp/InterpreterTest.cpp +++ b/unittests/CppInterOp/InterpreterTest.cpp @@ -76,14 +76,10 @@ TEST(InterpreterTest, Evaluate) { GTEST_SKIP() << "Test fails for OOP JIT builds"; } // EXPECT_TRUE(Cpp::Evaluate(I, "") == 0); - // EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); + //EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); // Due to a deficiency in the clang-repl implementation to get the value we // always must omit the ; - - // this needs to be added because TestUtils::CreateInterpreter creates - // Cpp::CreateInterpreter. But, the later goes out of scope(destroyed) at the - // end of every test due to TestUtils::CreateInterpreter. - Cpp::CreateInterpreter(); + TestUtils::CreateInterpreter(); EXPECT_TRUE(Cpp::Evaluate("__cplusplus") == 201402); bool HadError; From 9250d0fa850628c7f6bda20ee91e7517707d69e2 Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Thu, 2 Oct 2025 18:06:51 +0530 Subject: [PATCH 14/15] Formatting --- lib/CppInterOp/Compatibility.h | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/CppInterOp/Compatibility.h b/lib/CppInterOp/Compatibility.h index 45eb5d6be..a676ba5a1 100644 --- a/lib/CppInterOp/Compatibility.h +++ b/lib/CppInterOp/Compatibility.h @@ -291,6 +291,7 @@ createClangInterpreter(std::vector& args, int stdin_fd = 0, setvbuf(stdout, nullptr, _IONBF, 0); setvbuf(stderr, nullptr, _IONBF, 0); }; + #ifdef __APPLE__ std::string OrcRuntimePath = std::string(LLVM_BUILD_DIR) + "/lib/clang/" + From cfbb303d1083938cf4ab25bd1b95720ea88957df Mon Sep 17 00:00:00 2001 From: kr-2003 Date: Thu, 2 Oct 2025 18:07:53 +0530 Subject: [PATCH 15/15] Formatting changes --- lib/CppInterOp/Compatibility.h | 13 ++++++------- unittests/CppInterOp/InterpreterTest.cpp | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/CppInterOp/Compatibility.h b/lib/CppInterOp/Compatibility.h index a676ba5a1..864729da7 100644 --- a/lib/CppInterOp/Compatibility.h +++ b/lib/CppInterOp/Compatibility.h @@ -291,14 +291,13 @@ createClangInterpreter(std::vector& args, int stdin_fd = 0, setvbuf(stdout, nullptr, _IONBF, 0); setvbuf(stderr, nullptr, _IONBF, 0); }; - + #ifdef __APPLE__ - std::string OrcRuntimePath = - std::string(LLVM_BUILD_DIR) + "/lib/clang/" + - std::to_string(LLVM_VERSION_MAJOR) + "/lib/darwin/liborc_rt_osx.a"; + std::string OrcRuntimePath = std::string(LLVM_BUILD_DIR) + "/lib/clang/" + + std::to_string(LLVM_VERSION_MAJOR) + + "/lib/darwin/liborc_rt_osx.a"; #else - std::string OrcRuntimePath = std::string(LLVM_BUILD_DIR) + - "/lib/clang/" + + std::string OrcRuntimePath = std::string(LLVM_BUILD_DIR) + "/lib/clang/" + std::to_string(LLVM_VERSION_MAJOR) + "/lib/x86_64-unknown-linux-gnu/liborc_rt.a"; #endif @@ -462,7 +461,7 @@ class SynthesizingCodeRAII { "Failed to generate PTU:"); } }; -} +} // namespace compat #endif // CPPINTEROP_USE_REPL diff --git a/unittests/CppInterOp/InterpreterTest.cpp b/unittests/CppInterOp/InterpreterTest.cpp index 60a1bb682..51d4ea826 100644 --- a/unittests/CppInterOp/InterpreterTest.cpp +++ b/unittests/CppInterOp/InterpreterTest.cpp @@ -76,7 +76,7 @@ TEST(InterpreterTest, Evaluate) { GTEST_SKIP() << "Test fails for OOP JIT builds"; } // EXPECT_TRUE(Cpp::Evaluate(I, "") == 0); - //EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); + // EXPECT_TRUE(Cpp::Evaluate(I, "__cplusplus;") == 201402); // Due to a deficiency in the clang-repl implementation to get the value we // always must omit the ; TestUtils::CreateInterpreter();